charts more or less OK

NN/train_rl.py

@@ -34,7 +34,7 @@ class RLTradingEnvironment(gym.Env):
     Reinforcement Learning environment for trading with technical indicators
     from multiple timeframes
     """
-    def __init__(self, features_1m, features_5m, features_15m, window_size=20, trading_fee=0.001):
+    def __init__(self, features_1m, features_5m, features_15m, window_size=20, trading_fee=0.0025, min_trade_interval=15):
         super().__init__()
         
         # Initialize attributes before parent class
@@ -50,7 +50,8 @@ class RLTradingEnvironment(gym.Env):
         
         # Trading parameters
         self.initial_balance = 1.0
-        self.trading_fee = trading_fee
+        self.trading_fee = trading_fee  # Increased from 0.001 to 0.0025 (0.25%)
+        self.min_trade_interval = min_trade_interval  # Minimum steps between trades
         
         # Define action and observation spaces
         self.action_space = gym.spaces.Discrete(3)  # 0: Buy, 1: Sell, 2: Hold
@@ -76,6 +77,7 @@ class RLTradingEnvironment(gym.Env):
         self.wins = 0
         self.losses = 0
         self.trade_history = []
+        self.last_trade_step = -self.min_trade_interval  # Initialize to allow immediate first trade
         
         # Get initial observation
         observation = self._get_observation()
@@ -150,24 +152,40 @@ class RLTradingEnvironment(gym.Env):
         done = False
         profit_pct = None  # Initialize profit_pct variable
         
+        # Check if enough time has passed since last trade
+        trade_interval = self.current_step - self.last_trade_step
+        trade_interval_penalty = 0
+        
         # Execute action
         if action == 0:  # BUY
             if self.position == 0:  # Only buy if not already in position
+                # Apply extra penalty for trading too frequently
+                if trade_interval < self.min_trade_interval:
+                    trade_interval_penalty = -0.002 * (self.min_trade_interval - trade_interval)
+                    # Still allow the trade but with penalty
+                
                 self.position = self.balance * (1 - self.trading_fee)
                 self.balance = 0
                 self.trades += 1
-                reward = 0  # Neutral reward for entering position
+                reward = -0.001 + trade_interval_penalty  # Small cost for transaction + potential penalty
                 self.trade_entry_price = current_price
+                self.last_trade_step = self.current_step
                 
         elif action == 1:  # SELL
             if self.position > 0:  # Only sell if in position
+                # Apply extra penalty for trading too frequently
+                if trade_interval < self.min_trade_interval:
+                    trade_interval_penalty = -0.002 * (self.min_trade_interval - trade_interval)
+                    # Still allow the trade but with penalty
+                
                 # Calculate position value at current price
                 position_value = self.position * (1 + price_change)
                 self.balance = position_value * (1 - self.trading_fee)
                 
                 # Calculate profit/loss from trade
                 profit_pct = (next_price - self.trade_entry_price) / self.trade_entry_price
-                reward = profit_pct * 10  # Scale reward by profit percentage
+                # Scale reward by profit percentage and apply trade interval penalty
+                reward = (profit_pct * 10) + trade_interval_penalty
                 
                 # Update win/loss count
                 if profit_pct > 0:
@@ -179,11 +197,13 @@ class RLTradingEnvironment(gym.Env):
                 self.trade_history.append({
                     'entry_price': self.trade_entry_price,
                     'exit_price': next_price,
-                    'profit_pct': profit_pct
+                    'profit_pct': profit_pct,
+                    'trade_interval': trade_interval
                 })
                 
-                # Reset position
+                # Reset position and update last trade step
                 self.position = 0
+                self.last_trade_step = self.current_step
         
         # else: (action == 2 - HOLD) - no position change