Best checkpoint file not found

.env

@@ -1,6 +1,7 @@
 # MEXC API Configuration (Spot Trading)
 MEXC_API_KEY=mx0vglhVPZeIJ32Qw1
 MEXC_SECRET_KEY=3bfe4bd99d5541e4a1bca87ab257cc7e  
+#3bfe4bd99d5541e4a1bca87ab257cc7e 45d0b3c26f2644f19bfb98b07741b2f5
 
 # BASE ENDPOINTS: https://api.mexc.com wss://wbs-api.mexc.com/ws !!! DO NOT CHANGE THIS
 

config.yaml.backup_20250702_202543

@@ -0,0 +1,292 @@
+# Enhanced Multi-Modal Trading System Configuration
+
+# System Settings
+system:
+  timezone: "Europe/Sofia"     # Configurable timezone for all timestamps
+  log_level: "INFO"            # DEBUG, INFO, WARNING, ERROR
+  session_timeout: 3600        # Session timeout in seconds
+
+# Trading Symbols Configuration
+# Primary trading pair: ETH/USDT (main signals generation)
+# Reference pair: BTC/USDT (correlation analysis only, no trading signals)
+symbols:
+  - "ETH/USDT"  # MAIN TRADING PAIR - Generate signals and execute trades
+  - "BTC/USDT"  # REFERENCE ONLY - For correlation analysis, no direct trading
+
+# Timeframes for ultra-fast scalping (500x leverage)
+timeframes:  
+  - "1s"  # Primary scalping timeframe  
+  - "1m"  # Short-term confirmation  
+  - "1h"  # Medium-term trend 
+  - "1d"  # Long-term direction
+
+# Data Provider Settings
+data:
+  provider: "binance"
+  cache_enabled: true
+  cache_dir: "cache"
+  historical_limit: 1000
+  real_time_enabled: true
+  websocket_reconnect: true
+  feature_engineering:
+    technical_indicators: true
+    market_regime_detection: true
+    volatility_analysis: true
+
+# Enhanced CNN Configuration
+cnn:
+  window_size: 20
+  features: ["open", "high", "low", "close", "volume"]
+  timeframes: ["1m", "5m", "15m", "1h", "4h", "1d"]
+  hidden_layers: [64, 128, 256]
+  dropout: 0.2
+  learning_rate: 0.001
+  batch_size: 32
+  epochs: 100
+  confidence_threshold: 0.6
+  early_stopping_patience: 10
+  model_dir: "models/enhanced_cnn"  # Ultra-fast scalping weights (500x leverage)
+  timeframe_importance:
+    "1s": 0.60   # Primary scalping signal
+    "1m": 0.20   # Short-term confirmation
+    "1h": 0.15   # Medium-term trend
+    "1d": 0.05   # Long-term direction (minimal)
+
+# Enhanced RL Agent Configuration
+rl:
+  state_size: 100  # Will be calculated dynamically based on features
+  action_space: 3  # BUY, HOLD, SELL
+  hidden_size: 256
+  epsilon: 1.0
+  epsilon_decay: 0.995
+  epsilon_min: 0.01
+  learning_rate: 0.0001
+  gamma: 0.99
+  memory_size: 10000
+  batch_size: 64
+  target_update_freq: 1000
+  buffer_size: 10000
+  model_dir: "models/enhanced_rl"
+  # Market regime adaptation
+  market_regime_weights:
+    trending: 1.2    # Higher confidence in trending markets
+    ranging: 0.8     # Lower confidence in ranging markets
+    volatile: 0.6    # Much lower confidence in volatile markets
+  # Prioritized experience replay
+  replay_alpha: 0.6  # Priority exponent
+  replay_beta: 0.4   # Importance sampling exponent
+
+# Enhanced Orchestrator Settings
+orchestrator:
+  # Model weights for decision combination
+  cnn_weight: 0.7      # Weight for CNN predictions
+  rl_weight: 0.3       # Weight for RL decisions
+  confidence_threshold: 0.20  # Lowered from 0.35 for low-volatility markets
+  confidence_threshold_close: 0.10  # Lowered from 0.15 for easier exits
+  decision_frequency: 30     # Seconds between decisions (faster)
+  
+  # Multi-symbol coordination
+  symbol_correlation_matrix:
+    "ETH/USDT-BTC/USDT": 0.85  # ETH-BTC correlation
+    
+  # Perfect move marking
+  perfect_move_threshold: 0.02  # 2% price change to mark as significant
+  perfect_move_buffer_size: 10000
+  
+  # RL evaluation settings
+  evaluation_delay: 3600  # Evaluate actions after 1 hour
+  reward_calculation:
+    success_multiplier: 10    # Reward for correct predictions
+    failure_penalty: 5        # Penalty for wrong predictions
+    confidence_scaling: true   # Scale rewards by confidence
+
+# Training Configuration
+training:
+  learning_rate: 0.001
+  batch_size: 32
+  epochs: 100
+  validation_split: 0.2
+  early_stopping_patience: 10
+  
+  # CNN specific training
+  cnn_training_interval: 3600    # Train CNN every hour (was 6 hours)
+  min_perfect_moves: 50          # Reduced from 200 for faster learning
+  
+  # RL specific training  
+  rl_training_interval: 300      # Train RL every 5 minutes (was 1 hour)
+  min_experiences: 50            # Reduced from 100 for faster learning
+  training_steps_per_cycle: 20   # Increased from 10 for more learning
+
+  model_type: "optimized_short_term"
+  use_realtime: true
+  use_ticks: true
+  checkpoint_dir: "NN/models/saved/realtime_ticks_checkpoints"
+  save_best_model: true
+  save_final_model: false  # We only want to keep the best performing model
+  
+  # Continuous learning settings
+  continuous_learning: true
+  learning_from_trades: true
+  pattern_recognition: true
+  retrospective_learning: true
+
+# Trading Execution
+trading:
+  max_position_size: 0.05    # Maximum position size (5% of balance)
+  stop_loss: 0.02           # 2% stop loss
+  take_profit: 0.05         # 5% take profit
+  trading_fee: 0.0005       # 0.05% trading fee (MEXC taker fee - fallback)
+  
+  # MEXC Fee Structure (asymmetrical) - Updated 2025-05-28
+  trading_fees:
+    maker: 0.0000          # 0.00% maker fee (adds liquidity)
+    taker: 0.0005          # 0.05% taker fee (takes liquidity)  
+    default: 0.0005        # Default fallback fee (taker rate)
+  
+  # Risk management
+  max_daily_trades: 20      # Maximum trades per day
+  max_concurrent_positions: 2  # Max positions across symbols
+  position_sizing:
+    confidence_scaling: true  # Scale position by confidence
+    base_size: 0.02          # 2% base position
+    max_size: 0.05           # 5% maximum position
+
+# MEXC Trading API Configuration
+mexc_trading:
+  enabled: true
+  trading_mode: simulation  # simulation, testnet, live
+  
+  # FIXED: Meaningful position sizes for learning
+  base_position_usd: 25.0      # $25 base position (was $1)
+  max_position_value_usd: 50.0  # $50 max position (was $1)
+  min_position_value_usd: 10.0  # $10 min position (was $0.10)
+  
+  # Risk management
+  max_daily_trades: 100
+  max_daily_loss_usd: 200.0
+  max_concurrent_positions: 3
+  min_trade_interval_seconds: 30
+  
+  # Order configuration
+  order_type: market  # market or limit
+  
+  # Enhanced fee structure for better calculation
+  trading_fees:
+    maker_fee: 0.0002    # 0.02% maker fee
+    taker_fee: 0.0006    # 0.06% taker fee
+    default_fee: 0.0006  # Default to taker fee
+
+# Memory Management
+memory:
+  total_limit_gb: 28.0       # Total system memory limit
+  model_limit_gb: 4.0       # Per-model memory limit
+  cleanup_interval: 1800    # Memory cleanup every 30 minutes
+  
+# Real-time RL COB Trader Configuration
+realtime_rl:
+  # Model parameters for 400M parameter network (faster startup)
+  model:
+    input_size: 2000          # COB feature dimensions
+    hidden_size: 2048         # Optimized hidden layer size for 400M params
+    num_layers: 8             # Efficient transformer layers for faster training
+    learning_rate: 0.0001     # Higher learning rate for faster convergence
+    weight_decay: 0.00001     # Balanced L2 regularization
+    
+  # Inference configuration  
+  inference_interval_ms: 200  # Inference every 200ms
+  min_confidence_threshold: 0.7  # Minimum confidence for signal accumulation
+  required_confident_predictions: 3  # Need 3 confident predictions for trade
+  
+  # Training configuration
+  training_interval_s: 1.0    # Train every second
+  batch_size: 32              # Training batch size
+  replay_buffer_size: 1000    # Store last 1000 predictions for training
+  
+  # Signal accumulation
+  signal_buffer_size: 10      # Buffer size for signal accumulation
+  consensus_threshold: 3      # Need 3 signals in same direction
+  
+  # Model checkpointing
+  model_checkpoint_dir: "models/realtime_rl_cob"
+  save_interval_s: 300        # Save models every 5 minutes
+  
+  # COB integration
+  symbols: ["BTC/USDT", "ETH/USDT"]  # Symbols to trade
+  cob_feature_normalization: "robust"  # Feature normalization method
+  
+  # Reward engineering for RL
+  reward_structure:
+    correct_direction_base: 1.0     # Base reward for correct prediction
+    confidence_scaling: true        # Scale reward by confidence
+    magnitude_bonus: 0.5           # Bonus for predicting magnitude accurately
+    overconfidence_penalty: 1.5    # Penalty multiplier for wrong high-confidence predictions
+    trade_execution_multiplier: 10.0  # Higher weight for actual trade outcomes
+    
+  # Performance monitoring
+  statistics_interval_s: 60   # Print stats every minute
+  detailed_logging: true      # Enable detailed performance logging
+
+# Web Dashboard
+web:
+  host: "127.0.0.1"
+  port: 8050
+  debug: false
+  update_interval: 500  # Milliseconds
+  chart_history: 200     # Number of candles to show
+  
+  # Enhanced dashboard features
+  show_timeframe_analysis: true
+  show_confidence_scores: true
+  show_perfect_moves: true
+  show_rl_metrics: true
+
+# Logging
+logging:
+  level: "INFO"
+  format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+  file: "logs/enhanced_trading.log"
+  max_size: 10485760  # 10MB
+  backup_count: 5
+  
+  # Component-specific logging
+  orchestrator_level: "INFO"
+  cnn_level: "INFO"
+  rl_level: "INFO"
+  training_level: "INFO"
+
+# Model Directories
+model_dir: "models"
+data_dir: "data"
+cache_dir: "cache"
+logs_dir: "logs"
+
+# GPU/Performance
+gpu:
+  enabled: true
+  memory_fraction: 0.8  # Use 80% of GPU memory
+  allow_growth: true    # Allow dynamic memory allocation
+  
+# Monitoring and Alerting
+monitoring:
+  tensorboard_enabled: true
+  tensorboard_log_dir: "logs/tensorboard"
+  metrics_interval: 300     # Log metrics every 5 minutes
+  performance_alerts: true
+  
+  # Performance thresholds
+  min_confidence_threshold: 0.3
+  max_memory_usage: 0.9     # 90% of available memory
+  max_decision_latency: 10  # 10 seconds max per decision
+
+# Backtesting (for future implementation)
+backtesting:
+  start_date: "2024-01-01"
+  end_date: "2024-12-31"
+  initial_balance: 10000
+  commission: 0.0002
+  slippage: 0.0001
+
+model_paths:
+  realtime_model: "NN/models/saved/optimized_short_term_model_realtime_best.pt"
+  ticks_model: "NN/models/saved/optimized_short_term_model_ticks_best.pt"
+  backup_model: "NN/models/saved/realtime_ticks_checkpoints/checkpoint_epoch_50449_backup/model.pt" 

tests/test_mexc_account_privte.py

@@ -0,0 +1,41 @@
+from NN.exchanges.mexc_interface import MEXCInterface
+import logging
+
+# Set up logging to see debug info
+logging.basicConfig(level=logging.INFO)
+
+# Create interface with API credentials
+mexc = MEXCInterface(
+    api_key='mx0aBYs33eIilxBWC5', 
+    api_secret='45d0b3c26f2644f19bfb98b07741b2f5',
+    trading_mode='simulation'
+)
+
+print('MEXC Interface created successfully')
+
+# Test signature generation
+import time
+timestamp = int(time.time() * 1000)
+test_params = 'quantity=1&price=11&symbol=BTCUSDT&side=BUY&type=LIMIT&timestamp=' + str(timestamp)
+signature = mexc._generate_signature(timestamp, test_params)
+print(f'Generated signature: {signature}')
+
+# Test account info
+print('Testing account info...')
+account_info = mexc.get_account_info()
+print(f'Account info result: {account_info}')
+
+# Test ticker data
+print('Testing ticker data...')
+ticker = mexc.get_ticker('ETH/USDT')
+print(f'ETH/USDT ticker: {ticker}')
+
+# Test balance retrieval
+print('Testing balance retrieval...')
+usdt_balance = mexc.get_balance('USDT')
+print(f'USDT balance: {usdt_balance}')
+
+# Test a small order placement (simulation mode)
+print('Testing order placement in simulation mode...')
+order_result = mexc.place_order('ETH/USDT', 'buy', 'market', 0.001)
+print(f'Order result: {order_result}')

utils/checkpoint_manager.py

@@ -87,7 +87,7 @@ class CheckpointManager:
             performance_score = self._calculate_performance_score(performance_metrics)
             
             if not force_save and not self._should_save_checkpoint(model_name, performance_score):
-                logger.info(f"Skipping checkpoint save for {model_name} - performance not improved")
+                logger.debug(f"Skipping checkpoint save for {model_name} - performance not improved")
                 return None
             
             success = self._save_model_file(model, checkpoint_path, model_type)
@@ -140,10 +140,11 @@ class CheckpointManager:
             best_checkpoint = max(self.checkpoints[model_name], key=lambda x: x.performance_score)
             
             if not Path(best_checkpoint.file_path).exists():
-                logger.error(f"Best checkpoint file not found: {best_checkpoint.file_path}")
+                # temporary disable logging to avoid spam 
+                # logger.error(f"Best checkpoint file not found: {best_checkpoint.file_path}")
                 return None
             
-            logger.info(f"Loading best checkpoint for {model_name}: {best_checkpoint.checkpoint_id}")
+            logger.debug(f"Loading best checkpoint for {model_name}: {best_checkpoint.checkpoint_id}")
             return best_checkpoint.file_path, best_checkpoint
             
         except Exception as e:

web/clean_dashboard.py

@@ -5474,7 +5474,7 @@ class CleanTradingDashboard:
             if training_samples > 0:
                 avg_loss_info = f", avg_loss={total_loss/loss_count:.6f}" if loss_count > 0 else ""
                 performance_score = 100 / (1 + (total_loss/loss_count)) if loss_count > 0 else 0.1
-                logger.info(f"DECISION TRAINING: Processed {training_samples} decision fusion samples{avg_loss_info}, perf_score={performance_score:.4f}")
+                logger.debug(f"DECISION TRAINING: Processed {training_samples} decision fusion samples{avg_loss_info}, perf_score={performance_score:.4f}")
         except Exception as e:
             logger.error(f"Error in real decision fusion training: {e}")