From 1f4757672344a046aa4200a9b2c08a27f36cc1f8 Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Thu, 26 Jun 2025 14:18:04 +0300 Subject: [PATCH] training fixes --- .vscode/tasks.json | 14 ++- core/orchestrator.py | 6 +- scripts/kill_stale_processes.py | 156 +++++++++++++++++++++++++++++ testcases/negative/case_index.json | 50 ++++++++- web/clean_dashboard.py | 101 +++++++++++++------ 5 files changed, 291 insertions(+), 36 deletions(-) create mode 100644 scripts/kill_stale_processes.py diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 29fc0eb..ed31470 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -6,14 +6,18 @@ "type": "shell", "command": "python", "args": [ - "-c", - "import psutil; [p.kill() for p in psutil.process_iter() if any(x in p.name().lower() for x in [\"python\", \"tensorboard\"]) and any(x in \" \".join(p.cmdline()) for x in [\"scalping\", \"training\", \"tensorboard\"]) and p.pid != psutil.Process().pid]; print(\"Stale processes killed\")" + "scripts/kill_stale_processes.py" ], "presentation": { - "reveal": "silent", - "panel": "shared" + "reveal": "always", + "panel": "shared", + "clear": true }, - "problemMatcher": [] + "problemMatcher": [], + "group": { + "kind": "build", + "isDefault": false + } }, { "label": "Start TensorBoard", diff --git a/core/orchestrator.py b/core/orchestrator.py index 0275e0b..8350381 100644 --- a/core/orchestrator.py +++ b/core/orchestrator.py @@ -168,7 +168,10 @@ class TradingOrchestrator: # Initialize CNN Model try: from NN.models.enhanced_cnn import EnhancedCNN - self.cnn_model = EnhancedCNN() + # CNN model expects input_shape and n_actions parameters + cnn_input_shape = self.config.cnn.get('input_shape', 100) + cnn_n_actions = self.config.cnn.get('n_actions', 3) + self.cnn_model = EnhancedCNN(input_shape=cnn_input_shape, n_actions=cnn_n_actions) # Load best checkpoint and capture initial state if hasattr(self.cnn_model, 'load_best_checkpoint'): @@ -275,6 +278,7 @@ class TradingOrchestrator: except Exception as e: logger.error(f"Error initializing COB integration: {e}") + logger.info("COB integration will be disabled - dashboard will run with basic price data") self.cob_integration = None async def _start_cob_integration(self): diff --git a/scripts/kill_stale_processes.py b/scripts/kill_stale_processes.py new file mode 100644 index 0000000..f413689 --- /dev/null +++ b/scripts/kill_stale_processes.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +""" +Kill Stale Processes Script + +Safely terminates stale Python processes related to the trading dashboard +with proper error handling and graceful termination. +""" + +import os +import sys +import time +import signal +from pathlib import Path + +def kill_stale_processes(): + """Kill stale trading dashboard processes safely""" + try: + import psutil + except ImportError: + print("psutil not available - using fallback method") + return kill_stale_fallback() + + current_pid = os.getpid() + killed_processes = [] + failed_processes = [] + + # Keywords to identify trading dashboard processes + target_keywords = [ + 'dashboard', 'scalping', 'trading', 'tensorboard', + 'run_clean', 'run_main', 'gogo2', 'mexc' + ] + + try: + print("Scanning for stale processes...") + + # Get all Python processes + python_processes = [] + for proc in psutil.process_iter(['pid', 'name', 'cmdline']): + try: + if proc.info['pid'] == current_pid: + continue + + name = proc.info['name'].lower() + if 'python' in name or 'tensorboard' in name: + cmdline_str = ' '.join(proc.info['cmdline']) if proc.info['cmdline'] else '' + + # Check if this is a target process + if any(keyword in cmdline_str.lower() for keyword in target_keywords): + python_processes.append({ + 'proc': proc, + 'pid': proc.info['pid'], + 'name': proc.info['name'], + 'cmdline': cmdline_str + }) + + except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): + continue + + if not python_processes: + print("No stale processes found") + return True + + print(f"Found {len(python_processes)} target processes to terminate:") + for p in python_processes: + print(f" - PID {p['pid']}: {p['name']} - {p['cmdline'][:80]}...") + + # Graceful termination first + print("\nAttempting graceful termination...") + for p in python_processes: + try: + proc = p['proc'] + if proc.is_running(): + proc.terminate() + print(f" Sent SIGTERM to PID {p['pid']}") + except Exception as e: + failed_processes.append(f"Failed to terminate PID {p['pid']}: {e}") + + # Wait for graceful shutdown + time.sleep(2.0) + + # Force kill remaining processes + print("\nChecking for remaining processes...") + for p in python_processes: + try: + proc = p['proc'] + if proc.is_running(): + print(f" Force killing PID {p['pid']} ({p['name']})") + proc.kill() + killed_processes.append(f"Force killed PID {p['pid']} ({p['name']})") + else: + killed_processes.append(f"Gracefully terminated PID {p['pid']} ({p['name']})") + except (psutil.NoSuchProcess, psutil.AccessDenied): + killed_processes.append(f"Process PID {p['pid']} already terminated") + except Exception as e: + failed_processes.append(f"Failed to kill PID {p['pid']}: {e}") + + # Results + print(f"\n=== Process Cleanup Results ===") + if killed_processes: + print(f"Successfully cleaned up {len(killed_processes)} processes:") + for msg in killed_processes: + print(f" ✓ {msg}") + + if failed_processes: + print(f"\nFailed to clean up {len(failed_processes)} processes:") + for msg in failed_processes: + print(f" ✗ {msg}") + + print(f"\nCleanup completed. {len(killed_processes)} processes terminated.") + return len(failed_processes) == 0 + + except Exception as e: + print(f"Error during process cleanup: {e}") + return False + +def kill_stale_fallback(): + """Fallback method using basic OS commands""" + print("Using fallback process killing method...") + + try: + if os.name == 'nt': # Windows + import subprocess + # Kill Python processes with dashboard keywords + result = subprocess.run([ + 'taskkill', '/f', '/im', 'python.exe' + ], capture_output=True, text=True) + + if result.returncode == 0: + print("Windows: Killed all Python processes") + else: + print("Windows: No Python processes to kill or access denied") + + else: # Unix/Linux + import subprocess + # More targeted approach for Unix + subprocess.run(['pkill', '-f', 'dashboard'], capture_output=True) + subprocess.run(['pkill', '-f', 'scalping'], capture_output=True) + subprocess.run(['pkill', '-f', 'tensorboard'], capture_output=True) + print("Unix: Killed dashboard-related processes") + + return True + + except Exception as e: + print(f"Fallback method failed: {e}") + return False + +if __name__ == "__main__": + print("=" * 50) + print("STALE PROCESS CLEANUP") + print("=" * 50) + + success = kill_stale_processes() + exit_code = 0 if success else 1 + + print("=" * 50) + sys.exit(exit_code) \ No newline at end of file diff --git a/testcases/negative/case_index.json b/testcases/negative/case_index.json index 6197301..f3545eb 100644 --- a/testcases/negative/case_index.json +++ b/testcases/negative/case_index.json @@ -33,7 +33,55 @@ "technical_indicators": 7, "price_history": 50 } + }, + { + "case_id": "negative_20250626_140647_ETHUSDT_pnl_neg0p0220", + "timestamp": "2025-06-26T14:04:41.195630", + "symbol": "ETH/USDT", + "pnl": -0.02201592485230835, + "training_priority": 2, + "retraining_count": 0, + "feature_counts": { + "market_state": 0, + "cnn_features": 0, + "dqn_state": 2, + "cob_features": 0, + "technical_indicators": 7, + "price_history": 50 + } + }, + { + "case_id": "negative_20250626_140726_ETHUSDT_pnl_neg0p0220", + "timestamp": "2025-06-26T14:04:41.195630", + "symbol": "ETH/USDT", + "pnl": -0.02201592485230835, + "training_priority": 2, + "retraining_count": 0, + "feature_counts": { + "market_state": 0, + "cnn_features": 0, + "dqn_state": 2, + "cob_features": 0, + "technical_indicators": 7, + "price_history": 50 + } + }, + { + "case_id": "negative_20250626_140824_ETHUSDT_pnl_neg0p0071", + "timestamp": "2025-06-26T14:07:26.180914", + "symbol": "ETH/USDT", + "pnl": -0.007136478005372933, + "training_priority": 2, + "retraining_count": 0, + "feature_counts": { + "market_state": 0, + "cnn_features": 0, + "dqn_state": 2, + "cob_features": 0, + "technical_indicators": 7, + "price_history": 50 + } } ], - "last_updated": "2025-06-26T00:56:40.944179" + "last_updated": "2025-06-26T14:08:24.042558" } \ No newline at end of file diff --git a/web/clean_dashboard.py b/web/clean_dashboard.py index a826085..a208b73 100644 --- a/web/clean_dashboard.py +++ b/web/clean_dashboard.py @@ -40,6 +40,7 @@ from threading import Lock import warnings from dataclasses import asdict import math +import subprocess # Setup logger logger = logging.getLogger(__name__) @@ -258,18 +259,19 @@ class CleanTradingDashboard: def _setup_callbacks(self): """Setup dashboard callbacks""" + # Callbacks setup - no process killing needed + @self.app.callback( [Output('current-price', 'children'), Output('session-pnl', 'children'), Output('current-position', 'children'), - # Output('leverage-info', 'children'), Output('trade-count', 'children'), Output('portfolio-value', 'children'), Output('mexc-status', 'children')], [Input('interval-component', 'n_intervals')] ) def update_metrics(n): - """Update key metrics""" + """Update key metrics - FIXED callback mismatch""" try: # Sync position from trading executor first symbol = 'ETH/USDT' @@ -712,27 +714,44 @@ class CleanTradingDashboard: buy_trades = [] sell_trades = [] - for signal in executed_signals[-20:]: # Last 20 executed trades - signal_time = self._get_signal_attribute(signal, 'timestamp') + for signal in executed_signals[-50:]: # Last 50 executed trades (increased from 20) + # Try to get full timestamp first, fall back to string timestamp + signal_time = self._get_signal_attribute(signal, 'full_timestamp') + if not signal_time: + signal_time = self._get_signal_attribute(signal, 'timestamp') + signal_price = self._get_signal_attribute(signal, 'price', 0) signal_action = self._get_signal_attribute(signal, 'action', 'HOLD') signal_confidence = self._get_signal_attribute(signal, 'confidence', 0) if signal_time and signal_price and signal_confidence > 0: - # Convert timestamp if needed + # FIXED: Better timestamp conversion to prevent race conditions if isinstance(signal_time, str): try: - # Handle time-only format + # Handle time-only format with current date if ':' in signal_time and len(signal_time.split(':')) == 3: - signal_time = datetime.now().replace( - hour=int(signal_time.split(':')[0]), - minute=int(signal_time.split(':')[1]), - second=int(signal_time.split(':')[2]), + now = datetime.now() + time_parts = signal_time.split(':') + signal_time = now.replace( + hour=int(time_parts[0]), + minute=int(time_parts[1]), + second=int(time_parts[2]), microsecond=0 ) + # Handle day boundary issues - if signal seems from future, subtract a day + if signal_time > now + timedelta(minutes=5): + signal_time -= timedelta(days=1) else: signal_time = pd.to_datetime(signal_time) - except: + except Exception as e: + logger.debug(f"Error parsing timestamp {signal_time}: {e}") + continue + elif not isinstance(signal_time, datetime): + # Convert other timestamp formats to datetime + try: + signal_time = pd.to_datetime(signal_time) + except Exception as e: + logger.debug(f"Error converting timestamp to datetime: {e}") continue if signal_action == 'BUY': @@ -797,34 +816,51 @@ class CleanTradingDashboard: if not self.recent_decisions: return - # Show ALL signals on the mini chart - all_signals = self.recent_decisions[-50:] # Last 50 signals + # Show ALL signals on the mini chart - MORE SIGNALS for better visibility + all_signals = self.recent_decisions[-100:] # Last 100 signals (increased from 50) buy_signals = [] sell_signals = [] for signal in all_signals: - signal_time = self._get_signal_attribute(signal, 'timestamp') + # Try to get full timestamp first, fall back to string timestamp + signal_time = self._get_signal_attribute(signal, 'full_timestamp') + if not signal_time: + signal_time = self._get_signal_attribute(signal, 'timestamp') + signal_price = self._get_signal_attribute(signal, 'price', 0) signal_action = self._get_signal_attribute(signal, 'action', 'HOLD') signal_confidence = self._get_signal_attribute(signal, 'confidence', 0) is_executed = self._get_signal_attribute(signal, 'executed', False) if signal_time and signal_price and signal_confidence and signal_confidence > 0: - # Convert timestamp if needed + # FIXED: Same timestamp conversion as main chart if isinstance(signal_time, str): try: - # Handle time-only format + # Handle time-only format with current date if ':' in signal_time and len(signal_time.split(':')) == 3: - signal_time = datetime.now().replace( - hour=int(signal_time.split(':')[0]), - minute=int(signal_time.split(':')[1]), - second=int(signal_time.split(':')[2]), + now = datetime.now() + time_parts = signal_time.split(':') + signal_time = now.replace( + hour=int(time_parts[0]), + minute=int(time_parts[1]), + second=int(time_parts[2]), microsecond=0 ) + # Handle day boundary issues + if signal_time > now + timedelta(minutes=5): + signal_time -= timedelta(days=1) else: signal_time = pd.to_datetime(signal_time) - except: + except Exception as e: + logger.debug(f"Error parsing mini chart timestamp {signal_time}: {e}") + continue + elif not isinstance(signal_time, datetime): + # Convert other timestamp formats to datetime + try: + signal_time = pd.to_datetime(signal_time) + except Exception as e: + logger.debug(f"Error converting mini chart timestamp to datetime: {e}") continue signal_data = { @@ -1543,7 +1579,7 @@ class CleanTradingDashboard: next_pivot_price = recent_low + (price_range * 0.5) # Mid-range target confidence = base_confidence + cob_confidence_boost - # Calculate time prediction (in minutes) + # Calculate time prediction (in minutes) try: recent_closes = [float(x) for x in closes[-20:]] if len(recent_closes) > 1: @@ -1659,12 +1695,14 @@ class CleanTradingDashboard: # Don't generate HOLD signals - return None instead return None + now = datetime.now() return { 'action': action, 'symbol': symbol, 'price': current_price, 'confidence': confidence, - 'timestamp': datetime.now().strftime('%H:%M:%S'), + 'timestamp': now.strftime('%H:%M:%S'), + 'full_timestamp': now, # Add full timestamp for chart persistence 'size': 0.005, 'reason': f'Momentum signal (s={short_momentum:.4f}, m={medium_momentum:.4f})', 'model': 'Momentum' @@ -1937,9 +1975,11 @@ class CleanTradingDashboard: logger.warning(f"Failed to capture model inputs with COB data: {e}") model_inputs = {} - # Create manual trading decision + # Create manual trading decision with FULL TIMESTAMP for chart persistence + now = datetime.now() decision = { - 'timestamp': datetime.now().strftime('%H:%M:%S'), + 'timestamp': now.strftime('%H:%M:%S'), + 'full_timestamp': now, # Store full datetime for better chart positioning 'action': action, 'confidence': 1.0, # Manual trades have 100% confidence 'price': current_price, @@ -3124,12 +3164,14 @@ class CleanTradingDashboard: logger.debug(f"Ignoring BTC signal: {symbol}") return - # Convert orchestrator decision to dashboard format + # Convert orchestrator decision to dashboard format with FULL TIMESTAMP # Handle both TradingDecision objects and dictionary formats + now = datetime.now() if hasattr(decision, 'action'): # This is a TradingDecision object (dataclass) dashboard_decision = { - 'timestamp': datetime.now().strftime('%H:%M:%S'), + 'timestamp': now.strftime('%H:%M:%S'), + 'full_timestamp': now, # Add full timestamp for chart persistence 'action': decision.action, 'confidence': decision.confidence, 'price': decision.price, @@ -3141,7 +3183,8 @@ class CleanTradingDashboard: else: # This is a dictionary format dashboard_decision = { - 'timestamp': datetime.now().strftime('%H:%M:%S'), + 'timestamp': now.strftime('%H:%M:%S'), + 'full_timestamp': now, # Add full timestamp for chart persistence 'action': decision.get('action', 'UNKNOWN'), 'confidence': decision.get('confidence', 0), 'price': decision.get('price', 0), @@ -3359,7 +3402,7 @@ class CleanTradingDashboard: if not self.orchestrator: logger.warning("No orchestrator available for training") return - + # Check if DQN needs training dqn_status = self._is_model_actually_training('dqn') if not dqn_status['is_training'] and hasattr(self.orchestrator, 'rl_agent') and self.orchestrator.rl_agent: