popov/gogo2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

training fixes

Browse Source
This commit is contained in:
Dobromir Popov
2025-06-26 14:18:04 +03:00
parent b7ccd0f97b
commit 1f47576723
5 changed files with 291 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
.vscode/tasks.json vendored
View File

@ -6,14 +6,18 @@
"type": "shell",
"command": "python",
"args": [
"-c",
"import psutil; [p.kill() for p in psutil.process_iter() if any(x in p.name().lower() for x in [\"python\", \"tensorboard\"]) and any(x in \" \".join(p.cmdline()) for x in [\"scalping\", \"training\", \"tensorboard\"]) and p.pid != psutil.Process().pid]; print(\"Stale processes killed\")"
"scripts/kill_stale_processes.py"
],
"presentation": {
"reveal": "silent",
"panel": "shared"
"reveal": "always",
"panel": "shared",
"clear": true
},
"problemMatcher": []
"problemMatcher": [],
"group": {
"kind": "build",
"isDefault": false
}
},
{
"label": "Start TensorBoard",

6
core/orchestrator.py
View File

@ -168,7 +168,10 @@ class TradingOrchestrator:
# Initialize CNN Model
try:
from NN.models.enhanced_cnn import EnhancedCNN
self.cnn_model = EnhancedCNN()
# CNN model expects input_shape and n_actions parameters
cnn_input_shape = self.config.cnn.get('input_shape', 100)
cnn_n_actions = self.config.cnn.get('n_actions', 3)
self.cnn_model = EnhancedCNN(input_shape=cnn_input_shape, n_actions=cnn_n_actions)
# Load best checkpoint and capture initial state
if hasattr(self.cnn_model, 'load_best_checkpoint'):
@ -275,6 +278,7 @@ class TradingOrchestrator:
except Exception as e:
logger.error(f"Error initializing COB integration: {e}")
logger.info("COB integration will be disabled - dashboard will run with basic price data")
self.cob_integration = None
async def _start_cob_integration(self):

156
scripts/kill_stale_processes.py Normal file
View File

@ -0,0 +1,156 @@
#!/usr/bin/env python3
"""
Kill Stale Processes Script
Safely terminates stale Python processes related to the trading dashboard
with proper error handling and graceful termination.
"""
import os
import sys
import time
import signal
from pathlib import Path
def kill_stale_processes():
"""Kill stale trading dashboard processes safely"""
try:
import psutil
except ImportError:
print("psutil not available - using fallback method")
return kill_stale_fallback()
current_pid = os.getpid()
killed_processes = []
failed_processes = []
# Keywords to identify trading dashboard processes
target_keywords = [
'dashboard', 'scalping', 'trading', 'tensorboard',
'run_clean', 'run_main', 'gogo2', 'mexc'
]
try:
print("Scanning for stale processes...")
# Get all Python processes
python_processes = []
for proc in psutil.process_iter(['pid', 'name', 'cmdline']):
try:
if proc.info['pid'] == current_pid:
continue
name = proc.info['name'].lower()
if 'python' in name or 'tensorboard' in name:
cmdline_str = ' '.join(proc.info['cmdline']) if proc.info['cmdline'] else ''
# Check if this is a target process
if any(keyword in cmdline_str.lower() for keyword in target_keywords):
python_processes.append({
'proc': proc,
'pid': proc.info['pid'],
'name': proc.info['name'],
'cmdline': cmdline_str
})
except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
continue
if not python_processes:
print("No stale processes found")
return True
print(f"Found {len(python_processes)} target processes to terminate:")
for p in python_processes:
print(f" - PID {p['pid']}: {p['name']} - {p['cmdline'][:80]}...")
# Graceful termination first
print("\nAttempting graceful termination...")
for p in python_processes:
try:
proc = p['proc']
if proc.is_running():
proc.terminate()
print(f" Sent SIGTERM to PID {p['pid']}")
except Exception as e:
failed_processes.append(f"Failed to terminate PID {p['pid']}: {e}")
# Wait for graceful shutdown
time.sleep(2.0)
# Force kill remaining processes
print("\nChecking for remaining processes...")
for p in python_processes:
try:
proc = p['proc']
if proc.is_running():
print(f" Force killing PID {p['pid']} ({p['name']})")
proc.kill()
killed_processes.append(f"Force killed PID {p['pid']} ({p['name']})")
else:
killed_processes.append(f"Gracefully terminated PID {p['pid']} ({p['name']})")
except (psutil.NoSuchProcess, psutil.AccessDenied):
killed_processes.append(f"Process PID {p['pid']} already terminated")
except Exception as e:
failed_processes.append(f"Failed to kill PID {p['pid']}: {e}")
# Results
print(f"\n=== Process Cleanup Results ===")
if killed_processes:
print(f"Successfully cleaned up {len(killed_processes)} processes:")
for msg in killed_processes:
print(f" ✓ {msg}")
if failed_processes:
print(f"\nFailed to clean up {len(failed_processes)} processes:")
for msg in failed_processes:
print(f" ✗ {msg}")
print(f"\nCleanup completed. {len(killed_processes)} processes terminated.")
return len(failed_processes) == 0
except Exception as e:
print(f"Error during process cleanup: {e}")
return False
def kill_stale_fallback():
"""Fallback method using basic OS commands"""
print("Using fallback process killing method...")
try:
if os.name == 'nt': # Windows
import subprocess
# Kill Python processes with dashboard keywords
result = subprocess.run([
'taskkill', '/f', '/im', 'python.exe'
], capture_output=True, text=True)
if result.returncode == 0:
print("Windows: Killed all Python processes")
else:
print("Windows: No Python processes to kill or access denied")
else: # Unix/Linux
import subprocess
# More targeted approach for Unix
subprocess.run(['pkill', '-f', 'dashboard'], capture_output=True)
subprocess.run(['pkill', '-f', 'scalping'], capture_output=True)
subprocess.run(['pkill', '-f', 'tensorboard'], capture_output=True)
print("Unix: Killed dashboard-related processes")
return True
except Exception as e:
print(f"Fallback method failed: {e}")
return False
if __name__ == "__main__":
print("=" * 50)
print("STALE PROCESS CLEANUP")
print("=" * 50)
success = kill_stale_processes()
exit_code = 0 if success else 1
print("=" * 50)
sys.exit(exit_code)

50
testcases/negative/case_index.json
View File

@ -33,7 +33,55 @@
"technical_indicators": 7,
"price_history": 50
}
},
{
"case_id": "negative_20250626_140647_ETHUSDT_pnl_neg0p0220",
"timestamp": "2025-06-26T14:04:41.195630",
"symbol": "ETH/USDT",
"pnl": -0.02201592485230835,
"training_priority": 2,
"retraining_count": 0,
"feature_counts": {
"market_state": 0,
"cnn_features": 0,
"dqn_state": 2,
"cob_features": 0,
"technical_indicators": 7,
"price_history": 50
}
},
{
"case_id": "negative_20250626_140726_ETHUSDT_pnl_neg0p0220",
"timestamp": "2025-06-26T14:04:41.195630",
"symbol": "ETH/USDT",
"pnl": -0.02201592485230835,
"training_priority": 2,
"retraining_count": 0,
"feature_counts": {
"market_state": 0,
"cnn_features": 0,
"dqn_state": 2,
"cob_features": 0,
"technical_indicators": 7,
"price_history": 50
}
},
{
"case_id": "negative_20250626_140824_ETHUSDT_pnl_neg0p0071",
"timestamp": "2025-06-26T14:07:26.180914",
"symbol": "ETH/USDT",
"pnl": -0.007136478005372933,
"training_priority": 2,
"retraining_count": 0,
"feature_counts": {
"market_state": 0,
"cnn_features": 0,
"dqn_state": 2,
"cob_features": 0,
"technical_indicators": 7,
"price_history": 50
}
}
],
"last_updated": "2025-06-26T00:56:40.944179"
"last_updated": "2025-06-26T14:08:24.042558"
}

101
web/clean_dashboard.py
View File

@ -40,6 +40,7 @@ from threading import Lock
import warnings
from dataclasses import asdict
import math
import subprocess
# Setup logger
logger = logging.getLogger(__name__)
@ -258,18 +259,19 @@ class CleanTradingDashboard:
def _setup_callbacks(self):
"""Setup dashboard callbacks"""
# Callbacks setup - no process killing needed
@self.app.callback(
[Output('current-price', 'children'),
Output('session-pnl', 'children'),
Output('current-position', 'children'),
# Output('leverage-info', 'children'),
Output('trade-count', 'children'),
Output('portfolio-value', 'children'),
Output('mexc-status', 'children')],
[Input('interval-component', 'n_intervals')]
)
def update_metrics(n):
"""Update key metrics"""
"""Update key metrics - FIXED callback mismatch"""
try:
# Sync position from trading executor first
symbol = 'ETH/USDT'
@ -712,27 +714,44 @@ class CleanTradingDashboard:
buy_trades = []
sell_trades = []
for signal in executed_signals[-20:]: # Last 20 executed trades
signal_time = self._get_signal_attribute(signal, 'timestamp')
for signal in executed_signals[-50:]: # Last 50 executed trades (increased from 20)
# Try to get full timestamp first, fall back to string timestamp
signal_time = self._get_signal_attribute(signal, 'full_timestamp')
if not signal_time:
signal_time = self._get_signal_attribute(signal, 'timestamp')
signal_price = self._get_signal_attribute(signal, 'price', 0)
signal_action = self._get_signal_attribute(signal, 'action', 'HOLD')
signal_confidence = self._get_signal_attribute(signal, 'confidence', 0)
if signal_time and signal_price and signal_confidence > 0:
# Convert timestamp if needed
# FIXED: Better timestamp conversion to prevent race conditions
if isinstance(signal_time, str):
try:
# Handle time-only format
# Handle time-only format with current date
if ':' in signal_time and len(signal_time.split(':')) == 3:
signal_time = datetime.now().replace(
hour=int(signal_time.split(':')[0]),
minute=int(signal_time.split(':')[1]),
second=int(signal_time.split(':')[2]),
now = datetime.now()
time_parts = signal_time.split(':')
signal_time = now.replace(
hour=int(time_parts[0]),
minute=int(time_parts[1]),
second=int(time_parts[2]),
microsecond=0
)
# Handle day boundary issues - if signal seems from future, subtract a day
if signal_time > now + timedelta(minutes=5):
signal_time -= timedelta(days=1)
else:
signal_time = pd.to_datetime(signal_time)
except:
except Exception as e:
logger.debug(f"Error parsing timestamp {signal_time}: {e}")
continue
elif not isinstance(signal_time, datetime):
# Convert other timestamp formats to datetime
try:
signal_time = pd.to_datetime(signal_time)
except Exception as e:
logger.debug(f"Error converting timestamp to datetime: {e}")
continue
if signal_action == 'BUY':
@ -797,34 +816,51 @@ class CleanTradingDashboard:
if not self.recent_decisions:
return
# Show ALL signals on the mini chart
all_signals = self.recent_decisions[-50:] # Last 50 signals
# Show ALL signals on the mini chart - MORE SIGNALS for better visibility
all_signals = self.recent_decisions[-100:] # Last 100 signals (increased from 50)
buy_signals = []
sell_signals = []
for signal in all_signals:
signal_time = self._get_signal_attribute(signal, 'timestamp')
# Try to get full timestamp first, fall back to string timestamp
signal_time = self._get_signal_attribute(signal, 'full_timestamp')
if not signal_time:
signal_time = self._get_signal_attribute(signal, 'timestamp')
signal_price = self._get_signal_attribute(signal, 'price', 0)
signal_action = self._get_signal_attribute(signal, 'action', 'HOLD')
signal_confidence = self._get_signal_attribute(signal, 'confidence', 0)
is_executed = self._get_signal_attribute(signal, 'executed', False)
if signal_time and signal_price and signal_confidence and signal_confidence > 0:
# Convert timestamp if needed
# FIXED: Same timestamp conversion as main chart
if isinstance(signal_time, str):
try:
# Handle time-only format
# Handle time-only format with current date
if ':' in signal_time and len(signal_time.split(':')) == 3:
signal_time = datetime.now().replace(
hour=int(signal_time.split(':')[0]),
minute=int(signal_time.split(':')[1]),
second=int(signal_time.split(':')[2]),
now = datetime.now()
time_parts = signal_time.split(':')
signal_time = now.replace(
hour=int(time_parts[0]),
minute=int(time_parts[1]),
second=int(time_parts[2]),
microsecond=0
)
# Handle day boundary issues
if signal_time > now + timedelta(minutes=5):
signal_time -= timedelta(days=1)
else:
signal_time = pd.to_datetime(signal_time)
except:
except Exception as e:
logger.debug(f"Error parsing mini chart timestamp {signal_time}: {e}")
continue
elif not isinstance(signal_time, datetime):
# Convert other timestamp formats to datetime
try:
signal_time = pd.to_datetime(signal_time)
except Exception as e:
logger.debug(f"Error converting mini chart timestamp to datetime: {e}")
continue
signal_data = {
@ -1543,7 +1579,7 @@ class CleanTradingDashboard:
next_pivot_price = recent_low + (price_range * 0.5) # Mid-range target
confidence = base_confidence + cob_confidence_boost
# Calculate time prediction (in minutes)
# Calculate time prediction (in minutes)
try:
recent_closes = [float(x) for x in closes[-20:]]
if len(recent_closes) > 1:
@ -1659,12 +1695,14 @@ class CleanTradingDashboard:
# Don't generate HOLD signals - return None instead
return None
now = datetime.now()
return {
'action': action,
'symbol': symbol,
'price': current_price,
'confidence': confidence,
'timestamp': datetime.now().strftime('%H:%M:%S'),
'timestamp': now.strftime('%H:%M:%S'),
'full_timestamp': now, # Add full timestamp for chart persistence
'size': 0.005,
'reason': f'Momentum signal (s={short_momentum:.4f}, m={medium_momentum:.4f})',
'model': 'Momentum'
@ -1937,9 +1975,11 @@ class CleanTradingDashboard:
logger.warning(f"Failed to capture model inputs with COB data: {e}")
model_inputs = {}
# Create manual trading decision
# Create manual trading decision with FULL TIMESTAMP for chart persistence
now = datetime.now()
decision = {
'timestamp': datetime.now().strftime('%H:%M:%S'),
'timestamp': now.strftime('%H:%M:%S'),
'full_timestamp': now, # Store full datetime for better chart positioning
'action': action,
'confidence': 1.0, # Manual trades have 100% confidence
'price': current_price,
@ -3124,12 +3164,14 @@ class CleanTradingDashboard:
logger.debug(f"Ignoring BTC signal: {symbol}")
return
# Convert orchestrator decision to dashboard format
# Convert orchestrator decision to dashboard format with FULL TIMESTAMP
# Handle both TradingDecision objects and dictionary formats
now = datetime.now()
if hasattr(decision, 'action'):
# This is a TradingDecision object (dataclass)
dashboard_decision = {
'timestamp': datetime.now().strftime('%H:%M:%S'),
'timestamp': now.strftime('%H:%M:%S'),
'full_timestamp': now, # Add full timestamp for chart persistence
'action': decision.action,
'confidence': decision.confidence,
'price': decision.price,
@ -3141,7 +3183,8 @@ class CleanTradingDashboard:
else:
# This is a dictionary format
dashboard_decision = {
'timestamp': datetime.now().strftime('%H:%M:%S'),
'timestamp': now.strftime('%H:%M:%S'),
'full_timestamp': now, # Add full timestamp for chart persistence
'action': decision.get('action', 'UNKNOWN'),
'confidence': decision.get('confidence', 0),
'price': decision.get('price', 0),
@ -3359,7 +3402,7 @@ class CleanTradingDashboard:
if not self.orchestrator:
logger.warning("No orchestrator available for training")
return
# Check if DQN needs training
dqn_status = self._is_model_actually_training('dqn')
if not dqn_status['is_training'] and hasattr(self.orchestrator, 'rl_agent') and self.orchestrator.rl_agent: