diff --git a/ANNOTATE/core/real_training_adapter.py b/ANNOTATE/core/real_training_adapter.py
index b42f5e7..0dc5750 100644
--- a/ANNOTATE/core/real_training_adapter.py
+++ b/ANNOTATE/core/real_training_adapter.py
@@ -32,18 +32,20 @@ except ImportError:
logger = logging.getLogger(__name__)
-def parse_timestamp_to_utc(timestamp_str: str) -> datetime:
+def parse_timestamp_to_utc(timestamp_str) -> datetime:
"""
Unified timestamp parser that handles all formats and ensures UTC timezone.
Handles:
+ - pandas Timestamp objects
+ - datetime objects
- ISO format with timezone: '2025-10-27T14:00:00+00:00'
- ISO format with Z: '2025-10-27T14:00:00Z'
- Space-separated with seconds: '2025-10-27 14:00:00'
- Space-separated without seconds: '2025-10-27 14:00'
Args:
- timestamp_str: Timestamp string in various formats
+ timestamp_str: Timestamp string, pandas Timestamp, or datetime object
Returns:
Timezone-aware datetime object in UTC
@@ -51,6 +53,23 @@ def parse_timestamp_to_utc(timestamp_str: str) -> datetime:
Raises:
ValueError: If timestamp cannot be parsed
"""
+ # Handle pandas Timestamp objects
+ if hasattr(timestamp_str, 'to_pydatetime'):
+ dt = timestamp_str.to_pydatetime()
+ if dt.tzinfo is None:
+ dt = dt.replace(tzinfo=timezone.utc)
+ return dt
+
+ # Handle datetime objects directly
+ if isinstance(timestamp_str, datetime):
+ if timestamp_str.tzinfo is None:
+ return timestamp_str.replace(tzinfo=timezone.utc)
+ return timestamp_str
+
+ # Convert to string if not already
+ if not isinstance(timestamp_str, str):
+ timestamp_str = str(timestamp_str)
+
if not timestamp_str:
raise ValueError("Empty timestamp string")
@@ -2445,7 +2464,8 @@ class RealTrainingAdapter:
def start_realtime_inference(self, model_name: str, symbol: str, data_provider,
enable_live_training: bool = True,
train_every_candle: bool = False,
- timeframe: str = '1m') -> str:
+ timeframe: str = '1m',
+ training_strategy = None) -> str:
"""
Start real-time inference using orchestrator's REAL prediction methods
@@ -2453,9 +2473,10 @@ class RealTrainingAdapter:
model_name: Name of model to use for inference
symbol: Trading symbol
data_provider: Data provider for market data
- enable_live_training: If True, automatically train on L2 pivots
- train_every_candle: If True, train on every new candle (computationally expensive)
+ enable_live_training: If True, automatically train (deprecated - use training_strategy)
+ train_every_candle: If True, train on every candle (deprecated - use training_strategy)
timeframe: Timeframe for candle-based training (default: 1m)
+ training_strategy: TrainingStrategyManager for making training decisions
Returns:
inference_id: Unique ID for this inference session
@@ -2482,6 +2503,8 @@ class RealTrainingAdapter:
'train_every_candle': train_every_candle,
'timeframe': timeframe,
'data_provider': data_provider,
+ 'training_strategy': training_strategy, # Strategy manager for training decisions
+ 'pending_action': None, # Action to train on (set by strategy manager)
'metrics': {
'accuracy': 0.0,
'loss': 0.0,
@@ -2585,10 +2608,18 @@ class RealTrainingAdapter:
# Extract action
action_probs = outputs.get('action_probs')
if action_probs is not None:
- action_idx = torch.argmax(action_probs, dim=-1).item()
- confidence = action_probs[0, action_idx].item()
+ # Handle different tensor shapes: [batch, 3] or [3]
+ if action_probs.dim() == 1:
+ # Shape [3] - single prediction
+ action_idx = torch.argmax(action_probs, dim=0).item()
+ confidence = action_probs[action_idx].item()
+ else:
+ # Shape [batch, 3] - take first batch item
+ action_idx = torch.argmax(action_probs[0], dim=0).item()
+ confidence = action_probs[0, action_idx].item()
- actions = ['BUY', 'SELL', 'HOLD']
+ # Map to action string (must match training: 0=HOLD, 1=BUY, 2=SELL)
+ actions = ['HOLD', 'BUY', 'SELL']
action = actions[action_idx] if action_idx < len(actions) else 'HOLD'
# Handle predicted candles - DENORMALIZE them
@@ -2613,21 +2644,29 @@ class RealTrainingAdapter:
# Note: raw_candle[0] is the list of 5 values
candle_values = raw_candle[0]
+ # Ensure all values are Python floats (not numpy scalars or tensors)
+ def to_float(v):
+ if hasattr(v, 'item'):
+ return float(v.item())
+ return float(v)
+
denorm_candle = [
- candle_values[0] * (price_max - price_min) + price_min, # Open
- candle_values[1] * (price_max - price_min) + price_min, # High
- candle_values[2] * (price_max - price_min) + price_min, # Low
- candle_values[3] * (price_max - price_min) + price_min, # Close
- candle_values[4] * (vol_max - vol_min) + vol_min # Volume
+ to_float(candle_values[0] * (price_max - price_min) + price_min), # Open
+ to_float(candle_values[1] * (price_max - price_min) + price_min), # High
+ to_float(candle_values[2] * (price_max - price_min) + price_min), # Low
+ to_float(candle_values[3] * (price_max - price_min) + price_min), # Close
+ to_float(candle_values[4] * (vol_max - vol_min) + vol_min) # Volume
]
predicted_candles_denorm[tf] = denorm_candle
- # Calculate predicted price from candle close
+ # Calculate predicted price from candle close (ensure Python float)
predicted_price = None
if '1m' in predicted_candles_denorm:
- predicted_price = predicted_candles_denorm['1m'][3] # Close price
+ close_val = predicted_candles_denorm['1m'][3]
+ predicted_price = float(close_val.item() if hasattr(close_val, 'item') else close_val)
elif '1s' in predicted_candles_denorm:
- predicted_price = predicted_candles_denorm['1s'][3]
+ close_val = predicted_candles_denorm['1s'][3]
+ predicted_price = float(close_val.item() if hasattr(close_val, 'item') else close_val)
elif outputs.get('price_prediction') is not None:
# Fallback to price_prediction head if available (normalized)
# This would need separate denormalization based on reference price
@@ -2755,42 +2794,61 @@ class RealTrainingAdapter:
logger.debug(traceback.format_exc())
return None, None
- def _train_on_new_candle(self, session: Dict, symbol: str, timeframe: str, data_provider):
- """Train model on new candle when it closes"""
+ def _train_on_new_candle(self, session: Dict, symbol: str, timeframe: str, data_provider) -> Dict:
+ """
+ Train model on new candle - Pure model interface with NO business logic
+
+ Args:
+ session: Training session containing pending_action set by app
+ symbol: Trading symbol
+ timeframe: Timeframe for training
+ data_provider: Data provider for fetching candles
+
+ Returns:
+ Dict with training metrics: {loss, accuracy, training_steps}
+ """
try:
- # Get latest candle
+ # Get latest candles
df = data_provider.get_historical_data(symbol, timeframe, limit=2)
if df is None or len(df) < 2:
- return
+ return {'success': False, 'error': 'Insufficient data'}
# Check if we have a new candle
latest_candle_time = df.index[-1]
if session['last_candle_time'] == latest_candle_time:
- return # Same candle, no training needed
+ return {'success': False, 'error': 'Same candle, no training needed'}
logger.debug(f"New candle detected: {latest_candle_time} (last: {session['last_candle_time']})")
session['last_candle_time'] = latest_candle_time
- # Get the completed candle (second to last)
+ # Get the completed candle (second to last) and next candle
completed_candle = df.iloc[-2]
next_candle = df.iloc[-1]
- # Calculate if the prediction would have been correct
+ # Get action from session (set by app's training strategy)
+ action_label = session.get('pending_action')
+ if not action_label:
+ return {'success': False, 'error': 'No pending_action in session'}
+
+ # Fetch market state for training
+ market_state = self._fetch_market_state_for_candle(symbol, completed_candle.name, data_provider)
+
+ # Calculate price change
price_change = (next_candle['close'] - completed_candle['close']) / completed_candle['close']
# Create training sample
training_sample = {
'symbol': symbol,
'timestamp': completed_candle.name,
- 'market_state': self._fetch_market_state_for_candle(symbol, completed_candle.name, data_provider),
- 'action': 'BUY' if price_change > 0.001 else ('SELL' if price_change < -0.001 else 'HOLD'),
+ 'market_state': market_state,
+ 'action': action_label,
'entry_price': float(completed_candle['close']),
'exit_price': float(next_candle['close']),
'profit_loss_pct': price_change * 100,
- 'direction': 'LONG' if price_change > 0 else 'SHORT'
+ 'direction': 'LONG' if action_label == 'BUY' else ('SHORT' if action_label == 'SELL' else 'HOLD')
}
- # Train on this sample
+ # Train based on model type
model_name = session['model_name']
if model_name == 'Transformer':
self._train_transformer_on_sample(training_sample)
@@ -2801,15 +2859,25 @@ class RealTrainingAdapter:
session['metrics']['accuracy'] = sum(self.realtime_training_metrics['accuracies']) / len(self.realtime_training_metrics['accuracies'])
session['metrics']['steps'] = self.realtime_training_metrics['total_steps']
- logger.info(f"Trained on candle: {symbol} {timeframe} @ {completed_candle.name} (change: {price_change:+.2%})")
+ logger.info(f"Trained on candle: {symbol} {timeframe} @ {completed_candle.name} action={action_label} (change: {price_change:+.2%})")
+
+ return {
+ 'success': True,
+ 'loss': session['metrics']['loss'],
+ 'accuracy': session['metrics']['accuracy'],
+ 'training_steps': session['metrics']['steps']
+ }
+
+ return {'success': False, 'error': f'Unsupported model: {model_name}'}
except Exception as e:
logger.warning(f"Error training on new candle: {e}")
+ return {'success': False, 'error': str(e)}
def _fetch_market_state_for_candle(self, symbol: str, timestamp, data_provider) -> Dict:
- """Fetch market state at a specific candle time"""
+ """Fetch market state with OHLCV data for model training"""
try:
- # Simplified version - get recent data
+ # Get market state with OHLCV data only (NO business logic)
market_state = {'timeframes': {}, 'secondary_timeframes': {}}
for tf in ['1s', '1m', '1h', '1d']:
@@ -3192,8 +3260,15 @@ class RealTrainingAdapter:
# Extract action prediction
action_probs = outputs.get('action_probs')
if action_probs is not None:
- action_idx = torch.argmax(action_probs, dim=-1).item()
- confidence = action_probs[0, action_idx].item()
+ # Handle different tensor shapes: [batch, 3] or [3]
+ if action_probs.dim() == 1:
+ # Shape [3] - single prediction
+ action_idx = torch.argmax(action_probs, dim=0).item()
+ confidence = action_probs[action_idx].item()
+ else:
+ # Shape [batch, 3] - take first batch item
+ action_idx = torch.argmax(action_probs[0], dim=0).item()
+ confidence = action_probs[0, action_idx].item()
# Map to BUY/SELL/HOLD
actions = ['BUY', 'SELL', 'HOLD']
@@ -3291,29 +3366,125 @@ class RealTrainingAdapter:
else:
logger.info(f"Live Signal (NOT executed): {signal['action']} @ {signal['price']:.2f} (conf: {signal['confidence']:.2f}) - {self._get_rejection_reason(session, signal)}")
- # Store prediction for visualization WITH predicted_candle data for ghost candles
+ # Store prediction for visualization (INCLUDE predicted_candle for ghost candles!)
if self.orchestrator and hasattr(self.orchestrator, 'store_transformer_prediction'):
- stored_prediction = {
+ prediction_data = {
'timestamp': datetime.now(timezone.utc).isoformat(),
'current_price': current_price,
- 'predicted_price': prediction.get('predicted_price', current_price * (1.01 if prediction['action'] == 'BUY' else 0.99)),
+ 'predicted_price': prediction.get('predicted_price', current_price),
'price_change': 1.0 if prediction['action'] == 'BUY' else -1.0,
'confidence': prediction['confidence'],
'action': prediction['action'],
'horizon_minutes': 10,
'source': 'live_inference'
}
- # Include predicted_candle for ghost candle visualization
+
+ # Include REAL predicted_candle from model (for ghost candles)
if 'predicted_candle' in prediction and prediction['predicted_candle']:
- stored_prediction['predicted_candle'] = prediction['predicted_candle']
- stored_prediction['next_candles'] = prediction['predicted_candle'] # Alias for compatibility
- logger.debug(f"Stored prediction with {len(prediction['predicted_candle'])} timeframe candles")
-
- self.orchestrator.store_transformer_prediction(symbol, stored_prediction)
+ # Ensure predicted_candle values are Python native types (not tensors)
+ predicted_candle_clean = {}
+ for tf, candle_data in prediction['predicted_candle'].items():
+ if isinstance(candle_data, (list, tuple)):
+ # Convert list/tuple elements to Python scalars
+ predicted_candle_clean[tf] = [
+ float(v.item() if hasattr(v, 'item') else v)
+ for v in candle_data
+ ]
+ elif hasattr(candle_data, 'tolist'):
+ # Tensor array - convert to list
+ predicted_candle_clean[tf] = [float(v) for v in candle_data.tolist()]
+ else:
+ predicted_candle_clean[tf] = candle_data
+
+ prediction_data['predicted_candle'] = predicted_candle_clean
+
+ # Use actual predicted price from candle close (ensure it's a Python float)
+ predicted_price_val = None
+ if '1m' in predicted_candle_clean:
+ close_val = predicted_candle_clean['1m'][3]
+ predicted_price_val = float(close_val.item() if hasattr(close_val, 'item') else close_val)
+ elif '1s' in predicted_candle_clean:
+ close_val = predicted_candle_clean['1s'][3]
+ predicted_price_val = float(close_val.item() if hasattr(close_val, 'item') else close_val)
+
+ if predicted_price_val is not None:
+ prediction_data['predicted_price'] = predicted_price_val
+ prediction_data['price_change'] = ((predicted_price_val - current_price) / current_price) * 100
+ else:
+ prediction_data['predicted_price'] = prediction.get('predicted_price', current_price)
+ prediction_data['price_change'] = 1.0 if prediction['action'] == 'BUY' else -1.0
+ else:
+ # Fallback to estimated price if no candle prediction
+ prediction_data['predicted_price'] = prediction.get('predicted_price', current_price * (1.01 if prediction['action'] == 'BUY' else 0.99))
+ prediction_data['price_change'] = 1.0 if prediction['action'] == 'BUY' else -1.0
+
+ # Include trend_vector if available (convert tensors to Python types)
+ if 'trend_vector' in prediction:
+ trend_vec = prediction['trend_vector']
+ # Convert any tensors to Python native types
+ if isinstance(trend_vec, dict):
+ serialized_trend = {}
+ for key, value in trend_vec.items():
+ if hasattr(value, 'numel'): # Tensor
+ if value.numel() == 1: # Scalar tensor
+ serialized_trend[key] = value.item()
+ else: # Multi-element tensor
+ serialized_trend[key] = value.detach().cpu().tolist()
+ elif hasattr(value, 'tolist'): # Other array-like
+ serialized_trend[key] = value.tolist()
+ elif isinstance(value, (list, tuple)):
+ # Recursively convert list/tuple of tensors
+ serialized_trend[key] = []
+ for v in value:
+ if hasattr(v, 'numel'):
+ if v.numel() == 1:
+ serialized_trend[key].append(v.item())
+ else:
+ serialized_trend[key].append(v.detach().cpu().tolist())
+ elif hasattr(v, 'tolist'):
+ serialized_trend[key].append(v.tolist())
+ else:
+ serialized_trend[key].append(v)
+ else:
+ serialized_trend[key] = value
+ prediction_data['trend_vector'] = serialized_trend
+ else:
+ prediction_data['trend_vector'] = trend_vec
+
+ self.orchestrator.store_transformer_prediction(symbol, prediction_data)
- # Per-candle training mode
- if train_every_candle:
- self._train_on_new_candle(session, symbol, timeframe, data_provider)
+ # Training decision using strategy manager
+ training_strategy = session.get('training_strategy')
+ if training_strategy and training_strategy.mode != 'none':
+ # Get pivot markers for training decision
+ pivot_markers = {}
+ if hasattr(training_strategy, 'dashboard') and training_strategy.dashboard:
+ try:
+ df = data_provider.get_historical_data(symbol, timeframe, limit=200)
+ if df is not None and len(df) >= 10:
+ pivot_markers = training_strategy.dashboard._get_pivot_markers_for_timeframe(symbol, timeframe, df)
+ except Exception as e:
+ logger.debug(f"Could not get pivot markers: {e}")
+
+ # Get current candle timestamp
+ df_current = data_provider.get_historical_data(symbol, timeframe, limit=1)
+ if df_current is not None and len(df_current) > 0:
+ current_timestamp = df_current.index[-1]
+
+ # Ask strategy manager if we should train
+ should_train, action_data = training_strategy.should_train_on_candle(
+ symbol, timeframe, current_timestamp, pivot_markers
+ )
+
+ if should_train and action_data:
+ # Set action in session for training
+ session['pending_action'] = action_data['action']
+
+ # Call pure training method
+ train_result = self._train_on_new_candle(session, symbol, timeframe, data_provider)
+
+ if train_result.get('success'):
+ logger.info(f"Training completed: {action_data['action']} (reason: {action_data.get('reason', 'unknown')})")
# Sleep based on timeframe
sleep_time = self._get_sleep_time_for_timeframe(timeframe)
@@ -3321,6 +3492,8 @@ class RealTrainingAdapter:
except Exception as e:
logger.error(f"Error in inference loop: {e}")
+ import traceback
+ logger.error(f"Traceback: {traceback.format_exc()}")
time.sleep(5)
logger.info(f"Inference loop stopped: {inference_id}")
diff --git a/ANNOTATE/data/annotations/annotations_db.json b/ANNOTATE/data/annotations/annotations_db.json
index 076665a..d6c25c9 100644
--- a/ANNOTATE/data/annotations/annotations_db.json
+++ b/ANNOTATE/data/annotations/annotations_db.json
@@ -1,28 +1,5 @@
{
"annotations": [
- {
- "annotation_id": "dc35c362-6174-4db4-b4db-8cc58a4ba8e5",
- "symbol": "ETH/USDT",
- "timeframe": "1h",
- "entry": {
- "timestamp": "2025-10-07 13:00",
- "price": 4755,
- "index": 28
- },
- "exit": {
- "timestamp": "2025-10-11 21:00",
- "price": 3643.33,
- "index": 63
- },
- "direction": "SHORT",
- "profit_loss_pct": 23.378969505783388,
- "notes": "",
- "created_at": "2025-10-24T22:33:26.187249",
- "market_context": {
- "entry_state": {},
- "exit_state": {}
- }
- },
{
"annotation_id": "5d5c4354-12dd-4e0c-92a8-eff631a5dfab",
"symbol": "ETH/USDT",
@@ -115,29 +92,6 @@
"exit_state": {}
}
},
- {
- "annotation_id": "46cc0e20-0bfb-498c-9358-71b52a003d0f",
- "symbol": "ETH/USDT",
- "timeframe": "1s",
- "entry": {
- "timestamp": "2025-11-22 12:50",
- "price": 2712.11,
- "index": 26
- },
- "exit": {
- "timestamp": "2025-11-22 12:53:06",
- "price": 2721.44,
- "index": 45
- },
- "direction": "LONG",
- "profit_loss_pct": 0.3440125953593301,
- "notes": "",
- "created_at": "2025-11-22T15:19:00.480166",
- "market_context": {
- "entry_state": {},
- "exit_state": {}
- }
- },
{
"annotation_id": "b01fe6b2-7724-495e-ab01-3f3d3aa0da5d",
"symbol": "ETH/USDT",
@@ -160,10 +114,33 @@
"entry_state": {},
"exit_state": {}
}
+ },
+ {
+ "annotation_id": "19a566fa-63bb-4ce3-9f30-116127c9fe95",
+ "symbol": "ETH/USDT",
+ "timeframe": "1s",
+ "entry": {
+ "timestamp": "2025-11-22 22:25:17",
+ "price": 2761.97,
+ "index": 35
+ },
+ "exit": {
+ "timestamp": "2025-11-22 22:30:40",
+ "price": 2760.15,
+ "index": 49
+ },
+ "direction": "SHORT",
+ "profit_loss_pct": 0.06589499523889503,
+ "notes": "",
+ "created_at": "2025-11-22T22:35:55.606071+00:00",
+ "market_context": {
+ "entry_state": {},
+ "exit_state": {}
+ }
}
],
"metadata": {
- "total_annotations": 7,
- "last_updated": "2025-11-22T15:31:43.940190"
+ "total_annotations": 6,
+ "last_updated": "2025-11-22T22:35:55.606373+00:00"
}
}
\ No newline at end of file
diff --git a/ANNOTATE/web/app.py b/ANNOTATE/web/app.py
index ccc1573..b89acfa 100644
--- a/ANNOTATE/web/app.py
+++ b/ANNOTATE/web/app.py
@@ -17,7 +17,7 @@ from flask import Flask, render_template, request, jsonify, send_file
from dash import Dash, html
import logging
from datetime import datetime, timezone, timedelta
-from typing import Optional, Dict, List, Any
+from typing import Optional, Dict, List, Any, Tuple
import json
import pandas as pd
import numpy as np
@@ -370,8 +370,8 @@ class BacktestRunner:
action_idx = torch.argmax(action_probs, dim=-1).item()
confidence = action_probs[0, action_idx].item()
- # Map to BUY/SELL/HOLD
- actions = ['BUY', 'SELL', 'HOLD']
+ # Map to action string (must match training: 0=HOLD, 1=BUY, 2=SELL)
+ actions = ['HOLD', 'BUY', 'SELL']
if action_idx < len(actions):
action = actions[action_idx]
else:
@@ -490,6 +490,194 @@ class BacktestRunner:
state['stop_requested'] = True
+class TrainingStrategyManager:
+ """
+ Manages training strategies and decisions - Separates business logic from model interface
+
+ Training Modes:
+ - 'none': No training (inference only)
+ - 'every_candle': Train on every completed candle
+ - 'pivots_only': Train only on pivot points (BUY at L pivots, SELL at H pivots)
+ - 'manual': Training triggered manually by user button
+ """
+
+ def __init__(self, data_provider, training_adapter):
+ self.data_provider = data_provider
+ self.training_adapter = training_adapter
+ self.mode = 'none' # Default: no training
+ self.dashboard = None # Set by dashboard after initialization
+
+ # Statistics tracking
+ self.stats = {
+ 'total_trained': 0,
+ 'by_action': {'BUY': 0, 'SELL': 0, 'HOLD': 0},
+ 'profitable': 0
+ }
+
+ def should_train_on_candle(self, symbol: str, timeframe: str, candle_timestamp, pivot_markers: Dict = None) -> Tuple[bool, Optional[Dict]]:
+ """
+ Decide if we should train on this candle based on current mode
+
+ Args:
+ symbol: Trading symbol
+ timeframe: Candle timeframe
+ candle_timestamp: Timestamp of the candle
+ pivot_markers: Dict of pivot markers (timestamp -> pivot data)
+
+ Returns:
+ Tuple of (should_train: bool, action_data: Optional[Dict])
+ action_data contains: {'action': 'BUY'/'SELL'/'HOLD', 'pivot_level': int, 'pivot_strength': float}
+ """
+ if self.mode == 'none':
+ return False, None
+
+ elif self.mode == 'every_candle':
+ # Train on every candle - determine action from price movement or pivots
+ action_data = self._get_action_for_candle(symbol, timeframe, candle_timestamp, pivot_markers)
+ return True, action_data
+
+ elif self.mode == 'pivots_only':
+ # Train only on pivot candles
+ return self._is_pivot_candle(candle_timestamp, pivot_markers)
+
+ elif self.mode == 'manual':
+ # Manual training - don't auto-train
+ return False, None
+
+ return False, None
+
+ def _get_action_for_candle(self, symbol: str, timeframe: str, candle_timestamp, pivot_markers: Dict = None) -> Dict:
+ """
+ Determine action for any candle (pivot or non-pivot)
+ For pivot candles: BUY at L, SELL at H
+ For non-pivot candles: Use price movement thresholds
+ """
+ # First check if it's a pivot candle
+ is_pivot, pivot_action = self._is_pivot_candle(candle_timestamp, pivot_markers)
+ if is_pivot and pivot_action:
+ return pivot_action
+
+ # Not a pivot - use price movement based logic
+ # Get recent candles to determine trend
+ df = self.data_provider.get_historical_data(symbol, timeframe, limit=5)
+ if df is None or len(df) < 3:
+ return {'action': 'HOLD', 'reason': 'insufficient_data'}
+
+ # Simple momentum: if price going up, BUY, if going down, SELL
+ recent_change = (df.iloc[-1]['close'] - df.iloc[-3]['close']) / df.iloc[-3]['close']
+
+ if recent_change > 0.0005: # 0.05% up
+ action = 'BUY'
+ elif recent_change < -0.0005: # 0.05% down
+ action = 'SELL'
+ else:
+ action = 'HOLD'
+
+ return {
+ 'action': action,
+ 'reason': 'price_movement',
+ 'change_pct': recent_change * 100
+ }
+
+ def _is_pivot_candle(self, timestamp, pivot_markers: Dict = None) -> Tuple[bool, Optional[Dict]]:
+ """
+ Check if candle is a pivot point and return action
+
+ Returns:
+ Tuple of (is_pivot: bool, action_data: Optional[Dict])
+ """
+ if not pivot_markers:
+ return False, None
+
+ candle_timestamp = str(timestamp)
+ candle_pivots = pivot_markers.get(candle_timestamp, {})
+
+ if not candle_pivots:
+ return False, None
+
+ # BUY at L pivots (lows - support levels)
+ if 'lows' in candle_pivots and len(candle_pivots['lows']) > 0:
+ best_low = max(candle_pivots['lows'], key=lambda p: p.get('level', 0))
+ pivot_level = best_low.get('level', 1)
+ pivot_strength = best_low.get('strength', 0.5)
+
+ logger.info(f"L{pivot_level}L pivot detected @ {timestamp}, strength={pivot_strength:.2f} → BUY signal")
+
+ return True, {
+ 'action': 'BUY',
+ 'pivot_level': pivot_level,
+ 'pivot_strength': pivot_strength,
+ 'reason': 'low_pivot'
+ }
+
+ # SELL at H pivots (highs - resistance levels)
+ elif 'highs' in candle_pivots and len(candle_pivots['highs']) > 0:
+ best_high = max(candle_pivots['highs'], key=lambda p: p.get('level', 0))
+ pivot_level = best_high.get('level', 1)
+ pivot_strength = best_high.get('strength', 0.5)
+
+ logger.info(f"L{pivot_level}H pivot detected @ {timestamp}, strength={pivot_strength:.2f} → SELL signal")
+
+ return True, {
+ 'action': 'SELL',
+ 'pivot_level': pivot_level,
+ 'pivot_strength': pivot_strength,
+ 'reason': 'high_pivot'
+ }
+
+ return False, None
+
+ def train_manually(self, symbol: str, timeframe: str, action: str) -> Dict:
+ """
+ Manually trigger training with specified action
+
+ Args:
+ symbol: Trading symbol
+ timeframe: Timeframe
+ action: Action to train ('BUY', 'SELL', or 'HOLD')
+
+ Returns:
+ Training result dict with metrics
+ """
+ logger.info(f"Manual training triggered: {action} on {symbol} {timeframe}")
+
+ # Create action data
+ action_data = {
+ 'action': action,
+ 'reason': 'manual_trigger'
+ }
+
+ # Update stats
+ self.stats['total_trained'] += 1
+ self.stats['by_action'][action] = self.stats['by_action'].get(action, 0) + 1
+
+ return {
+ 'success': True,
+ 'action': action,
+ 'triggered_by': 'manual'
+ }
+
+ def get_stats(self) -> Dict:
+ """Get training statistics"""
+ total = self.stats['total_trained']
+ if total == 0:
+ return {
+ 'total_trained': 0,
+ 'by_action': {'BUY': '0%', 'SELL': '0%', 'HOLD': '0%'},
+ 'mode': self.mode
+ }
+
+ return {
+ 'total_trained': total,
+ 'by_action': {
+ 'BUY': f"{(self.stats['by_action'].get('BUY', 0) / total * 100):.1f}%",
+ 'SELL': f"{(self.stats['by_action'].get('SELL', 0) / total * 100):.1f}%",
+ 'HOLD': f"{(self.stats['by_action'].get('HOLD', 0) / total * 100):.1f}%"
+ },
+ 'mode': self.mode
+ }
+
+
class AnnotationDashboard:
"""Main annotation dashboard application"""
@@ -586,12 +774,19 @@ class AnnotationDashboard:
self.annotation_manager = AnnotationManager()
# Use REAL training adapter - NO SIMULATION!
self.training_adapter = RealTrainingAdapter(None, self.data_provider)
+ # Initialize training strategy manager (controls training decisions)
+ self.training_strategy = TrainingStrategyManager(self.data_provider, self.training_adapter)
+ self.training_strategy.dashboard = self
# Pass socketio to training adapter for live trade updates
if self.has_socketio and self.socketio:
self.training_adapter.socketio = self.socketio
# Backtest runner for replaying visible chart with predictions
self.backtest_runner = BacktestRunner()
+ # Prediction cache for training: stores inference inputs/outputs to compare with actual candles
+ # Format: {symbol: {timeframe: [{'timestamp': ts, 'inputs': {...}, 'outputs': {...}, 'norm_params': {...}}, ...]}}
+ self.prediction_cache = {}
+
# Check if we should auto-load a model at startup
auto_load_model = os.getenv('AUTO_LOAD_MODEL', 'Transformer') # Default: Transformer
@@ -2121,14 +2316,21 @@ class AnnotationDashboard:
@self.server.route('/api/realtime-inference/start', methods=['POST'])
def start_realtime_inference():
- """Start real-time inference mode with optional training modes"""
+ """Start real-time inference mode with configurable training strategy"""
try:
data = request.get_json()
model_name = data.get('model_name')
symbol = data.get('symbol', 'ETH/USDT')
timeframe = data.get('timeframe', '1m')
- enable_live_training = data.get('enable_live_training', False) # Pivot-based training
- train_every_candle = data.get('train_every_candle', False) # Per-candle training
+
+ # New unified training_mode parameter
+ training_mode = data.get('training_mode', 'none') # 'none', 'every_candle', 'pivots_only', 'manual'
+
+ # Backward compatibility with old parameters
+ if 'enable_live_training' in data or 'train_every_candle' in data:
+ enable_live_training = data.get('enable_live_training', False)
+ train_every_candle = data.get('train_every_candle', False)
+ training_mode = 'every_candle' if train_every_candle else ('pivots_only' if enable_live_training else 'none')
if not self.training_adapter:
return jsonify({
@@ -2139,18 +2341,21 @@ class AnnotationDashboard:
}
})
- # Start real-time inference with optional training modes
+ # Set training mode on strategy manager
+ self.training_strategy.mode = training_mode
+ logger.info(f"Training strategy mode set to: {training_mode}")
+
+ # Start real-time inference - pass strategy manager for training decisions
inference_id = self.training_adapter.start_realtime_inference(
model_name=model_name,
symbol=symbol,
data_provider=self.data_provider,
- enable_live_training=enable_live_training,
- train_every_candle=train_every_candle,
- timeframe=timeframe
+ enable_live_training=(training_mode != 'none'),
+ train_every_candle=(training_mode == 'every_candle'),
+ timeframe=timeframe,
+ training_strategy=self.training_strategy # Pass strategy manager
)
- training_mode = "per-candle" if train_every_candle else ("pivot-based" if enable_live_training else "inference-only")
-
return jsonify({
'success': True,
'inference_id': inference_id,
@@ -2259,20 +2464,17 @@ class AnnotationDashboard:
if hasattr(self.orchestrator, 'recent_transformer_predictions') and symbol in self.orchestrator.recent_transformer_predictions:
transformer_preds = list(self.orchestrator.recent_transformer_predictions[symbol])
if transformer_preds:
- # Use the most recent stored prediction (from inference loop)
- predictions['transformer'] = transformer_preds[-1]
- logger.debug(f"Using stored prediction: {list(transformer_preds[-1].keys())}")
- else:
- # Fallback: generate new prediction if no stored predictions
- transformer_pred = self._get_live_transformer_prediction(symbol)
- if transformer_pred:
- predictions['transformer'] = transformer_pred
+ # Convert any remaining tensors to Python types before JSON serialization
+ transformer_pred = transformer_preds[-1].copy()
+ predictions['transformer'] = self._serialize_prediction(transformer_pred)
if predictions:
response['prediction'] = predictions
except Exception as e:
logger.debug(f"Error getting predictions: {e}")
+ import traceback
+ logger.debug(traceback.format_exc())
return jsonify(response)
@@ -2322,10 +2524,101 @@ class AnnotationDashboard:
}
})
+ @self.server.route('/api/realtime-inference/train-manual', methods=['POST'])
+ def train_manual():
+ """Manually trigger training on current candle with specified action"""
+ try:
+ data = request.get_json()
+ inference_id = data.get('inference_id')
+ action = data.get('action', 'HOLD')
+
+ if not self.training_adapter:
+ return jsonify({
+ 'success': False,
+ 'error': 'Training adapter not available'
+ })
+
+ # Get active inference session
+ if not hasattr(self.training_adapter, 'inference_sessions'):
+ return jsonify({
+ 'success': False,
+ 'error': 'No active inference sessions'
+ })
+
+ session = self.training_adapter.inference_sessions.get(inference_id)
+ if not session:
+ return jsonify({
+ 'success': False,
+ 'error': 'Inference session not found'
+ })
+
+ # Set pending action for training
+ session['pending_action'] = action
+
+ # Get session parameters
+ symbol = session.get('symbol', 'ETH/USDT')
+ timeframe = session.get('timeframe', '1m')
+ data_provider = session.get('data_provider')
+
+ # Call training method
+ train_result = self.training_adapter._train_on_new_candle(
+ session, symbol, timeframe, data_provider
+ )
+
+ if train_result.get('success'):
+ return jsonify({
+ 'success': True,
+ 'action': action,
+ 'metrics': {
+ 'loss': train_result.get('loss', 0.0),
+ 'accuracy': train_result.get('accuracy', 0.0),
+ 'training_steps': train_result.get('training_steps', 0)
+ }
+ })
+ else:
+ return jsonify({
+ 'success': False,
+ 'error': train_result.get('error', 'Training failed')
+ })
+
+ except Exception as e:
+ logger.error(f"Error in manual training: {e}")
+ return jsonify({
+ 'success': False,
+ 'error': str(e)
+ })
+
# WebSocket event handlers (if SocketIO is available)
if self.has_socketio:
self._setup_websocket_handlers()
+ def _serialize_prediction(self, prediction: Dict) -> Dict:
+ """Convert PyTorch tensors in prediction dict to JSON-serializable Python types"""
+ try:
+ import torch
+ serialized = {}
+ for key, value in prediction.items():
+ if isinstance(value, torch.Tensor):
+ if value.numel() == 1: # Scalar tensor
+ serialized[key] = value.item()
+ else: # Multi-element tensor
+ serialized[key] = value.detach().cpu().tolist()
+ elif isinstance(value, dict):
+ serialized[key] = self._serialize_prediction(value) # Recursive
+ elif isinstance(value, (list, tuple)):
+ serialized[key] = [
+ v.item() if isinstance(v, torch.Tensor) and v.numel() == 1 else
+ (v.detach().cpu().tolist() if isinstance(v, torch.Tensor) else v)
+ for v in value
+ ]
+ else:
+ serialized[key] = value
+ return serialized
+ except Exception as e:
+ logger.warning(f"Error serializing prediction: {e}")
+ # Fallback: return as-is (might fail JSON serialization but won't crash)
+ return prediction
+
def _setup_websocket_handlers(self):
"""Setup WebSocket event handlers for real-time updates"""
if not self.has_socketio:
@@ -2748,35 +3041,209 @@ class AnnotationDashboard:
return {}
def _get_live_prediction(self, symbol: str, timeframe: str, prediction_steps: int = 1):
- """Get live prediction from model"""
+ """
+ Get live prediction from model using trainer inference
+
+ Caches inference data (inputs/outputs) for later training when actual candle arrives.
+ This allows us to:
+ 1. Compare predicted vs actual candle values
+ 2. Calculate loss
+ 3. Do backpropagation with correct outputs
+
+ Returns:
+ Dict with prediction results including predicted_candle for ghost candle display
+ """
try:
- if not self.orchestrator or not hasattr(self.orchestrator, 'primary_transformer'):
+ if not self.orchestrator:
return None
- # Get recent candles for prediction
- candles = self.data_provider.get_ohlcv(symbol, timeframe, limit=200)
- if not candles or len(candles) < 200:
+ # Get trainer from orchestrator
+ trainer = getattr(self.orchestrator, 'primary_transformer_trainer', None)
+ if not trainer or not trainer.model:
+ logger.debug("No transformer trainer available for live prediction")
return None
- # TODO: Implement actual prediction logic
- # For now, return placeholder
- import random
+ # Get market data using training adapter's method (reuses existing logic)
+ if not hasattr(self.training_adapter, '_get_realtime_market_data'):
+ logger.warning("Training adapter missing _get_realtime_market_data method")
+ return None
+ market_data, norm_params = self.training_adapter._get_realtime_market_data(symbol, self.data_provider)
+ if not market_data:
+ logger.debug(f"No market data available for {symbol} {timeframe}")
+ return None
+
+ # Make prediction with model
+ import torch
+ timestamp = datetime.now(timezone.utc)
+
+ with torch.no_grad():
+ trainer.model.eval()
+ outputs = trainer.model(**market_data)
+
+ # Extract action prediction
+ action_probs = outputs.get('action_probs')
+ if action_probs is None:
+ logger.debug("No action_probs in model output")
+ return None
+
+ action_idx = torch.argmax(action_probs, dim=-1).item()
+ confidence = action_probs[0, action_idx].item()
+
+ # Map to action string (must match training: 0=HOLD, 1=BUY, 2=SELL)
+ actions = ['HOLD', 'BUY', 'SELL']
+ action = actions[action_idx] if action_idx < len(actions) else 'HOLD'
+
+ # Extract predicted candles and denormalize
+ predicted_candles_raw = {}
+ if 'next_candles' in outputs:
+ for tf, tensor in outputs['next_candles'].items():
+ predicted_candles_raw[tf] = tensor.detach().cpu().numpy().tolist()
+
+ # Denormalize predicted candles
+ predicted_candles_denorm = {}
+ if predicted_candles_raw and norm_params:
+ for tf, raw_candle in predicted_candles_raw.items():
+ if tf in norm_params:
+ params = norm_params[tf]
+ price_min = params['price_min']
+ price_max = params['price_max']
+ vol_min = params['volume_min']
+ vol_max = params['volume_max']
+
+ # raw_candle is [1, 5] list
+ candle_values = raw_candle[0]
+
+ denorm_candle = [
+ candle_values[0] * (price_max - price_min) + price_min, # Open
+ candle_values[1] * (price_max - price_min) + price_min, # High
+ candle_values[2] * (price_max - price_min) + price_min, # Low
+ candle_values[3] * (price_max - price_min) + price_min, # Close
+ candle_values[4] * (vol_max - vol_min) + vol_min # Volume
+ ]
+ predicted_candles_denorm[tf] = denorm_candle
+
+ # Get predicted price from candle close
+ predicted_price = None
+ if timeframe in predicted_candles_denorm:
+ predicted_price = predicted_candles_denorm[timeframe][3] # Close
+ elif '1m' in predicted_candles_denorm:
+ predicted_price = predicted_candles_denorm['1m'][3]
+ elif '1s' in predicted_candles_denorm:
+ predicted_price = predicted_candles_denorm['1s'][3]
+
+ # CACHE inference data for later training
+ # Store inputs, outputs, and normalization params so we can train when actual candle arrives
+ if symbol not in self.prediction_cache:
+ self.prediction_cache[symbol] = {}
+ if timeframe not in self.prediction_cache[symbol]:
+ self.prediction_cache[symbol][timeframe] = []
+
+ # Store cached inference data (convert tensors to CPU for storage)
+ cached_data = {
+ 'timestamp': timestamp,
+ 'symbol': symbol,
+ 'timeframe': timeframe,
+ 'model_inputs': {k: v.cpu().clone() if isinstance(v, torch.Tensor) else v
+ for k, v in market_data.items()},
+ 'model_outputs': {k: v.cpu().clone() if isinstance(v, torch.Tensor) else v
+ for k, v in outputs.items()},
+ 'normalization_params': norm_params,
+ 'predicted_candle': predicted_candles_denorm.get(timeframe),
+ 'prediction_steps': prediction_steps
+ }
+
+ self.prediction_cache[symbol][timeframe].append(cached_data)
+
+ # Keep only last 100 predictions per symbol/timeframe to prevent memory bloat
+ if len(self.prediction_cache[symbol][timeframe]) > 100:
+ self.prediction_cache[symbol][timeframe] = self.prediction_cache[symbol][timeframe][-100:]
+
+ logger.debug(f"Cached prediction for {symbol} {timeframe} @ {timestamp.isoformat()}")
+
+ # Return prediction result (same format as before for compatibility)
return {
'symbol': symbol,
'timeframe': timeframe,
- 'timestamp': datetime.now(timezone.utc).isoformat(),
- 'action': random.choice(['BUY', 'SELL', 'HOLD']),
- 'confidence': random.uniform(0.6, 0.95),
- 'predicted_price': candles[-1].get('close', 0) * (1 + random.uniform(-0.01, 0.01)),
+ 'timestamp': timestamp.isoformat(),
+ 'action': action,
+ 'confidence': confidence,
+ 'predicted_price': predicted_price,
+ 'predicted_candle': predicted_candles_denorm,
'prediction_steps': prediction_steps
}
except Exception as e:
logger.error(f"Error getting live prediction: {e}")
+ import traceback
+ logger.debug(traceback.format_exc())
return None
- def run(self, host='127.0.0.1', port=8052, debug=False):
+ def get_cached_predictions_for_training(self, symbol: str, timeframe: str, actual_candle_timestamp) -> List[Dict]:
+ """
+ Retrieve cached predictions that match a specific candle timestamp for training
+
+ When an actual candle arrives, we can:
+ 1. Find cached predictions made before this candle
+ 2. Compare predicted vs actual candle values
+ 3. Calculate loss and do backpropagation
+
+ Args:
+ symbol: Trading symbol
+ timeframe: Timeframe
+ actual_candle_timestamp: Timestamp of the actual candle that just arrived
+
+ Returns:
+ List of cached prediction dicts that should be trained on
+ """
+ try:
+ if symbol not in self.prediction_cache:
+ return []
+ if timeframe not in self.prediction_cache[symbol]:
+ return []
+
+ # Find predictions made before this candle timestamp
+ # Predictions should be for candles that have now completed
+ matching_predictions = []
+ actual_time = actual_candle_timestamp if isinstance(actual_candle_timestamp, datetime) else datetime.fromisoformat(str(actual_candle_timestamp).replace('Z', '+00:00'))
+
+ for cached_pred in self.prediction_cache[symbol][timeframe]:
+ pred_time = cached_pred['timestamp']
+ if isinstance(pred_time, str):
+ pred_time = datetime.fromisoformat(pred_time.replace('Z', '+00:00'))
+
+ # Prediction should be for a candle that comes after the prediction time
+ # We match predictions that were made before the actual candle closed
+ if pred_time < actual_time:
+ matching_predictions.append(cached_pred)
+
+ return matching_predictions
+
+ except Exception as e:
+ logger.error(f"Error getting cached predictions for training: {e}")
+ return []
+
+ def clear_old_cached_predictions(self, symbol: str, timeframe: str, before_timestamp: datetime):
+ """
+ Clear cached predictions older than a certain timestamp
+
+ Useful for cleaning up old predictions that are no longer needed
+ """
+ try:
+ if symbol not in self.prediction_cache:
+ return
+ if timeframe not in self.prediction_cache[symbol]:
+ return
+
+ self.prediction_cache[symbol][timeframe] = [
+ pred for pred in self.prediction_cache[symbol][timeframe]
+ if pred['timestamp'] >= before_timestamp
+ ]
+
+ except Exception as e:
+ logger.debug(f"Error clearing old cached predictions: {e}")
+
+ def run(self, host='127.0.0.1', port=8051, debug=False):
"""Run the application"""
logger.info(f"Starting Annotation Dashboard on http://{host}:{port}")
diff --git a/ANNOTATE/web/templates/components/training_panel.html b/ANNOTATE/web/templates/components/training_panel.html
index 62c056f..ec32c78 100644
--- a/ANNOTATE/web/templates/components/training_panel.html
+++ b/ANNOTATE/web/templates/components/training_panel.html
@@ -100,6 +100,10 @@
Stop Inference
+
@@ -628,7 +632,7 @@
});
// Helper function to start inference with different modes
- function startInference(enableLiveTraining, trainEveryCandle) {
+ function startInference(trainingMode) {
const modelName = document.getElementById('model-select').value;
if (!modelName) {
@@ -639,7 +643,7 @@
// Get timeframe
const timeframe = document.getElementById('primary-timeframe-select').value;
- // Start real-time inference
+ // Start real-time inference with unified training_mode parameter
fetch('/api/realtime-inference/start', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
@@ -647,8 +651,7 @@
model_name: modelName,
symbol: appState.currentSymbol,
timeframe: timeframe,
- enable_live_training: enableLiveTraining,
- train_every_candle: trainEveryCandle
+ training_mode: trainingMode // 'none', 'every_candle', 'pivots_only', 'manual'
})
})
.then(response => response.json())
@@ -664,6 +667,11 @@
document.getElementById('inference-status').style.display = 'block';
document.getElementById('inference-controls').style.display = 'block';
+ // Show manual training button if in manual mode
+ if (trainingMode === 'manual') {
+ document.getElementById('manual-train-btn').style.display = 'block';
+ }
+
// Display active timeframe
document.getElementById('active-timeframe').textContent = timeframe.toUpperCase();
@@ -708,15 +716,15 @@
// Button handlers for different inference modes
document.getElementById('start-inference-btn').addEventListener('click', function () {
- startInference(false, false); // No training
+ startInference('none'); // No training (inference only)
});
document.getElementById('start-inference-pivot-btn').addEventListener('click', function () {
- startInference(true, false); // Pivot-based training
+ startInference('pivots_only'); // Pivot-based training
});
document.getElementById('start-inference-candle-btn').addEventListener('click', function () {
- startInference(false, true); // Per-candle training
+ startInference('every_candle'); // Per-candle training
});
document.getElementById('stop-inference-btn').addEventListener('click', function () {
@@ -736,6 +744,7 @@
document.getElementById('start-inference-pivot-btn').style.display = 'block';
document.getElementById('start-inference-candle-btn').style.display = 'block';
document.getElementById('stop-inference-btn').style.display = 'none';
+ document.getElementById('manual-train-btn').style.display = 'none';
document.getElementById('inference-status').style.display = 'none';
document.getElementById('inference-controls').style.display = 'none';
@@ -762,6 +771,42 @@
showError('Network error: ' + error.message);
});
});
+
+ // Manual training button handler
+ document.getElementById('manual-train-btn').addEventListener('click', function () {
+ if (!currentInferenceId) {
+ showError('No active inference session');
+ return;
+ }
+
+ // Get user's action choice (could add a dropdown, for now use BUY as example)
+ const action = prompt('Enter action (BUY, SELL, or HOLD):', 'BUY');
+ if (!action || !['BUY', 'SELL', 'HOLD'].includes(action.toUpperCase())) {
+ showError('Invalid action. Must be BUY, SELL, or HOLD');
+ return;
+ }
+
+ // Trigger manual training
+ fetch('/api/realtime-inference/train-manual', {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({
+ inference_id: currentInferenceId,
+ action: action.toUpperCase()
+ })
+ })
+ .then(response => response.json())
+ .then(data => {
+ if (data.success) {
+ showSuccess(`Manual training completed: ${data.action} (${data.metrics ? 'Loss: ' + data.metrics.loss.toFixed(4) : ''})`);
+ } else {
+ showError('Manual training failed: ' + (data.error || 'Unknown error'));
+ }
+ })
+ .catch(error => {
+ showError('Network error: ' + error.message);
+ });
+ });
// Backtest controls
let currentBacktestId = null;
diff --git a/NN/models/advanced_transformer_trading.py b/NN/models/advanced_transformer_trading.py
index d04df0b..2e00038 100644
--- a/NN/models/advanced_transformer_trading.py
+++ b/NN/models/advanced_transformer_trading.py
@@ -162,7 +162,8 @@ class DeepMultiScaleAttention(nn.Module):
scores = torch.matmul(Q, K.transpose(-2, -1)) / math.sqrt(self.head_dim)
if mask is not None:
- scores.masked_fill_(mask == 0, -1e9)
+ # Use non-inplace version to avoid gradient computation issues
+ scores = scores.masked_fill(mask == 0, -1e9)
attention = F.softmax(scores, dim=-1)
attention = self.dropout(attention)
@@ -1089,8 +1090,11 @@ class TradingTransformerTrainer:
pct_start=0.1
)
- # Loss functions
- self.action_criterion = nn.CrossEntropyLoss()
+ # Loss functions with class weights
+ # Pivot-based training: BUY at L pivots, SELL at H pivots (naturally balanced)
+ # Weights: [HOLD=0, BUY=1, SELL=2] - equal weighting for pivot-based trades
+ class_weights = torch.tensor([0.5, 1.0, 1.0], dtype=torch.float32, device=self.device)
+ self.action_criterion = nn.CrossEntropyLoss(weight=class_weights)
self.price_criterion = nn.MSELoss()
self.confidence_criterion = nn.BCELoss()
@@ -1182,19 +1186,30 @@ class TradingTransformerTrainer:
Returns:
Denormalized OHLCV tensor
"""
- denorm = normalized_candle.clone()
-
- # Denormalize OHLC (first 4 values)
+ # Avoid inplace operations by creating new tensors instead of slice assignment
price_min = norm_params.get('price_min', 0.0)
price_max = norm_params.get('price_max', 1.0)
- if price_max > price_min:
- denorm[..., :4] = denorm[..., :4] * (price_max - price_min) + price_min
-
- # Denormalize volume (5th value)
volume_min = norm_params.get('volume_min', 0.0)
volume_max = norm_params.get('volume_max', 1.0)
+
+ # Denormalize OHLC (first 4 values) - create new tensor, no inplace operations
+ if price_max > price_min:
+ price_scale = (price_max - price_min)
+ price_offset = price_min
+ denorm_ohlc = normalized_candle[..., :4] * price_scale + price_offset
+ else:
+ denorm_ohlc = normalized_candle[..., :4]
+
+ # Denormalize volume (5th value) - create new tensor, no inplace operations
if volume_max > volume_min:
- denorm[..., 4] = denorm[..., 4] * (volume_max - volume_min) + volume_min
+ volume_scale = (volume_max - volume_min)
+ volume_offset = volume_min
+ denorm_volume = (normalized_candle[..., 4:5] * volume_scale + volume_offset)
+ else:
+ denorm_volume = normalized_candle[..., 4:5]
+
+ # Concatenate OHLC and Volume to create final tensor (no inplace operations)
+ denorm = torch.cat([denorm_ohlc, denorm_volume], dim=-1)
return denorm
@@ -1675,9 +1690,46 @@ class TradingTransformerTrainer:
"""Load model and training state"""
checkpoint = torch.load(path, map_location=self.device)
- self.model.load_state_dict(checkpoint['model_state_dict'])
- self.optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
- self.scheduler.load_state_dict(checkpoint['scheduler_state_dict'])
+ # Load model state (with strict=False to handle architecture changes)
+ try:
+ self.model.load_state_dict(checkpoint['model_state_dict'], strict=False)
+ except Exception as e:
+ logger.warning(f"Error loading model state dict: {e}, continuing with partial load")
+
+ # Load optimizer state (handle mismatched states gracefully)
+ try:
+ optimizer_state = checkpoint.get('optimizer_state_dict')
+ if optimizer_state:
+ try:
+ # Try to load optimizer state
+ self.optimizer.load_state_dict(optimizer_state)
+ except (KeyError, ValueError, RuntimeError) as e:
+ logger.warning(f"Error loading optimizer state: {e}. Resetting optimizer.")
+ # Recreate optimizer (same pattern as __init__)
+ self.optimizer = torch.optim.AdamW(
+ self.model.parameters(),
+ lr=self.config.learning_rate,
+ weight_decay=self.config.weight_decay
+ )
+ else:
+ logger.warning("No optimizer state found in checkpoint. Using fresh optimizer.")
+ except Exception as e:
+ logger.warning(f"Error loading optimizer state: {e}. Resetting optimizer.")
+ # Recreate optimizer (same pattern as __init__)
+ self.optimizer = torch.optim.AdamW(
+ self.model.parameters(),
+ lr=self.config.learning_rate,
+ weight_decay=self.config.weight_decay
+ )
+
+ # Load scheduler state
+ try:
+ scheduler_state = checkpoint.get('scheduler_state_dict')
+ if scheduler_state:
+ self.scheduler.load_state_dict(scheduler_state)
+ except Exception as e:
+ logger.warning(f"Error loading scheduler state: {e}, continuing without scheduler state")
+
self.training_history = checkpoint.get('training_history', self.training_history)
logger.info(f"Model loaded from {path}")
diff --git a/PLACEHOLDERS_AND_MISSING_IMPLEMENTATIONS.md b/PLACEHOLDERS_AND_MISSING_IMPLEMENTATIONS.md
new file mode 100644
index 0000000..f5dc911
--- /dev/null
+++ b/PLACEHOLDERS_AND_MISSING_IMPLEMENTATIONS.md
@@ -0,0 +1,208 @@
+# Placeholders and Missing Implementations Report
+
+**Generated**: 2025-11-23
+**Purpose**: Identify all TODO, placeholder, and missing implementations that violate "no synthetic data" policy
+
+---
+
+## 🔴 **CRITICAL - Synthetic Data Violations**
+
+### 1. **core/negative_case_trainer.py** (Lines 396-397)
+**Issue**: Uses `np.random.uniform()` for synthetic training improvements
+```python
+session.loss_improvement = np.random.uniform(0.1, 0.5) # 10-50% improvement
+session.accuracy_improvement = np.random.uniform(0.05, 0.2) # 5-20% improvement
+```
+**Fix Required**: Calculate actual improvements from real training metrics
+
+---
+
+## 🟡 **HIGH PRIORITY - Missing Core Functionality**
+
+### 2. **core/orchestrator.py** (Line 2020)
+**Issue**: `_get_all_predictions()` returns empty list - not implemented
+```python
+async def _get_all_predictions(self, symbol: str) -> List[Prediction]:
+ predictions = []
+ # TODO: Implement proper prediction gathering from all registered models
+ logger.warning(f"_get_all_predictions not fully implemented for {symbol}")
+ return predictions
+```
+**Impact**: Orchestrator cannot gather predictions from all models for decision fusion
+**Fix Required**: Implement actual prediction gathering from registered models
+
+### 3. **ANNOTATE/core/real_training_adapter.py** (Line 2339-2341)
+**Issue**: Extrema training uses placeholder loss
+```python
+# TODO: Implement actual extrema training
+session.current_loss = 0.5 / (epoch + 1) # Placeholder
+```
+**Fix Required**: Implement real extrema training logic
+
+### 4. **ANNOTATE/core/real_training_adapter.py** (Line 1577)
+**Issue**: Placeholder `time_in_position_minutes`
+```python
+time_in_position_minutes = 1.0 # Placeholder, will be more accurate with actual timestamps
+```
+**Fix Required**: Calculate actual time from entry timestamp
+
+---
+
+## 🟢 **MEDIUM PRIORITY - Missing Features**
+
+### 5. **web/clean_dashboard.py** (Lines 8759, 8768)
+**Issue**: TODO for technical indicators and pivot points
+```python
+def _get_technical_indicators(self, symbol: str) -> Dict[str, float]:
+ # TODO: Implement technical indicators calculation
+ return {}
+
+def _get_pivot_points(self, symbol: str) -> List['PivotPoint']:
+ # TODO: Implement pivot points calculation
+ return []
+```
+**Note**: Pivot points ARE implemented elsewhere (Williams Market Structure), but not in this method
+**Fix Required**: Implement or delegate to existing pivot calculation
+
+### 6. **web/clean_dashboard.py** (Line 8665)
+**Issue**: TODO for cross-model predictions
+```python
+last_predictions={} # TODO: Add cross-model predictions
+```
+**Fix Required**: Gather predictions from all models (similar to orchestrator)
+
+### 7. **web/clean_dashboard.py** (Line 8696)
+**Issue**: TODO for technical indicators in bar data
+```python
+indicators={} # TODO: Add technical indicators
+```
+**Fix Required**: Calculate and include technical indicators
+
+### 8. **web/clean_dashboard.py** (Line 9542)
+**Issue**: Placeholder features array
+```python
+'features': [current_price, 0, 0, 0, 0] # Placeholder features
+```
+**Fix Required**: Extract real features from market data
+
+---
+
+## 🔵 **LOW PRIORITY - Acceptable Placeholders**
+
+### 9. **ANNOTATE/core/real_training_adapter.py** (Line 1421)
+**Issue**: Placeholder COB data (zeros)
+```python
+# Create placeholder COB data (zeros if not available)
+cob_data = torch.zeros(1, target_seq_len, 100, dtype=torch.float32)
+```
+**Status**: ✅ **ACCEPTABLE** - Returns zeros when COB data unavailable (not synthetic, just missing)
+
+### 10. **ANNOTATE/core/real_training_adapter.py** (Lines 2746-2748)
+**Issue**: Placeholder tech/market/COB data (zeros)
+```python
+data['tech_data'] = torch.zeros(1, 40, dtype=torch.float32)
+data['market_data'] = torch.zeros(1, 30, dtype=torch.float32)
+data['cob_data'] = torch.zeros(1, 600, 100, dtype=torch.float32)
+```
+**Status**: ✅ **ACCEPTABLE** - Model requires these inputs, zeros are safe defaults when unavailable
+
+### 11. **ANNOTATE/core/real_training_adapter.py** (Line 2887)
+**Issue**: Placeholder action in annotation conversion
+```python
+'action': 'BUY', # Placeholder, not used for candle prediction training
+```
+**Status**: ✅ **ACCEPTABLE** - Comment indicates it's not used for this training path
+
+---
+
+## ⚠️ **QUESTIONABLE - Needs Review**
+
+### 12. **core/orchestrator.py** (Lines 2098-2101)
+**Issue**: Uses `random.uniform()` for tie-breaking
+```python
+import random
+for action in action_scores:
+ # Add tiny random noise (±0.001) to break exact ties
+ action_scores[action] += random.uniform(-0.001, 0.001)
+```
+**Status**: ⚠️ **QUESTIONABLE** - This is for tie-breaking, not synthetic data generation
+**Recommendation**: Consider deterministic tie-breaking (e.g., alphabetical order) instead
+
+---
+
+## 📋 **Other TODOs Found**
+
+### 13. **ANNOTATE/web/app.py** (Lines 2745, 2850)
+**Issue**: Hardcoded symbols
+```python
+for symbol in ['ETH/USDT', 'BTC/USDT']: # TODO: Get from active subscriptions
+symbol = 'ETH/USDT' # TODO: Get from active trading pair
+```
+**Fix Required**: Get from active subscriptions/trading pairs
+
+### 14. **ANNOTATE/web/static/js/chart_manager.js** (Line 1193)
+**Issue**: TODO for visual markers
+```python
+# TODO: Add visual markers using Plotly annotations
+```
+**Fix Required**: Add visual markers if needed
+
+### 15. **ANNOTATE/web/templates/components/inference_panel.html** (Line 259)
+**Issue**: TODO for Plotly chart update
+```python
+// TODO: Update Plotly chart with prediction marker
+```
+**Fix Required**: Implement prediction marker update
+
+### 16. **ANNOTATE/core/data_loader.py** (Line 434)
+**Issue**: MEXC time range fetch not implemented
+```python
+logger.warning("MEXC time range fetch not implemented yet")
+```
+**Fix Required**: Implement MEXC time range fetch or remove if not needed
+
+### 17. **core/multi_horizon_prediction_manager.py** (Lines 690-713)
+**Issue**: Placeholder methods for CNN/RL feature preparation
+```python
+def _prepare_cnn_features_for_horizon(...) -> np.ndarray:
+ """Prepare CNN features for specific horizon (placeholder - not yet implemented)"""
+ return np.array([]) # Return empty array instead of synthetic data
+
+def _prepare_rl_state_for_horizon(...) -> np.ndarray:
+ """Prepare RL state for specific horizon (placeholder - not yet implemented)"""
+ return np.array([]) # Return empty array instead of synthetic data
+```
+**Status**: ✅ **ACCEPTABLE** - Returns empty arrays (not synthetic data), methods not yet needed
+
+---
+
+## 📊 **Summary**
+
+### By Priority:
+- **🔴 Critical (Synthetic Data)**: 1 issue
+- **🟡 High Priority (Missing Core)**: 3 issues
+- **🟢 Medium Priority (Missing Features)**: 4 issues
+- **🔵 Low Priority (Acceptable)**: 3 issues
+- **⚠️ Questionable**: 1 issue
+- **📋 Other TODOs**: 5 issues
+
+### Total Issues Found: 17
+
+---
+
+## 🎯 **Recommended Fix Order**
+
+1. **Fix synthetic data violation** (negative_case_trainer.py) - **URGENT**
+2. **Implement `_get_all_predictions()`** (orchestrator.py) - **HIGH**
+3. **Implement extrema training** (real_training_adapter.py) - **HIGH**
+4. **Fix time_in_position calculation** (real_training_adapter.py) - **MEDIUM**
+5. **Implement technical indicators** (clean_dashboard.py) - **MEDIUM**
+6. **Review tie-breaking logic** (orchestrator.py) - **LOW**
+
+---
+
+## ✅ **Already Fixed**
+
+- ✅ `_get_live_prediction()` - Now uses real model inference with caching
+- ✅ Ghost candles - Now includes `predicted_candle` in predictions
+- ✅ JSON serialization - Fixed tensor serialization errors
diff --git a/core/negative_case_trainer.py b/core/negative_case_trainer.py
index b0f6f34..9346e34 100644
--- a/core/negative_case_trainer.py
+++ b/core/negative_case_trainer.py
@@ -392,9 +392,13 @@ class NegativeCaseTrainer:
case.retraining_count += 1
case.last_retrained = datetime.now()
- # Calculate improvements (simulated)
- session.loss_improvement = np.random.uniform(0.1, 0.5) # 10-50% improvement
- session.accuracy_improvement = np.random.uniform(0.05, 0.2) # 5-20% improvement
+ # Calculate improvements from actual training metrics (NO SYNTHETIC DATA)
+ # If actual training metrics are not available, set to 0.0 instead of random values
+ # TODO: Replace with actual model training that returns real loss/accuracy improvements
+ session.loss_improvement = 0.0 # Set to 0 until real training metrics available
+ session.accuracy_improvement = 0.0 # Set to 0 until real training metrics available
+
+ logger.warning(f"Training session completed but improvements not calculated - intensive training not yet implemented")
# Store training session results
self._store_training_session(session)
diff --git a/core/orchestrator.py b/core/orchestrator.py
index 798b6f9..c5fcb3d 100644
--- a/core/orchestrator.py
+++ b/core/orchestrator.py
@@ -2097,13 +2097,20 @@ class TradingOrchestrator:
# Choose best action - safe way to handle max with key function
if action_scores:
- # Add small random component to break ties and prevent pure bias
- import random
- for action in action_scores:
- # Add tiny random noise (±0.001) to break exact ties
- action_scores[action] += random.uniform(-0.001, 0.001)
+ # Break exact ties deterministically (NO RANDOM DATA)
+ # Use action order as tie-breaker: BUY > SELL > HOLD
+ action_order = {'BUY': 3, 'SELL': 2, 'HOLD': 1}
+
+ # Find max score
+ max_score = max(action_scores.values())
+
+ # If multiple actions have same score, prefer BUY > SELL > HOLD
+ tied_actions = [action for action, score in action_scores.items() if score == max_score]
+ if len(tied_actions) > 1:
+ best_action = max(tied_actions, key=lambda a: action_order.get(a, 0))
+ else:
+ best_action = tied_actions[0]
- best_action = max(action_scores.keys(), key=lambda k: action_scores[k])
best_confidence = action_scores[best_action]
# DEBUG: Log action scores to understand bias