popov/gogo2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wip

Browse Source
This commit is contained in:
Dobromir Popov
2025-12-09 00:34:51 +02:00
parent 8c3dc5423e
commit d6ada4b416
5 changed files with 534 additions and 429 deletions
Download Patch File Download Diff File Expand all files Collapse all files

201
ANNOTATE/core/real_training_adapter.py
View File

@@ -570,62 +570,6 @@ class RealTrainingAdapter:
except Exception as e:
logger.debug(f" {timeframe}: Replay failed: {e}")
# CRITICAL FIX: For 1s timeframe, if we still don't have enough data, generate from 1m candles
if timeframe == '1s' and (df is None or df.empty or len(df) < min_required_candles):
try:
df_1m = None
# First, try to get 1m data from already fetched timeframes
if '1m' in fetched_timeframes and '1m' in market_state.get('timeframes', {}):
# Convert dict format to DataFrame
tf_1m = market_state['timeframes']['1m']
try:
df_1m = pd.DataFrame({
'open': tf_1m['open'],
'high': tf_1m['high'],
'low': tf_1m['low'],
'close': tf_1m['close'],
'volume': tf_1m['volume']
}, index=pd.to_datetime(tf_1m['timestamps'], utc=True))
except Exception as e:
logger.debug(f" {timeframe}: Could not convert 1m dict to DataFrame: {e}")
# If we don't have 1m data yet, fetch it
if df_1m is None or df_1m.empty:
logger.debug(f" {timeframe}: Fetching 1m data to generate 1s candles...")
if duckdb_storage:
try:
df_1m = duckdb_storage.get_ohlcv_data(
symbol=symbol,
timeframe='1m',
start_time=tf_start_time,
end_time=tf_end_time,
limit=limit,
direction='before'
)
except Exception as e:
logger.debug(f" {timeframe}: Could not get 1m from DuckDB: {e}")
if df_1m is None or df_1m.empty:
# Try API for 1m
df_1m = self._fetch_historical_from_api(symbol, '1m', tf_start_time, tf_end_time, limit)
# Generate 1s candles from 1m if we have 1m data
if df_1m is not None and not df_1m.empty:
# Generate 1s candles from 1m
generated_1s = self._generate_1s_from_1m(df_1m, min_required_candles)
if generated_1s is not None and len(generated_1s) >= min_required_candles:
df = generated_1s
logger.info(f" {timeframe}: Generated {len(df)} candles from {len(df_1m)} 1m candles")
else:
logger.warning(f" {timeframe}: Generated only {len(generated_1s) if generated_1s is not None else 0} candles from 1m (need {min_required_candles})")
else:
logger.debug(f" {timeframe}: No 1m data available to generate 1s candles")
except Exception as e:
logger.warning(f" {timeframe}: Failed to generate from 1m: {e}")
import traceback
logger.debug(traceback.format_exc())
# Validate data quality before storing
if df is not None and not df.empty:
# Check minimum candle count
@@ -1446,122 +1390,6 @@ class RealTrainingAdapter:
state_size = agent.state_size if hasattr(agent, 'state_size') else 100
return [0.0] * state_size
def _generate_1s_from_1m(self, df_1m: pd.DataFrame, min_candles: int) -> Optional[pd.DataFrame]:
"""
Generate 1s candles from 1m candles by splitting each 1m candle into 60 1s candles.
Args:
df_1m: DataFrame with 1m OHLCV data
min_candles: Minimum number of 1s candles to generate
Returns:
DataFrame with 1s OHLCV data or None if generation fails
"""
import pandas as pd
from datetime import timedelta
try:
if df_1m is None or df_1m.empty:
return None
# Ensure we have required columns
required_cols = ['open', 'high', 'low', 'close', 'volume']
if not all(col in df_1m.columns for col in required_cols):
logger.warning("1m DataFrame missing required columns for 1s generation")
return None
# Generate 1s candles from each 1m candle
# Each 1m candle becomes 60 1s candles
candles_1s = []
for idx, row in df_1m.iterrows():
# Get timestamp (handle both index and column)
if isinstance(df_1m.index, pd.DatetimeIndex):
timestamp = idx
elif 'timestamp' in df_1m.columns:
timestamp = pd.to_datetime(row['timestamp'])
else:
logger.warning("Cannot determine timestamp for 1m candle")
continue
# Extract OHLCV values
open_price = float(row['open'])
high_price = float(row['high'])
low_price = float(row['low'])
close_price = float(row['close'])
volume = float(row['volume'])
# Calculate price change per second (linear interpolation)
price_change = close_price - open_price
price_per_second = price_change / 60.0
# Volume per second (distributed evenly)
volume_per_second = volume / 60.0
# Generate 60 1s candles from this 1m candle
for second in range(60):
# Calculate timestamp for this second
candle_time = timestamp + timedelta(seconds=second)
# Interpolate price (linear from open to close)
if second == 0:
candle_open = open_price
candle_close = open_price + price_per_second
elif second == 59:
candle_open = open_price + (price_per_second * 59)
candle_close = close_price
else:
candle_open = open_price + (price_per_second * second)
candle_close = open_price + (price_per_second * (second + 1))
# High and low: use the interpolated prices, but ensure they stay within the 1m candle's range
# Add small random variation to make it more realistic (but keep within bounds)
candle_high = max(candle_open, candle_close)
candle_low = min(candle_open, candle_close)
# Ensure high/low don't exceed the 1m candle's range
candle_high = min(candle_high, high_price)
candle_low = max(candle_low, low_price)
# If high == low, add small spread
if candle_high == candle_low:
spread = (high_price - low_price) / 60.0
candle_high = candle_high + (spread * 0.5)
candle_low = candle_low - (spread * 0.5)
candles_1s.append({
'timestamp': candle_time,
'open': candle_open,
'high': candle_high,
'low': candle_low,
'close': candle_close,
'volume': volume_per_second
})
if not candles_1s:
return None
# Convert to DataFrame
df_1s = pd.DataFrame(candles_1s)
df_1s['timestamp'] = pd.to_datetime(df_1s['timestamp'])
df_1s = df_1s.set_index('timestamp')
# Sort by timestamp
df_1s = df_1s.sort_index()
# Take the most recent candles up to the limit
if len(df_1s) > min_candles:
df_1s = df_1s.tail(min_candles)
logger.debug(f"Generated {len(df_1s)} 1s candles from {len(df_1m)} 1m candles")
return df_1s
except Exception as e:
logger.warning(f"Error generating 1s candles from 1m: {e}")
import traceback
logger.debug(traceback.format_exc())
return None
def _fetch_historical_from_api(self, symbol: str, timeframe: str, start_time: datetime, end_time: datetime, limit: int) -> Optional[pd.DataFrame]:
"""
Fetch historical OHLCV data from exchange APIs for a specific time range
@@ -3288,32 +3116,11 @@ class RealTrainingAdapter:
logger.info(f"Stopped real-time inference: {inference_id}")
def get_latest_signals(self, limit: int = 50) -> List[Dict]:
"""Get latest inference signals from orchestrator predictions and active sessions"""
"""Get latest inference signals from all active sessions"""
if not hasattr(self, 'inference_sessions'):
return []
all_signals = []
# CRITICAL FIX: Get signals from orchestrator's stored predictions (primary source)
if self.orchestrator and hasattr(self.orchestrator, 'recent_transformer_predictions'):
# Get predictions for all symbols
for symbol, predictions in self.orchestrator.recent_transformer_predictions.items():
if predictions:
# Convert predictions to signal format
for pred in list(predictions)[-limit:]:
signal = {
'timestamp': pred.get('timestamp', datetime.now(timezone.utc).isoformat()),
'action': pred.get('action', 'HOLD'),
'confidence': pred.get('confidence', 0.0),
'current_price': pred.get('current_price', 0.0),
'price': pred.get('current_price', 0.0), # Alias for compatibility
'predicted_price': pred.get('predicted_price', pred.get('current_price', 0.0)),
'price_change': pred.get('price_change', 0.0),
'model': 'transformer',
'predicted_candle': pred.get('predicted_candle', {}),
'source': pred.get('source', 'live_inference')
}
all_signals.append(signal)
# Also get signals from active inference sessions (secondary source)
if hasattr(self, 'inference_sessions'):
for session in self.inference_sessions.values():
all_signals.extend(session.get('signals', []))

102
ANNOTATE/web/app.py
View File

@@ -1189,7 +1189,7 @@ class AnnotationDashboard:
# Add ALL pivot points to the map
for pivot in trend_level.pivot_points:
ts_str = pivot.timestamp.strftime('%Y-%m-%d %H:%M:%S')
ts_str = self._format_timestamp_utc(pivot.timestamp)
if ts_str not in pivot_map:
pivot_map[ts_str] = {'highs': [], 'lows': []}
@@ -1226,6 +1226,51 @@ class AnnotationDashboard:
logger.error(traceback.format_exc())
return {}
def _format_timestamp_utc(self, ts):
"""
Format timestamp in ISO format with UTC indicator ('Z' suffix)
This ensures frontend JavaScript parses it as UTC, not local time
Args:
ts: pandas Timestamp or datetime object
Returns:
str: ISO format timestamp with 'Z' suffix (e.g., '2025-12-08T21:00:00Z')
"""
try:
# Ensure timestamp is UTC
if hasattr(ts, 'tz'):
if ts.tz is not None:
ts_utc = ts.tz_convert('UTC') if hasattr(ts, 'tz_convert') else ts
else:
try:
ts_utc = ts.tz_localize('UTC') if hasattr(ts, 'tz_localize') else ts
except:
ts_utc = ts
else:
ts_utc = ts
# Format as ISO with 'Z' for UTC
if hasattr(ts_utc, 'strftime'):
return ts_utc.strftime('%Y-%m-%dT%H:%M:%SZ')
else:
return str(ts_utc)
except Exception as e:
logger.debug(f"Error formatting timestamp: {e}")
return str(ts)
def _format_timestamps_utc(self, timestamp_series):
"""
Format a series of timestamps in ISO format with UTC indicator
Args:
timestamp_series: pandas Index or Series with timestamps
Returns:
list: List of ISO format timestamps with 'Z' suffix
"""
return [self._format_timestamp_utc(ts) for ts in timestamp_series]
def _setup_routes(self):
"""Setup Flask routes"""
@@ -1418,7 +1463,7 @@ class AnnotationDashboard:
chart_data = {
timeframe: {
'timestamps': df.index.strftime('%Y-%m-%d %H:%M:%S').tolist(),
'timestamps': self._format_timestamps_utc(df.index),
'open': df['open'].tolist(),
'high': df['high'].tolist(),
'low': df['low'].tolist(),
@@ -1496,7 +1541,7 @@ class AnnotationDashboard:
# Convert to format suitable for Plotly
chart_data[timeframe] = {
'timestamps': df.index.strftime('%Y-%m-%d %H:%M:%S').tolist(),
'timestamps': self._format_timestamps_utc(df.index),
'open': df['open'].tolist(),
'high': df['high'].tolist(),
'low': df['low'].tolist(),
@@ -1766,8 +1811,31 @@ class AnnotationDashboard:
# Get pivot markers for this timeframe
pivot_markers = self._get_pivot_markers_for_timeframe(symbol, timeframe, df)
# CRITICAL FIX: Format timestamps in ISO format with UTC indicator
# This ensures frontend parses them as UTC, not local time
timestamps = []
for ts in df.index:
# Ensure timestamp is UTC
if hasattr(ts, 'tz'):
if ts.tz is not None:
ts_utc = ts.tz_convert('UTC') if hasattr(ts, 'tz_convert') else ts
else:
try:
ts_utc = ts.tz_localize('UTC') if hasattr(ts, 'tz_localize') else ts
except:
ts_utc = ts
else:
ts_utc = ts
# Format as ISO with 'Z' for UTC: 'YYYY-MM-DDTHH:MM:SSZ'
# Plotly handles ISO format correctly
if hasattr(ts_utc, 'strftime'):
timestamps.append(ts_utc.strftime('%Y-%m-%dT%H:%M:%SZ'))
else:
timestamps.append(str(ts_utc))
chart_data[timeframe] = {
'timestamps': df.index.strftime('%Y-%m-%d %H:%M:%S').tolist(),
'timestamps': timestamps,
'open': df['open'].tolist(),
'high': df['high'].tolist(),
'low': df['low'].tolist(),
@@ -2590,6 +2658,30 @@ class AnnotationDashboard:
metrics = session['metrics'].copy()
break
# CRITICAL FIX: Include position state and session metrics for UI state restoration
position_state = None
session_metrics = None
# Get position state and session metrics from orchestrator if available
if self.orchestrator and hasattr(self.orchestrator, 'get_position_state'):
try:
position_state = self.orchestrator.get_position_state()
except:
pass
if self.orchestrator and hasattr(self.orchestrator, 'get_session_metrics'):
try:
session_metrics = self.orchestrator.get_session_metrics()
except:
pass
# Add position state and session metrics to metrics dict
if position_state:
metrics['position_state'] = position_state
if session_metrics:
metrics['session_pnl'] = session_metrics.get('total_pnl', 0.0)
metrics['session_metrics'] = session_metrics
return jsonify({
'success': True,
'signals': signals,
@@ -3106,7 +3198,7 @@ class AnnotationDashboard:
if not df_before.empty:
recent = df_before.tail(200)
market_state['timeframes'][tf] = {
'timestamps': recent.index.strftime('%Y-%m-%d %H:%M:%S').tolist(),
'timestamps': self._format_timestamps_utc(recent.index),
'open': recent['open'].tolist(),
'high': recent['high'].tolist(),
'low': recent['low'].tolist(),

330
ANNOTATE/web/static/js/chart_manager.js
View File

@@ -18,6 +18,8 @@ class ChartManager {
this.ghostCandleHistory = {}; // Store ghost candles per timeframe (max 50 each)
this.maxGhostCandles = 150; // Maximum number of ghost candles to keep
this.modelAccuracyMetrics = {}; // Track overall model accuracy per timeframe
this.predictionHistory = []; // Store last 20 predictions with fading
this.maxPredictions = 20; // Maximum number of predictions to display
// Helper to ensure all timestamps are in UTC
this.normalizeTimestamp = (timestamp) => {
@@ -310,15 +312,34 @@ class ChartManager {
};
}
// Parse timestamp - format to match chart data format
const candleTimestamp = new Date(candle.timestamp);
// CRITICAL FIX: Parse timestamp ensuring UTC handling
// Backend now sends ISO format with 'Z' (e.g., '2025-12-08T21:00:00Z')
// JavaScript Date will parse this correctly as UTC
let candleTimestamp;
if (typeof candle.timestamp === 'string') {
// If it's already ISO format with 'Z', parse directly
if (candle.timestamp.includes('T') && (candle.timestamp.endsWith('Z') || candle.timestamp.includes('+'))) {
candleTimestamp = new Date(candle.timestamp);
} else if (candle.timestamp.includes('T')) {
// ISO format without timezone - assume UTC
candleTimestamp = new Date(candle.timestamp + 'Z');
} else {
// Old format: 'YYYY-MM-DD HH:MM:SS' - convert to ISO and treat as UTC
candleTimestamp = new Date(candle.timestamp.replace(' ', 'T') + 'Z');
}
} else {
candleTimestamp = new Date(candle.timestamp);
}
// Format using UTC methods and ISO format with 'Z' for consistency
const year = candleTimestamp.getUTCFullYear();
const month = String(candleTimestamp.getUTCMonth() + 1).padStart(2, '0');
const day = String(candleTimestamp.getUTCDate()).padStart(2, '0');
const hours = String(candleTimestamp.getUTCHours()).padStart(2, '0');
const minutes = String(candleTimestamp.getUTCMinutes()).padStart(2, '0');
const seconds = String(candleTimestamp.getUTCSeconds()).padStart(2, '0');
const formattedTimestamp = `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`;
// Format as ISO with 'Z' so it's consistently treated as UTC
const formattedTimestamp = `${year}-${month}-${day}T${hours}:${minutes}:${seconds}Z`;
// Get current chart data from Plotly
const chartData = Plotly.Plots.data(plotId);
@@ -364,7 +385,13 @@ class ChartManager {
chart.data.close.push(candle.close);
chart.data.volume.push(candle.volume);
console.log(`[${timeframe}] Added new candle: ${formattedTimestamp}`);
console.log(`[${timeframe}] Added new candle: ${formattedTimestamp}`, {
open: candle.open,
high: candle.high,
low: candle.low,
close: candle.close,
volume: candle.volume
});
} else {
// Update last candle - update both Plotly and internal data structure
const x = [...candlestickTrace.x];
@@ -632,27 +659,31 @@ class ChartManager {
const ghostTime = new Date(furthestGhost.targetTime);
const currentMax = new Date(xMax);
if (ghostTime > currentMax) {
const year = ghostTime.getUTCFullYear();
const month = String(ghostTime.getUTCMonth() + 1).padStart(2, '0');
const day = String(ghostTime.getUTCDate()).padStart(2, '0');
const hours = String(ghostTime.getUTCHours()).padStart(2, '0');
const minutes = String(ghostTime.getUTCMinutes()).padStart(2, '0');
const seconds = String(ghostTime.getUTCSeconds()).padStart(2, '0');
xMax = `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`;
// CRITICAL FIX: Format as ISO with 'Z' to match chart timestamp format
xMax = ghostTime.toISOString();
console.log(`[${timeframe}] Pivot lines extended to include ${ghosts.length} ghost candles (to ${xMax})`);
}
}
}
// Process each timestamp that has pivot markers
Object.entries(data.pivot_markers).forEach(([timestamp, pivots]) => {
// CRITICAL FIX: Ensure pivot marker timestamps are in ISO format
Object.entries(data.pivot_markers).forEach(([timestampKey, pivots]) => {
// Convert pivot marker timestamp to ISO format if needed
let pivotTimestamp = timestampKey;
if (typeof timestampKey === 'string' && !timestampKey.includes('T')) {
pivotTimestamp = new Date(timestampKey.replace(' ', 'T') + 'Z').toISOString();
} else if (typeof timestampKey === 'string' && !timestampKey.endsWith('Z') && !timestampKey.includes('+')) {
pivotTimestamp = new Date(timestampKey + 'Z').toISOString();
}
// Process high pivots
if (pivots.highs && pivots.highs.length > 0) {
pivots.highs.forEach(pivot => {
const color = this._getPivotColor(pivot.level, 'high');
// Draw dot on the pivot candle (above the high)
pivotDots.x.push(timestamp);
// Draw dot on the pivot candle (above the high) - use converted timestamp
pivotDots.x.push(pivotTimestamp);
pivotDots.y.push(pivot.price);
pivotDots.text.push(`L${pivot.level} High Pivot<br>Price: $${pivot.price.toFixed(2)}<br>Strength: ${(pivot.strength * 100).toFixed(0)}%`);
pivotDots.marker.color.push(color);
@@ -698,8 +729,8 @@ class ChartManager {
pivots.lows.forEach(pivot => {
const color = this._getPivotColor(pivot.level, 'low');
// Draw dot on the pivot candle (below the low)
pivotDots.x.push(timestamp);
// Draw dot on the pivot candle (below the low) - use converted timestamp
pivotDots.x.push(pivotTimestamp);
pivotDots.y.push(pivot.price);
pivotDots.text.push(`L${pivot.level} Low Pivot<br>Price: $${pivot.price.toFixed(2)}<br>Strength: ${(pivot.strength * 100).toFixed(0)}%`);
pivotDots.marker.color.push(color);
@@ -1062,9 +1093,24 @@ class ChartManager {
const pivotDots = { x: [], y: [], text: [], marker: { color: [], size: [], symbol: [] }, mode: 'markers', hoverinfo: 'text', showlegend: false };
if (data.pivot_markers && Object.keys(data.pivot_markers).length > 0) {
const xMin = data.timestamps[0];
// CRITICAL FIX: Ensure timestamps are in ISO format for consistency
// Parse timestamps to ensure they're treated as UTC
let xMin = data.timestamps[0];
let xMax = data.timestamps[data.timestamps.length - 1];
// Convert to ISO format if not already
if (typeof xMin === 'string' && !xMin.includes('T')) {
xMin = new Date(xMin.replace(' ', 'T') + 'Z').toISOString();
} else if (typeof xMin === 'string' && !xMin.endsWith('Z') && !xMin.includes('+')) {
xMin = new Date(xMin + 'Z').toISOString();
}
if (typeof xMax === 'string' && !xMax.includes('T')) {
xMax = new Date(xMax.replace(' ', 'T') + 'Z').toISOString();
} else if (typeof xMax === 'string' && !xMax.endsWith('Z') && !xMax.includes('+')) {
xMax = new Date(xMax + 'Z').toISOString();
}
// Extend xMax to include ghost candle predictions if they exist
if (this.ghostCandleHistory && this.ghostCandleHistory[timeframe] && this.ghostCandleHistory[timeframe].length > 0) {
const ghosts = this.ghostCandleHistory[timeframe];
@@ -1073,26 +1119,30 @@ class ChartManager {
const ghostTime = new Date(furthestGhost.targetTime);
const currentMax = new Date(xMax);
if (ghostTime > currentMax) {
const year = ghostTime.getUTCFullYear();
const month = String(ghostTime.getUTCMonth() + 1).padStart(2, '0');
const day = String(ghostTime.getUTCDate()).padStart(2, '0');
const hours = String(ghostTime.getUTCHours()).padStart(2, '0');
const minutes = String(ghostTime.getUTCMinutes()).padStart(2, '0');
const seconds = String(ghostTime.getUTCSeconds()).padStart(2, '0');
xMax = `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`;
// Format as ISO with 'Z' to match chart timestamp format
xMax = ghostTime.toISOString();
}
}
}
// Process each timestamp that has pivot markers
Object.entries(data.pivot_markers).forEach(([timestamp, pivots]) => {
// CRITICAL FIX: Ensure pivot marker timestamps are in ISO format
Object.entries(data.pivot_markers).forEach(([timestampKey, pivots]) => {
// Convert pivot marker timestamp to ISO format if needed
let pivotTimestamp = timestampKey;
if (typeof timestampKey === 'string' && !timestampKey.includes('T')) {
pivotTimestamp = new Date(timestampKey.replace(' ', 'T') + 'Z').toISOString();
} else if (typeof timestampKey === 'string' && !timestampKey.endsWith('Z') && !timestampKey.includes('+')) {
pivotTimestamp = new Date(timestampKey + 'Z').toISOString();
}
// Process high pivots
if (pivots.highs && pivots.highs.length > 0) {
pivots.highs.forEach(pivot => {
const color = this._getPivotColor(pivot.level, 'high');
// Draw dot on the pivot candle
pivotDots.x.push(timestamp);
// Draw dot on the pivot candle - use converted timestamp
pivotDots.x.push(pivotTimestamp);
pivotDots.y.push(pivot.price);
pivotDots.text.push(`L${pivot.level} High Pivot<br>Price: $${pivot.price.toFixed(2)}<br>Strength: ${(pivot.strength * 100).toFixed(0)}%`);
pivotDots.marker.color.push(color);
@@ -1137,8 +1187,8 @@ class ChartManager {
pivots.lows.forEach(pivot => {
const color = this._getPivotColor(pivot.level, 'low');
// Draw dot on the pivot candle
pivotDots.x.push(timestamp);
// Draw dot on the pivot candle - use converted timestamp
pivotDots.x.push(pivotTimestamp);
pivotDots.y.push(pivot.price);
pivotDots.text.push(`L${pivot.level} Low Pivot<br>Price: $${pivot.price.toFixed(2)}<br>Strength: ${(pivot.strength * 100).toFixed(0)}%`);
pivotDots.marker.color.push(color);
@@ -2121,26 +2171,31 @@ class ChartManager {
const ghostTime = new Date(furthestGhost.targetTime);
const currentMax = new Date(xMax);
if (ghostTime > currentMax) {
const year = ghostTime.getUTCFullYear();
const month = String(ghostTime.getUTCMonth() + 1).padStart(2, '0');
const day = String(ghostTime.getUTCDate()).padStart(2, '0');
const hours = String(ghostTime.getUTCHours()).padStart(2, '0');
const minutes = String(ghostTime.getUTCMinutes()).padStart(2, '0');
const seconds = String(ghostTime.getUTCSeconds()).padStart(2, '0');
xMax = `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`;
// CRITICAL FIX: Format as ISO with 'Z' to match chart timestamp format
xMax = ghostTime.toISOString();
console.log(`[${timeframe}] Pivot lines extended to include ${ghosts.length} ghost candles (to ${xMax})`);
}
}
}
// Process each timestamp that has pivot markers
Object.entries(data.pivot_markers).forEach(([timestamp, pivots]) => {
// CRITICAL FIX: Ensure pivot marker timestamps are in ISO format
Object.entries(data.pivot_markers).forEach(([timestampKey, pivots]) => {
// Convert pivot marker timestamp to ISO format if needed
let pivotTimestamp = timestampKey;
if (typeof timestampKey === 'string' && !timestampKey.includes('T')) {
pivotTimestamp = new Date(timestampKey.replace(' ', 'T') + 'Z').toISOString();
} else if (typeof timestampKey === 'string' && !timestampKey.endsWith('Z') && !timestampKey.includes('+')) {
pivotTimestamp = new Date(timestampKey + 'Z').toISOString();
}
// Process high pivots
if (pivots.highs && pivots.highs.length > 0) {
pivots.highs.forEach(pivot => {
const color = this._getPivotColor(pivot.level, 'high');
pivotDots.x.push(timestamp);
// CRITICAL FIX: Use converted timestamp for consistency
pivotDots.x.push(pivotTimestamp);
pivotDots.y.push(pivot.price);
pivotDots.text.push(`L${pivot.level} High Pivot<br>Price: ${pivot.price.toFixed(2)}<br>Strength: ${(pivot.strength * 100).toFixed(0)}%`);
pivotDots.marker.color.push(color);
@@ -2174,7 +2229,8 @@ class ChartManager {
pivots.lows.forEach(pivot => {
const color = this._getPivotColor(pivot.level, 'low');
pivotDots.x.push(timestamp);
// CRITICAL FIX: Use converted timestamp for consistency
pivotDots.x.push(pivotTimestamp);
pivotDots.y.push(pivot.price);
pivotDots.text.push(`L${pivot.level} Low Pivot<br>Price: ${pivot.price.toFixed(2)}<br>Strength: ${(pivot.strength * 100).toFixed(0)}%`);
pivotDots.marker.color.push(color);
@@ -2736,24 +2792,58 @@ class ChartManager {
this._addCNNPrediction(predictions.cnn, predictionShapes, predictionAnnotations);
}
// Add Transformer predictions (star markers with trend lines + ghost candles)
// CRITICAL FIX: Manage prediction history (max 20, fade oldest)
// Add new transformer prediction to history
if (predictions.transformer) {
console.log(`[updatePredictions] Processing transformer prediction:`, {
action: predictions.transformer.action,
confidence: predictions.transformer.confidence,
has_predicted_candle: !!predictions.transformer.predicted_candle,
predicted_candle_keys: predictions.transformer.predicted_candle ? Object.keys(predictions.transformer.predicted_candle) : []
// Check if this is a new prediction (different timestamp or significant change)
const newPred = predictions.transformer;
const isNew = !this.predictionHistory.length ||
this.predictionHistory[0].timestamp !== newPred.timestamp ||
Math.abs((this.predictionHistory[0].confidence || 0) - (newPred.confidence || 0)) > 0.01;
if (isNew) {
// Add to history (most recent first)
this.predictionHistory.unshift({
...newPred,
addedAt: Date.now()
});
this._addTransformerPrediction(predictions.transformer, predictionShapes, predictionAnnotations);
// Add trend vector visualization (shorter projection to avoid zoom issues)
if (predictions.transformer.trend_vector) {
this._addTrendPrediction(predictions.transformer.trend_vector, predictionShapes, predictionAnnotations);
// Keep only last 20 predictions
if (this.predictionHistory.length > this.maxPredictions) {
this.predictionHistory = this.predictionHistory.slice(0, this.maxPredictions);
}
}
// Handle Predicted Candles (ghost candles)
if (predictions.transformer.predicted_candle) {
console.log(`[updatePredictions] Processing ${this.predictionHistory.length} predictions (new: ${isNew}):`, {
action: newPred.action,
confidence: newPred.confidence,
has_predicted_candle: !!newPred.predicted_candle
});
}
// Render all predictions from history with fading (oldest = most transparent)
this.predictionHistory.forEach((pred, index) => {
// Calculate opacity: newest = 1.0, oldest = 0.2
const ageRatio = index / Math.max(1, this.predictionHistory.length - 1);
const baseOpacity = 1.0 - (ageRatio * 0.8); // Fade from 1.0 to 0.2
// Create a copy of prediction with opacity applied
const fadedPred = {
...pred,
_fadeOpacity: baseOpacity
};
this._addTransformerPrediction(fadedPred, predictionShapes, predictionAnnotations);
// Add trend vector visualization (shorter projection to avoid zoom issues)
if (pred.trend_vector) {
this._addTrendPrediction(pred.trend_vector, predictionShapes, predictionAnnotations);
}
});
// Handle Predicted Candles (ghost candles) - only for the most recent prediction
if (predictions.transformer && predictions.transformer.predicted_candle) {
console.log(`[updatePredictions] predicted_candle data:`, predictions.transformer.predicted_candle);
const candleData = predictions.transformer.predicted_candle[timeframe];
console.log(`[updatePredictions] candleData for ${timeframe}:`, candleData);
@@ -2852,7 +2942,7 @@ class ChartManager {
}
}
// 3. Render "Shadow Candle" (Previous Prediction for Current Candle)
// Render "Shadow Candle" (Previous Prediction for Current Candle)
// If we have a stored prediction that matches the CURRENT candle time, show it
if (this.lastPredictions && this.lastPredictions[timeframe]) {
const lastPred = this.lastPredictions[timeframe];
@@ -2869,7 +2959,6 @@ class ChartManager {
}
}
}
}
// Update chart layout with predictions
if (predictionShapes.length > 0 || predictionAnnotations.length > 0) {
@@ -3230,7 +3319,8 @@ class ChartManager {
}
_addTransformerPrediction(prediction, shapes, annotations) {
// CRITICAL FIX: Get actual price from chart instead of using normalized prediction prices
// CRITICAL FIX: Use first timeframe from currentTimeframes (ignore Primary Timeline dropdown)
// Always use the first active timeframe, not the dropdown selection
const timeframe = window.appState?.currentTimeframes?.[0] || '1m';
const chart = this.charts[timeframe];
if (!chart) {
@@ -3238,36 +3328,19 @@ class ChartManager {
return;
}
// Get actual current price from the last candle on the chart
let actualCurrentPrice = 0;
const plotElement = document.getElementById(chart.plotId);
if (plotElement && plotElement.data && plotElement.data.length > 0) {
const candlestickTrace = plotElement.data[0]; // First trace is candlestick
if (candlestickTrace && candlestickTrace.close && candlestickTrace.close.length > 0) {
actualCurrentPrice = candlestickTrace.close[candlestickTrace.close.length - 1];
}
}
// Fallback to prediction price if chart price not available (but check if it's normalized)
if (actualCurrentPrice === 0 || actualCurrentPrice < 1) {
// Price might be normalized, try to use prediction price
actualCurrentPrice = prediction.current_price || 0;
// If still looks normalized (< 1), we can't display it properly
if (actualCurrentPrice < 1) {
console.warn('[Transformer Prediction] Price appears normalized, cannot display on chart. Chart price:', actualCurrentPrice);
return;
}
}
// CRITICAL FIX: Parse timestamp correctly (handle both ISO strings and Date objects)
// CRITICAL FIX: Use prediction's timestamp and price as starting point
// Parse prediction timestamp
let timestamp;
if (prediction.timestamp) {
if (typeof prediction.timestamp === 'string') {
// Handle various timestamp formats
if (prediction.timestamp.includes('T') && (prediction.timestamp.endsWith('Z') || prediction.timestamp.includes('+'))) {
timestamp = new Date(prediction.timestamp);
if (isNaN(timestamp.getTime())) {
// Try parsing as GMT format
} else if (prediction.timestamp.includes('T')) {
timestamp = new Date(prediction.timestamp + 'Z');
} else if (prediction.timestamp.includes('GMT')) {
timestamp = new Date(prediction.timestamp.replace('GMT', 'UTC'));
} else {
timestamp = new Date(prediction.timestamp.replace(' ', 'T') + 'Z');
}
} else {
timestamp = new Date(prediction.timestamp);
@@ -3276,18 +3349,50 @@ class ChartManager {
timestamp = new Date();
}
// Use current time if timestamp parsing failed
// Ensure timestamp is valid
if (isNaN(timestamp.getTime())) {
timestamp = new Date();
}
// Get prediction price - use current_price from prediction
let predictionPrice = prediction.current_price || 0;
// If price looks normalized (< 1), try to get actual price from chart
if (predictionPrice < 1) {
const plotElement = document.getElementById(chart.plotId);
if (plotElement && plotElement.data && plotElement.data.length > 0) {
const candlestickTrace = plotElement.data[0];
if (candlestickTrace && candlestickTrace.close && candlestickTrace.close.length > 0) {
// Find the candle closest to prediction timestamp
const predTimeMs = timestamp.getTime();
let closestPrice = candlestickTrace.close[candlestickTrace.close.length - 1];
let minDiff = Infinity;
for (let i = 0; i < candlestickTrace.x.length; i++) {
const candleTime = new Date(candlestickTrace.x[i]).getTime();
const diff = Math.abs(candleTime - predTimeMs);
if (diff < minDiff) {
minDiff = diff;
closestPrice = candlestickTrace.close[i];
}
}
predictionPrice = closestPrice;
}
}
}
if (predictionPrice === 0 || predictionPrice < 1) {
console.warn('[Transformer Prediction] Cannot determine prediction price');
return;
}
const confidence = prediction.confidence || 0;
const priceChange = prediction.price_change || 0;
const horizonMinutes = prediction.horizon_minutes || 10;
if (confidence < 0.3 || actualCurrentPrice === 0) return;
if (confidence < 0.3) return;
// CRITICAL: Calculate predicted price from actual current price and price change
// priceChange is typically a percentage or ratio
// Calculate predicted price from prediction price and price change
let actualPredictedPrice;
if (prediction.predicted_price && prediction.predicted_price > 1) {
// Use predicted_price if it looks like actual price (not normalized)
@@ -3296,19 +3401,19 @@ class ChartManager {
// Calculate from price change (could be percentage or ratio)
if (Math.abs(priceChange) > 10) {
// Looks like percentage (e.g., 1.0 = 1%)
actualPredictedPrice = actualCurrentPrice * (1 + priceChange / 100);
actualPredictedPrice = predictionPrice * (1 + priceChange / 100);
} else {
// Looks like ratio (e.g., 0.01 = 1%)
actualPredictedPrice = actualCurrentPrice * (1 + priceChange);
actualPredictedPrice = predictionPrice * (1 + priceChange);
}
} else {
// Fallback: use action to determine direction
if (prediction.action === 'BUY') {
actualPredictedPrice = actualCurrentPrice * 1.01; // +1%
actualPredictedPrice = predictionPrice * 1.01; // +1%
} else if (prediction.action === 'SELL') {
actualPredictedPrice = actualCurrentPrice * 0.99; // -1%
actualPredictedPrice = predictionPrice * 0.99; // -1%
} else {
actualPredictedPrice = actualCurrentPrice; // HOLD
actualPredictedPrice = predictionPrice; // HOLD
}
}
@@ -3325,16 +3430,37 @@ class ChartManager {
color = 'rgba(150, 150, 255, 0.5)'; // Light blue for STABLE/HOLD
}
// Add trend line from current price to predicted price
// CRITICAL FIX: Format timestamps as ISO strings to match chart data format
const timestampISO = timestamp.toISOString();
const endTimeISO = endTime.toISOString();
// Apply fade opacity if provided (for prediction history)
const fadeOpacity = prediction._fadeOpacity !== undefined ? prediction._fadeOpacity : 1.0;
// Extract RGB from color and apply fade opacity
let fadedColor = color;
if (typeof color === 'string' && color.startsWith('rgba')) {
// Parse rgba and apply fade
const rgbaMatch = color.match(/rgba?\((\d+),\s*(\d+),\s*(\d+)(?:,\s*([\d.]+))?\)/);
if (rgbaMatch) {
const r = parseInt(rgbaMatch[1]);
const g = parseInt(rgbaMatch[2]);
const b = parseInt(rgbaMatch[3]);
const originalAlpha = rgbaMatch[4] ? parseFloat(rgbaMatch[4]) : 0.6;
fadedColor = `rgba(${r}, ${g}, ${b}, ${originalAlpha * fadeOpacity})`;
}
}
// Add trend line from prediction timestamp/price to predicted price
shapes.push({
type: 'line',
x0: timestamp,
y0: actualCurrentPrice,
x1: endTime,
y1: actualPredictedPrice,
x0: timestampISO, // Start at prediction timestamp
y0: predictionPrice, // Start at prediction price
x1: endTimeISO, // End at predicted time
y1: actualPredictedPrice, // End at predicted price
line: {
color: color,
width: 2 + confidence * 2,
color: fadedColor,
width: (2 + confidence * 2) * fadeOpacity, // Also fade width slightly
dash: 'dashdot'
},
layer: 'above'
@@ -3343,20 +3469,20 @@ class ChartManager {
// Add star marker at target with action label
const actionText = prediction.action === 'BUY' ? '▲' : prediction.action === 'SELL' ? '▼' : '★';
annotations.push({
x: endTime,
x: endTimeISO, // Use ISO string format to match chart timestamps
y: actualPredictedPrice,
text: `${actionText} ${(confidence * 100).toFixed(0)}%`,
showarrow: false,
font: {
size: 12 + confidence * 4,
color: color
size: (12 + confidence * 4) * fadeOpacity, // Fade font size
color: fadedColor
},
bgcolor: 'rgba(31, 41, 55, 0.8)',
bgcolor: `rgba(31, 41, 55, ${0.8 * fadeOpacity})`, // Fade background
borderpad: 3,
opacity: 0.8 + confidence * 0.2
opacity: (0.8 + confidence * 0.2) * fadeOpacity // Apply fade to overall opacity
});
console.log(`[Transformer Prediction] Added prediction marker: ${prediction.action} @ ${actualCurrentPrice.toFixed(2)} -> ${actualPredictedPrice.toFixed(2)} (${(confidence * 100).toFixed(1)}% confidence)`);
console.log(`[Transformer Prediction] Added prediction marker: ${prediction.action} @ ${predictionPrice.toFixed(2)} -> ${actualPredictedPrice.toFixed(2)} (${(confidence * 100).toFixed(1)}% confidence)`);
}
/**

5
ANNOTATE/web/templates/annotation_dashboard.html
View File

@@ -102,6 +102,11 @@
checkActiveTraining();
}
// Check for active inference session (resume PnL and position state after page reload)
if (typeof checkActiveInference === 'function') {
checkActiveInference();
}
// Keyboard shortcuts for chart maximization
document.addEventListener('keydown', function(e) {
// ESC key to exit maximized mode

75
ANNOTATE/web/templates/components/training_panel.html
View File

@@ -234,6 +234,24 @@
activeTrainingId = data.session.training_id;
showTrainingStatus();
// CRITICAL FIX: Immediately restore training progress state
// Don't wait for first poll - restore current state now
if (data.session.current_epoch !== undefined) {
document.getElementById('training-epoch').textContent = data.session.current_epoch || 0;
}
if (data.session.total_epochs !== undefined) {
document.getElementById('training-total-epochs').textContent = data.session.total_epochs || 0;
}
if (data.session.current_loss !== undefined && data.session.current_loss !== null) {
document.getElementById('training-loss').textContent = data.session.current_loss.toFixed(4);
}
// Update progress bar immediately
if (data.session.current_epoch && data.session.total_epochs) {
const percentage = (data.session.current_epoch / data.session.total_epochs) * 100;
document.getElementById('training-progress-bar').style.width = percentage + '%';
}
// Populate annotation count and timeframe if available
if (data.session.annotation_count) {
document.getElementById('training-annotation-count').textContent = data.session.annotation_count;
@@ -242,6 +260,7 @@
document.getElementById('training-timeframe').textContent = data.session.timeframe.toUpperCase();
}
// Start polling for continued updates (will update GPU/CPU and future progress)
pollTrainingProgress(activeTrainingId);
} else {
console.log('No active training session');
@@ -252,6 +271,62 @@
});
}
function checkActiveInference() {
/**
* Check if there's an active real-time inference session on page load
* This allows resuming PnL tracking and position state after page reload
*/
fetch('/api/realtime-inference/signals')
.then(response => response.json())
.then(data => {
if (data.success) {
// Check if inference is active (signals endpoint returns data if active)
if (data.signals && data.signals.length > 0) {
console.log('Active inference session found, restoring state');
// Restore PnL and position state from metrics
if (data.metrics) {
// Update session PnL if available
if (data.metrics.session_pnl !== undefined) {
const sessionPnlEl = document.getElementById('session-pnl');
if (sessionPnlEl) {
const totalPnl = data.metrics.session_pnl || 0;
const pnlColor = totalPnl >= 0 ? 'text-success' : 'text-danger';
const pnlSign = totalPnl >= 0 ? '+' : '';
sessionPnlEl.textContent = `${pnlSign}$${totalPnl.toFixed(2)}`;
sessionPnlEl.className = `fw-bold ${pnlColor}`;
}
}
// Update position state if available
if (data.metrics.position_state) {
updatePositionStateDisplay(data.metrics.position_state, data.metrics.session_metrics || data.metrics);
}
}
// Restore live metrics (accuracy, loss) if available
if (data.metrics) {
if (data.metrics.accuracy !== undefined) {
const liveAccuracyEl = document.getElementById('live-accuracy');
if (liveAccuracyEl) {
liveAccuracyEl.textContent = (data.metrics.accuracy * 100).toFixed(1) + '%';
}
}
if (data.metrics.loss !== undefined) {
const liveLossEl = document.getElementById('live-loss');
if (liveLossEl) {
liveLossEl.textContent = data.metrics.loss.toFixed(4);
}
}
}
}
}
})
.catch(error => {
console.error('Error checking active inference:', error);
});
}
function loadAvailableModels() {
fetch('/api/available-models')
.then(response => response.json())