ghost T predictions plotting on chart live chart updates
This commit is contained in:
Dobromir Popov
@@ -65,14 +65,26 @@ class HistoricalDataLoader:
|
||||
|
||||
# Check memory cache first (exclude direction from cache key for infinite scroll)
|
||||
cache_key = f"{symbol}_{timeframe}_{start_time}_{end_time}_{limit}"
|
||||
|
||||
# Determine TTL based on timeframe
|
||||
current_ttl = self.cache_ttl
|
||||
if timeframe == '1s':
|
||||
current_ttl = timedelta(seconds=1)
|
||||
elif timeframe == '1m':
|
||||
current_ttl = timedelta(seconds=5)
|
||||
|
||||
if cache_key in self.memory_cache and direction == 'latest':
|
||||
cached_data, cached_time = self.memory_cache[cache_key]
|
||||
if datetime.now() - cached_time < self.cache_ttl:
|
||||
elapsed_ms = (time.time() - start_time_ms) * 1000
|
||||
logger.debug(f"⚡ Memory cache hit for {symbol} {timeframe} ({elapsed_ms:.1f}ms)")
|
||||
if datetime.now() - cached_time < current_ttl:
|
||||
# For 1s/1m, we want to return immediately if valid
|
||||
if timeframe not in ['1s', '1m']:
|
||||
elapsed_ms = (time.time() - start_time_ms) * 1000
|
||||
logger.debug(f"⚡ Memory cache hit for {symbol} {timeframe} ({elapsed_ms:.1f}ms)")
|
||||
return cached_data
|
||||
|
||||
try:
|
||||
# FORCE refresh for 1s/1m if requesting latest data
|
||||
force_refresh = (timeframe in ['1s', '1m'] and not start_time and not end_time)
|
||||
# Try to get data from DataProvider's cached data first (most efficient)
|
||||
if hasattr(self.data_provider, 'cached_data'):
|
||||
with self.data_provider.data_lock:
|
||||
@@ -215,7 +227,7 @@ class HistoricalDataLoader:
|
||||
return None
|
||||
|
||||
# Fallback: Use DataProvider for latest data (startup mode or no time range)
|
||||
if self.startup_mode and not (start_time or end_time):
|
||||
if self.startup_mode and not (start_time or end_time) and not force_refresh:
|
||||
logger.info(f"Loading data for {symbol} {timeframe} (startup mode: allow stale cache)")
|
||||
df = self.data_provider.get_historical_data(
|
||||
symbol=symbol,
|
||||
@@ -225,7 +237,12 @@ class HistoricalDataLoader:
|
||||
)
|
||||
else:
|
||||
# Fetch from API and store in DuckDB (no time range specified)
|
||||
logger.info(f"Fetching latest data from API for {symbol} {timeframe}")
|
||||
# For 1s/1m, logging every request is too verbose, use debug
|
||||
if timeframe in ['1s', '1m']:
|
||||
logger.debug(f"Fetching latest data from API for {symbol} {timeframe}")
|
||||
else:
|
||||
logger.info(f"Fetching latest data from API for {symbol} {timeframe}")
|
||||
|
||||
df = self.data_provider.get_historical_data(
|
||||
symbol=symbol,
|
||||
timeframe=timeframe,
|
||||
|
||||
@@ -16,12 +16,13 @@ import uuid
|
||||
import time
|
||||
import threading
|
||||
import os
|
||||
from typing import Dict, List, Optional, Any
|
||||
from typing import Dict, List, Optional, Any, Tuple
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
import numpy as np
|
||||
|
||||
try:
|
||||
import pytz
|
||||
@@ -2488,7 +2489,7 @@ class RealTrainingAdapter:
|
||||
trainer = getattr(self.orchestrator, 'primary_transformer_trainer', None)
|
||||
if trainer and trainer.model:
|
||||
# Get recent market data
|
||||
market_data = self._get_realtime_market_data(symbol, data_provider)
|
||||
market_data, norm_params = self._get_realtime_market_data(symbol, data_provider)
|
||||
if not market_data:
|
||||
return None
|
||||
|
||||
@@ -2507,41 +2508,162 @@ class RealTrainingAdapter:
|
||||
actions = ['BUY', 'SELL', 'HOLD']
|
||||
action = actions[action_idx] if action_idx < len(actions) else 'HOLD'
|
||||
|
||||
# Handle predicted candles - DENORMALIZE them
|
||||
predicted_candles_raw = {}
|
||||
if 'next_candles' in outputs:
|
||||
for tf, tensor in outputs['next_candles'].items():
|
||||
predicted_candles_raw[tf] = tensor.detach().cpu().numpy().tolist()
|
||||
|
||||
# Denormalize if we have params
|
||||
predicted_candles_denorm = {}
|
||||
if predicted_candles_raw and norm_params:
|
||||
for tf, raw_candle in predicted_candles_raw.items():
|
||||
# raw_candle is [1, 5] list
|
||||
if tf in norm_params:
|
||||
params = norm_params[tf]
|
||||
price_min = params['price_min']
|
||||
price_max = params['price_max']
|
||||
vol_min = params['volume_min']
|
||||
vol_max = params['volume_max']
|
||||
|
||||
# Denormalize [Open, High, Low, Close, Volume]
|
||||
# Note: raw_candle[0] is the list of 5 values
|
||||
candle_values = raw_candle[0]
|
||||
|
||||
denorm_candle = [
|
||||
candle_values[0] * (price_max - price_min) + price_min, # Open
|
||||
candle_values[1] * (price_max - price_min) + price_min, # High
|
||||
candle_values[2] * (price_max - price_min) + price_min, # Low
|
||||
candle_values[3] * (price_max - price_min) + price_min, # Close
|
||||
candle_values[4] * (vol_max - vol_min) + vol_min # Volume
|
||||
]
|
||||
predicted_candles_denorm[tf] = denorm_candle
|
||||
|
||||
# Calculate predicted price from candle close
|
||||
predicted_price = None
|
||||
if '1m' in predicted_candles_denorm:
|
||||
predicted_price = predicted_candles_denorm['1m'][3] # Close price
|
||||
elif '1s' in predicted_candles_denorm:
|
||||
predicted_price = predicted_candles_denorm['1s'][3]
|
||||
elif outputs.get('price_prediction') is not None:
|
||||
# Fallback to price_prediction head if available (normalized)
|
||||
# This would need separate denormalization based on reference price
|
||||
pass
|
||||
|
||||
return {
|
||||
'action': action,
|
||||
'confidence': confidence
|
||||
'confidence': confidence,
|
||||
'predicted_price': predicted_price,
|
||||
'predicted_candle': predicted_candles_denorm
|
||||
}
|
||||
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.debug(f"Error making realtime prediction: {e}")
|
||||
import traceback
|
||||
logger.debug(traceback.format_exc())
|
||||
return None
|
||||
|
||||
def _get_realtime_market_data(self, symbol: str, data_provider) -> Dict:
|
||||
"""Get current market data for prediction"""
|
||||
def _get_realtime_market_data(self, symbol: str, data_provider) -> Tuple[Dict, Dict]:
|
||||
"""
|
||||
Get current market data for prediction AND normalization parameters
|
||||
|
||||
Returns:
|
||||
Tuple of (model_inputs_dict, normalization_params_dict)
|
||||
"""
|
||||
try:
|
||||
# Get recent candles for all timeframes
|
||||
data = {}
|
||||
norm_params = {}
|
||||
|
||||
for tf in ['1s', '1m', '1h', '1d']:
|
||||
df = data_provider.get_historical_data(symbol, tf, limit=200)
|
||||
# Get historical data (raw)
|
||||
# Force refresh for 1s/1m to ensure we have the very latest candle for prediction
|
||||
refresh = tf in ['1s', '1m']
|
||||
df = data_provider.get_historical_data(symbol, tf, limit=600, refresh=refresh)
|
||||
if df is not None and not df.empty:
|
||||
# Convert to tensor format (simplified)
|
||||
import torch
|
||||
import numpy as np
|
||||
# Extract raw arrays
|
||||
opens = df['open'].values.astype(np.float32)
|
||||
highs = df['high'].values.astype(np.float32)
|
||||
lows = df['low'].values.astype(np.float32)
|
||||
closes = df['close'].values.astype(np.float32)
|
||||
volumes = df['volume'].values.astype(np.float32)
|
||||
|
||||
candles = df[['open', 'high', 'low', 'close', 'volume']].values
|
||||
candles_tensor = torch.tensor(candles, dtype=torch.float32).unsqueeze(0)
|
||||
# Need at least 1 candle
|
||||
if len(closes) == 0:
|
||||
continue
|
||||
|
||||
# Prepare OHLCV for normalization logic
|
||||
# Padding if needed (though limit=600 usually suffices)
|
||||
if len(closes) < 600:
|
||||
pad_len = 600 - len(closes)
|
||||
# Pad with first value
|
||||
opens = np.pad(opens, (pad_len, 0), mode='edge')
|
||||
highs = np.pad(highs, (pad_len, 0), mode='edge')
|
||||
lows = np.pad(lows, (pad_len, 0), mode='edge')
|
||||
closes = np.pad(closes, (pad_len, 0), mode='edge')
|
||||
volumes = np.pad(volumes, (pad_len, 0), mode='edge')
|
||||
else:
|
||||
# Take last 600
|
||||
opens = opens[-600:]
|
||||
highs = highs[-600:]
|
||||
lows = lows[-600:]
|
||||
closes = closes[-600:]
|
||||
volumes = volumes[-600:]
|
||||
|
||||
# Stack OHLCV [seq_len, 5]
|
||||
ohlcv = np.stack([opens, highs, lows, closes, volumes], axis=-1)
|
||||
|
||||
# Calculate min/max for normalization
|
||||
price_min = np.min(ohlcv[:, :4])
|
||||
price_max = np.max(ohlcv[:, :4])
|
||||
volume_min = np.min(ohlcv[:, 4])
|
||||
volume_max = np.max(ohlcv[:, 4])
|
||||
|
||||
# Avoid division by zero
|
||||
if price_max == price_min: price_max += 1.0
|
||||
if volume_max == volume_min: volume_max += 1.0
|
||||
|
||||
# Store params for denormalization later
|
||||
norm_params[tf] = {
|
||||
'price_min': float(price_min),
|
||||
'price_max': float(price_max),
|
||||
'volume_min': float(volume_min),
|
||||
'volume_max': float(volume_max)
|
||||
}
|
||||
|
||||
# Normalize in-place
|
||||
ohlcv[:, :4] = (ohlcv[:, :4] - price_min) / (price_max - price_min)
|
||||
ohlcv[:, 4] = (ohlcv[:, 4] - volume_min) / (volume_max - volume_min)
|
||||
|
||||
# Convert to tensor [1, seq_len, 5]
|
||||
import torch
|
||||
candles_tensor = torch.tensor(ohlcv, dtype=torch.float32).unsqueeze(0)
|
||||
data[f'price_data_{tf}'] = candles_tensor
|
||||
|
||||
# Add placeholder data for other inputs
|
||||
import torch
|
||||
data['tech_data'] = torch.zeros(1, 40, dtype=torch.float32)
|
||||
data['market_data'] = torch.zeros(1, 30, dtype=torch.float32)
|
||||
# Add placeholder data for other inputs if we have at least one timeframe
|
||||
if data:
|
||||
import torch
|
||||
# Correct shapes based on model expectation
|
||||
# tech_data: [1, 40]
|
||||
# market_data: [1, 30]
|
||||
# cob_data: [1, 600, 100]
|
||||
data['tech_data'] = torch.zeros(1, 40, dtype=torch.float32)
|
||||
data['market_data'] = torch.zeros(1, 30, dtype=torch.float32)
|
||||
data['cob_data'] = torch.zeros(1, 600, 100, dtype=torch.float32)
|
||||
|
||||
# Move to device if available
|
||||
if hasattr(self.orchestrator, 'device'):
|
||||
device = self.orchestrator.device
|
||||
for k, v in data.items():
|
||||
data[k] = v.to(device)
|
||||
|
||||
return data if data else None
|
||||
return data, norm_params if data else (None, None)
|
||||
except Exception as e:
|
||||
logger.debug(f"Error getting realtime market data: {e}")
|
||||
return None
|
||||
import traceback
|
||||
logger.debug(traceback.format_exc())
|
||||
return None, None
|
||||
|
||||
def _train_on_new_candle(self, session: Dict, symbol: str, timeframe: str, data_provider):
|
||||
"""Train model on new candle when it closes"""
|
||||
@@ -2734,7 +2856,9 @@ class RealTrainingAdapter:
|
||||
'model': model_name,
|
||||
'action': prediction['action'],
|
||||
'confidence': prediction['confidence'],
|
||||
'price': current_price
|
||||
'price': current_price,
|
||||
'predicted_price': prediction.get('predicted_price'),
|
||||
'predicted_candle': prediction.get('predicted_candle')
|
||||
}
|
||||
|
||||
session['signals'].append(signal)
|
||||
|
||||
Reference in New Issue
Block a user