popov/gogo2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wip training

Browse Source
This commit is contained in:
Dobromir Popov
2025-12-08 22:33:36 +02:00
parent 600bee98f3
commit 5a4b0cf35b
4 changed files with 263 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files

201
ANNOTATE/core/real_training_adapter.py
View File

@@ -570,6 +570,62 @@ class RealTrainingAdapter:
except Exception as e:
logger.debug(f" {timeframe}: Replay failed: {e}")
# CRITICAL FIX: For 1s timeframe, if we still don't have enough data, generate from 1m candles
if timeframe == '1s' and (df is None or df.empty or len(df) < min_required_candles):
try:
df_1m = None
# First, try to get 1m data from already fetched timeframes
if '1m' in fetched_timeframes and '1m' in market_state.get('timeframes', {}):
# Convert dict format to DataFrame
tf_1m = market_state['timeframes']['1m']
try:
df_1m = pd.DataFrame({
'open': tf_1m['open'],
'high': tf_1m['high'],
'low': tf_1m['low'],
'close': tf_1m['close'],
'volume': tf_1m['volume']
}, index=pd.to_datetime(tf_1m['timestamps'], utc=True))
except Exception as e:
logger.debug(f" {timeframe}: Could not convert 1m dict to DataFrame: {e}")
# If we don't have 1m data yet, fetch it
if df_1m is None or df_1m.empty:
logger.debug(f" {timeframe}: Fetching 1m data to generate 1s candles...")
if duckdb_storage:
try:
df_1m = duckdb_storage.get_ohlcv_data(
symbol=symbol,
timeframe='1m',
start_time=tf_start_time,
end_time=tf_end_time,
limit=limit,
direction='before'
)
except Exception as e:
logger.debug(f" {timeframe}: Could not get 1m from DuckDB: {e}")
if df_1m is None or df_1m.empty:
# Try API for 1m
df_1m = self._fetch_historical_from_api(symbol, '1m', tf_start_time, tf_end_time, limit)
# Generate 1s candles from 1m if we have 1m data
if df_1m is not None and not df_1m.empty:
# Generate 1s candles from 1m
generated_1s = self._generate_1s_from_1m(df_1m, min_required_candles)
if generated_1s is not None and len(generated_1s) >= min_required_candles:
df = generated_1s
logger.info(f" {timeframe}: Generated {len(df)} candles from {len(df_1m)} 1m candles")
else:
logger.warning(f" {timeframe}: Generated only {len(generated_1s) if generated_1s is not None else 0} candles from 1m (need {min_required_candles})")
else:
logger.debug(f" {timeframe}: No 1m data available to generate 1s candles")
except Exception as e:
logger.warning(f" {timeframe}: Failed to generate from 1m: {e}")
import traceback
logger.debug(traceback.format_exc())
# Validate data quality before storing
if df is not None and not df.empty:
# Check minimum candle count
@@ -1390,6 +1446,122 @@ class RealTrainingAdapter:
state_size = agent.state_size if hasattr(agent, 'state_size') else 100
return [0.0] * state_size
def _generate_1s_from_1m(self, df_1m: pd.DataFrame, min_candles: int) -> Optional[pd.DataFrame]:
"""
Generate 1s candles from 1m candles by splitting each 1m candle into 60 1s candles.
Args:
df_1m: DataFrame with 1m OHLCV data
min_candles: Minimum number of 1s candles to generate
Returns:
DataFrame with 1s OHLCV data or None if generation fails
"""
import pandas as pd
from datetime import timedelta
try:
if df_1m is None or df_1m.empty:
return None
# Ensure we have required columns
required_cols = ['open', 'high', 'low', 'close', 'volume']
if not all(col in df_1m.columns for col in required_cols):
logger.warning("1m DataFrame missing required columns for 1s generation")
return None
# Generate 1s candles from each 1m candle
# Each 1m candle becomes 60 1s candles
candles_1s = []
for idx, row in df_1m.iterrows():
# Get timestamp (handle both index and column)
if isinstance(df_1m.index, pd.DatetimeIndex):
timestamp = idx
elif 'timestamp' in df_1m.columns:
timestamp = pd.to_datetime(row['timestamp'])
else:
logger.warning("Cannot determine timestamp for 1m candle")
continue
# Extract OHLCV values
open_price = float(row['open'])
high_price = float(row['high'])
low_price = float(row['low'])
close_price = float(row['close'])
volume = float(row['volume'])
# Calculate price change per second (linear interpolation)
price_change = close_price - open_price
price_per_second = price_change / 60.0
# Volume per second (distributed evenly)
volume_per_second = volume / 60.0
# Generate 60 1s candles from this 1m candle
for second in range(60):
# Calculate timestamp for this second
candle_time = timestamp + timedelta(seconds=second)
# Interpolate price (linear from open to close)
if second == 0:
candle_open = open_price
candle_close = open_price + price_per_second
elif second == 59:
candle_open = open_price + (price_per_second * 59)
candle_close = close_price
else:
candle_open = open_price + (price_per_second * second)
candle_close = open_price + (price_per_second * (second + 1))
# High and low: use the interpolated prices, but ensure they stay within the 1m candle's range
# Add small random variation to make it more realistic (but keep within bounds)
candle_high = max(candle_open, candle_close)
candle_low = min(candle_open, candle_close)
# Ensure high/low don't exceed the 1m candle's range
candle_high = min(candle_high, high_price)
candle_low = max(candle_low, low_price)
# If high == low, add small spread
if candle_high == candle_low:
spread = (high_price - low_price) / 60.0
candle_high = candle_high + (spread * 0.5)
candle_low = candle_low - (spread * 0.5)
candles_1s.append({
'timestamp': candle_time,
'open': candle_open,
'high': candle_high,
'low': candle_low,
'close': candle_close,
'volume': volume_per_second
})
if not candles_1s:
return None
# Convert to DataFrame
df_1s = pd.DataFrame(candles_1s)
df_1s['timestamp'] = pd.to_datetime(df_1s['timestamp'])
df_1s = df_1s.set_index('timestamp')
# Sort by timestamp
df_1s = df_1s.sort_index()
# Take the most recent candles up to the limit
if len(df_1s) > min_candles:
df_1s = df_1s.tail(min_candles)
logger.debug(f"Generated {len(df_1s)} 1s candles from {len(df_1m)} 1m candles")
return df_1s
except Exception as e:
logger.warning(f"Error generating 1s candles from 1m: {e}")
import traceback
logger.debug(traceback.format_exc())
return None
def _fetch_historical_from_api(self, symbol: str, timeframe: str, start_time: datetime, end_time: datetime, limit: int) -> Optional[pd.DataFrame]:
"""
Fetch historical OHLCV data from exchange APIs for a specific time range
@@ -3116,11 +3288,32 @@ class RealTrainingAdapter:
logger.info(f"Stopped real-time inference: {inference_id}")
def get_latest_signals(self, limit: int = 50) -> List[Dict]:
"""Get latest inference signals from all active sessions"""
if not hasattr(self, 'inference_sessions'):
return []
"""Get latest inference signals from orchestrator predictions and active sessions"""
all_signals = []
# CRITICAL FIX: Get signals from orchestrator's stored predictions (primary source)
if self.orchestrator and hasattr(self.orchestrator, 'recent_transformer_predictions'):
# Get predictions for all symbols
for symbol, predictions in self.orchestrator.recent_transformer_predictions.items():
if predictions:
# Convert predictions to signal format
for pred in list(predictions)[-limit:]:
signal = {
'timestamp': pred.get('timestamp', datetime.now(timezone.utc).isoformat()),
'action': pred.get('action', 'HOLD'),
'confidence': pred.get('confidence', 0.0),
'current_price': pred.get('current_price', 0.0),
'price': pred.get('current_price', 0.0), # Alias for compatibility
'predicted_price': pred.get('predicted_price', pred.get('current_price', 0.0)),
'price_change': pred.get('price_change', 0.0),
'model': 'transformer',
'predicted_candle': pred.get('predicted_candle', {}),
'source': pred.get('source', 'live_inference')
}
all_signals.append(signal)
# Also get signals from active inference sessions (secondary source)
if hasattr(self, 'inference_sessions'):
for session in self.inference_sessions.values():
all_signals.extend(session.get('signals', []))

152
ANNOTATE/data/annotations/annotations_db.json
View File

@@ -1,161 +1,69 @@
{
"annotations": [
{
"annotation_id": "5d5c4354-12dd-4e0c-92a8-eff631a5dfab",
"symbol": "ETH/USDT",
"timeframe": "1h",
"entry": {
"timestamp": "2025-10-23 20:00",
"price": 3818.72,
"index": 5
},
"exit": {
"timestamp": "2025-10-24 05:00",
"price": 3989.2,
"index": 6
},
"direction": "LONG",
"profit_loss_pct": 4.4643231239787164,
"notes": "",
"created_at": "2025-10-24T23:35:14.215744",
"market_context": {
"entry_state": {},
"exit_state": {}
}
},
{
"annotation_id": "479eb310-c963-4837-b712-70e5a42afb53",
"symbol": "ETH/USDT",
"timeframe": "1h",
"entry": {
"timestamp": "2025-10-27 14:00",
"price": 4124.52,
"index": 329
},
"exit": {
"timestamp": "2025-10-30 20:00",
"price": 3680,
"index": 352
},
"direction": "SHORT",
"profit_loss_pct": 10.777496532929902,
"notes": "",
"created_at": "2025-10-31T00:35:00.543886",
"market_context": {
"entry_state": {},
"exit_state": {}
}
},
{
"annotation_id": "6b529132-8a3e-488d-b354-db8785ddaa71",
"annotation_id": "c6a996e4-b475-4419-b51c-93090f555744",
"symbol": "ETH/USDT",
"timeframe": "1m",
"entry": {
"timestamp": "2025-11-11 12:07",
"price": 3594.33,
"index": 144
"timestamp": "2025-12-08 14:53",
"price": 3165.73,
"index": 237
},
"exit": {
"timestamp": "2025-11-11 20:46",
"price": 3429.24,
"index": 329
"timestamp": "2025-12-08 18:58",
"price": 3075.38,
"index": 317
},
"direction": "SHORT",
"profit_loss_pct": 4.593067414511193,
"profit_loss_pct": 2.8540020785095352,
"notes": "",
"created_at": "2025-11-11T23:23:00.643510",
"created_at": "2025-12-08T20:11:31.371840+00:00",
"market_context": {
"entry_state": {},
"exit_state": {}
}
},
{
"annotation_id": "cdf32bb9-8f0f-467d-abc2-d409a7c22bcc",
"symbol": "ETH/USDT",
"timeframe": "1m",
"entry": {
"timestamp": "2025-11-12 07:58",
"price": 3424.58,
"index": 284
},
"exit": {
"timestamp": "2025-11-12 11:08",
"price": 3546.35,
"index": 329
},
"direction": "LONG",
"profit_loss_pct": 3.5557645025083366,
"notes": "",
"created_at": "2025-11-12T13:11:31.267142",
"market_context": {
"entry_state": {},
"exit_state": {}
}
},
{
"annotation_id": "b01fe6b2-7724-495e-ab01-3f3d3aa0da5d",
"annotation_id": "908534ca-a803-438b-9130-47219bb8d311",
"symbol": "ETH/USDT",
"timeframe": "1s",
"entry": {
"timestamp": "2025-11-22 13:22:23",
"price": 2727.52,
"index": 53
"timestamp": "2025-12-08 20:02:03",
"price": 3123.03,
"index": 52
},
"exit": {
"timestamp": "2025-11-22 13:31:18",
"price": 2717.9,
"index": 104
"timestamp": "2025-12-08 20:09:30",
"price": 3134.03,
"index": 102
},
"direction": "SHORT",
"profit_loss_pct": 0.3527013550771357,
"direction": "LONG",
"profit_loss_pct": 0.3522220407745042,
"notes": "",
"created_at": "2025-11-22T15:31:43.939943",
"created_at": "2025-12-08T20:11:52.510067+00:00",
"market_context": {
"entry_state": {},
"exit_state": {}
}
},
{
"annotation_id": "19a566fa-63bb-4ce3-9f30-116127c9fe95",
"symbol": "ETH/USDT",
"timeframe": "1s",
"entry": {
"timestamp": "2025-11-22 22:25:17",
"price": 2761.97,
"index": 35
},
"exit": {
"timestamp": "2025-11-22 22:30:40",
"price": 2760.15,
"index": 49
},
"direction": "SHORT",
"profit_loss_pct": 0.06589499523889503,
"notes": "",
"created_at": "2025-11-22T22:35:55.606071+00:00",
"market_context": {
"entry_state": {},
"exit_state": {}
}
},
{
"annotation_id": "d8fdf474-d122-4474-b4ad-1f3829b1e46d",
"annotation_id": "7be60d4b-f7cf-4563-aed2-98d869ff361c",
"symbol": "ETH/USDT",
"timeframe": "1m",
"entry": {
"timestamp": "2025-12-08 14:33",
"price": 3178.42,
"index": 309
"timestamp": "2025-12-08 14:53",
"price": 3165.73,
"index": 235
},
"exit": {
"timestamp": "2025-12-08 15:44",
"price": 3088.83,
"index": 331
"timestamp": "2025-12-08 18:58",
"price": 3075.38,
"index": 315
},
"direction": "SHORT",
"profit_loss_pct": 2.8186960817009754,
"profit_loss_pct": 2.8540020785095352,
"notes": "",
"created_at": "2025-12-08T16:34:38.144316+00:00",
"created_at": "2025-12-08T20:27:50.266059+00:00",
"market_context": {
"entry_state": {},
"exit_state": {}
@@ -163,7 +71,7 @@
}
],
"metadata": {
"total_annotations": 7,
"last_updated": "2025-12-08T16:34:38.145818+00:00"
"total_annotations": 3,
"last_updated": "2025-12-08T20:27:50.267314+00:00"
}
}

19
ANNOTATE/web/app.py
View File

@@ -2559,12 +2559,25 @@ class AnnotationDashboard:
signals = self.training_adapter.get_latest_signals()
# Get metrics from active inference sessions
# Get metrics from active inference sessions or orchestrator
metrics = {'accuracy': 0.0, 'loss': 0.0}
# Try to get metrics from orchestrator first (most recent)
if self.orchestrator and hasattr(self.orchestrator, 'primary_transformer_trainer'):
trainer = self.orchestrator.primary_transformer_trainer
if trainer and hasattr(trainer, 'training_history'):
history = trainer.training_history
if history.get('train_accuracy'):
metrics['accuracy'] = history['train_accuracy'][-1] if history['train_accuracy'] else 0.0
if history.get('train_loss'):
metrics['loss'] = history['train_loss'][-1] if history['train_loss'] else 0.0
# Fallback to inference session metrics
if metrics['accuracy'] == 0.0 and metrics['loss'] == 0.0:
if hasattr(self.training_adapter, 'inference_sessions'):
for session in self.training_adapter.inference_sessions.values():
if 'metrics' in session:
metrics = session['metrics']
if 'metrics' in session and session['metrics']:
metrics = session['metrics'].copy()
break
return jsonify({

16
ANNOTATE/web/templates/components/training_panel.html
View File

@@ -1236,7 +1236,21 @@
const accuracyPct = (data.metrics.accuracy * 100).toFixed(1);
const lossVal = data.metrics.loss ? data.metrics.loss.toFixed(4) : '--';
document.getElementById('metric-accuracy').textContent = accuracyPct + '%';
// CRITICAL FIX: Update live-accuracy and live-loss elements
const liveAccuracyEl = document.getElementById('live-accuracy');
const liveLossEl = document.getElementById('live-loss');
if (liveAccuracyEl) {
liveAccuracyEl.textContent = accuracyPct + '%';
}
if (liveLossEl) {
liveLossEl.textContent = lossVal;
}
// Also update metric-accuracy if it exists
const metricAccuracyEl = document.getElementById('metric-accuracy');
if (metricAccuracyEl) {
metricAccuracyEl.textContent = accuracyPct + '%';
}
// Update live banner with metrics
const banner = document.getElementById('inference-status');