diff --git a/ANNOTATE/core/real_training_adapter.py b/ANNOTATE/core/real_training_adapter.py index 015464c..edaf649 100644 --- a/ANNOTATE/core/real_training_adapter.py +++ b/ANNOTATE/core/real_training_adapter.py @@ -2579,8 +2579,9 @@ class RealTrainingAdapter: for tf in ['1s', '1m', '1h', '1d']: # Get historical data (raw) # Force refresh for 1s/1m to ensure we have the very latest candle for prediction + # But set persist=False to avoid locking the database with high-frequency writes refresh = tf in ['1s', '1m'] - df = data_provider.get_historical_data(symbol, tf, limit=600, refresh=refresh) + df = data_provider.get_historical_data(symbol, tf, limit=600, refresh=refresh, persist=False) if df is not None and not df.empty: # Extract raw arrays opens = df['open'].values.astype(np.float32) diff --git a/ANNOTATE/web/static/js/chart_manager.js b/ANNOTATE/web/static/js/chart_manager.js index 68f1270..fdc4ad9 100644 --- a/ANNOTATE/web/static/js/chart_manager.js +++ b/ANNOTATE/web/static/js/chart_manager.js @@ -1902,10 +1902,11 @@ class ChartManager { // Add prediction traces (ghost candles) if (predictionTraces.length > 0) { - // Remove existing ghost traces first (heuristic: traces with name 'Ghost Prediction') + // Remove existing ghost traces safely + // We iterate backwards to avoid index shifting issues when deleting const currentTraces = plotElement.data.length; const indicesToRemove = []; - for (let i = 0; i < currentTraces; i++) { + for (let i = currentTraces - 1; i >= 0; i--) { if (plotElement.data[i].name === 'Ghost Prediction') { indicesToRemove.push(i); } diff --git a/ANNOTATE/web/templates/components/training_panel.html b/ANNOTATE/web/templates/components/training_panel.html index 25aac1b..dddcfa6 100644 --- a/ANNOTATE/web/templates/components/training_panel.html +++ b/ANNOTATE/web/templates/components/training_panel.html @@ -849,8 +849,8 @@ return; } - // Display last 5 predictions (most recent first) - const html = predictionHistory.slice(0, 5).map(pred => { + // Display last 15 predictions (most recent first) + const html = predictionHistory.slice(0, 15).map(pred => { // Safely parse timestamp let timeStr = '--:--:--'; try { @@ -868,12 +868,14 @@ pred.action === 'SELL' ? 'text-danger' : 'text-secondary'; const confidence = (pred.confidence * 100).toFixed(1); const price = (pred.predicted_price && !isNaN(pred.predicted_price)) ? pred.predicted_price.toFixed(2) : '--'; + const timeframe = pred.timeframe || '1m'; return `
+ ${timeframe} ${pred.action} - ${timeStr} + ${timeStr}
${confidence}%
@@ -930,17 +932,23 @@ timestamp = latest.timestamp; } - // Add to prediction history (keep last 5) - predictionHistory.unshift({ + // Add to prediction history (keep last 15) + const newPrediction = { timestamp: timestamp, action: latest.action, confidence: latest.confidence, - predicted_price: latest.predicted_price - }); - if (predictionHistory.length > 5) { - predictionHistory = predictionHistory.slice(0, 5); + predicted_price: latest.predicted_price, + timeframe: appState.currentTimeframes ? appState.currentTimeframes[0] : '1m' + }; + + // Filter out undefined/invalid predictions before adding + if (latest.action && !isNaN(latest.confidence)) { + predictionHistory.unshift(newPrediction); + if (predictionHistory.length > 15) { + predictionHistory = predictionHistory.slice(0, 15); + } + updatePredictionHistory(); } - updatePredictionHistory(); // Update chart with signal markers and predictions if (window.appState && window.appState.chartManager) { diff --git a/core/data_provider.py b/core/data_provider.py index 1c26a45..6dc1b08 100644 --- a/core/data_provider.py +++ b/core/data_provider.py @@ -1623,7 +1623,7 @@ class DataProvider: logger.error(f"Error getting market state at time: {e}") return {} - def get_historical_data(self, symbol: str, timeframe: str, limit: int = 1000, refresh: bool = False, allow_stale_cache: bool = False) -> Optional[pd.DataFrame]: + def get_historical_data(self, symbol: str, timeframe: str, limit: int = 1000, refresh: bool = False, allow_stale_cache: bool = False, persist: bool = True) -> Optional[pd.DataFrame]: """Get historical OHLCV data. - Prefer cached data for low latency. - If cache is empty or refresh=True, fetch real data from exchanges. @@ -1635,6 +1635,7 @@ class DataProvider: limit: Number of candles to return refresh: Force refresh from exchange allow_stale_cache: Allow loading stale cache (for startup performance) + persist: Whether to save fetched data to DuckDB (default: True). Set False for high-frequency polling. """ try: # Serve from cache when available @@ -1644,7 +1645,7 @@ class DataProvider: return cached_df.tail(limit) # Try loading from DuckDB first (fast Parquet queries) - if allow_stale_cache: + if allow_stale_cache and not refresh: cached_df = self._load_from_duckdb(symbol, timeframe, limit=1500) if cached_df is not None and not cached_df.empty: logger.info(f"Loaded {len(cached_df)} candles from DuckDB for {symbol} {timeframe} (startup mode)") @@ -1662,8 +1663,8 @@ class DataProvider: if df is not None and not df.empty: df = self._ensure_datetime_index(df) - # Store in DuckDB (Parquet + SQL in one) - if self.duckdb_storage: + # Store in DuckDB (Parquet + SQL in one) - Only if persist is True + if self.duckdb_storage and persist: try: self.duckdb_storage.store_ohlcv_data(symbol, timeframe, df) except Exception as e: @@ -1680,7 +1681,12 @@ class DataProvider: combined_df = combined_df.sort_index() self.cached_data[symbol][timeframe] = combined_df.tail(1500) - logger.info(f"Stored {len(df)} candles for {symbol} {timeframe} (DuckDB + memory cache)") + if persist: + logger.info(f"Stored {len(df)} candles for {symbol} {timeframe} (DuckDB + memory cache)") + else: + # Less verbose for high-frequency polling + logger.debug(f"Updated memory cache with {len(df)} candles for {symbol} {timeframe}") + return self.cached_data[symbol][timeframe].tail(limit) logger.warning(f"No real data available for {symbol} {timeframe} at request time")