From 4765b1b1e11e4a064a1362c9eefd297b91cf9892 Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Wed, 23 Jul 2025 22:49:54 +0300 Subject: [PATCH] cob data providers tests --- core/enhanced_cnn_adapter.py | 75 +++++- test_cob_audit.py | 87 ------- .../cob/test_cob_comparison.py | 0 .../cob/test_cob_data_stability.py | 245 ++++++------------ web/clean_dashboard.py | 185 ++++++++++--- 5 files changed, 298 insertions(+), 294 deletions(-) delete mode 100644 test_cob_audit.py rename test_cob_comparison.py => tests/cob/test_cob_comparison.py (100%) rename test_cob_data_stability.py => tests/cob/test_cob_data_stability.py (52%) diff --git a/core/enhanced_cnn_adapter.py b/core/enhanced_cnn_adapter.py index 8a66b7a..bffbc7e 100644 --- a/core/enhanced_cnn_adapter.py +++ b/core/enhanced_cnn_adapter.py @@ -48,6 +48,16 @@ class EnhancedCNNAdapter: self.learning_rate = 0.0001 self.model_name = "enhanced_cnn_v1" + # Enhanced metrics tracking + self.last_inference_time = None + self.last_inference_duration = 0.0 + self.last_prediction_output = None + self.last_training_time = None + self.last_training_duration = 0.0 + self.last_training_loss = 0.0 + self.inference_count = 0 + self.training_count = 0 + # Create checkpoint directory if it doesn't exist os.makedirs(checkpoint_dir, exist_ok=True) @@ -181,6 +191,10 @@ class EnhancedCNNAdapter: ModelOutput: Standardized model output """ try: + # Track inference timing + start_time = datetime.now() + inference_start = start_time.timestamp() + # Convert BaseDataInput to features features = self._convert_base_data_to_features(base_data) @@ -204,6 +218,18 @@ class EnhancedCNNAdapter: actions = ['BUY', 'SELL', 'HOLD'] action = actions[action_idx] + # Extract pivot price prediction (simplified - take first value from price_pred) + pivot_price = None + if price_pred is not None and len(price_pred.squeeze()) > 0: + # Get current price from base_data for context + current_price = 0.0 + if base_data.ohlcv_1s and len(base_data.ohlcv_1s) > 0: + current_price = base_data.ohlcv_1s[-1].close + + # Calculate pivot price as current price + predicted change + price_change_pct = float(price_pred.squeeze()[0].item()) # First prediction value + pivot_price = current_price * (1 + price_change_pct * 0.01) # Convert percentage to price + # Create predictions dictionary predictions = { 'action': action, @@ -211,7 +237,8 @@ class EnhancedCNNAdapter: 'sell_probability': float(action_probs[0, 1].item()), 'hold_probability': float(action_probs[0, 2].item()), 'extrema': extrema_pred.squeeze(0).cpu().numpy().tolist(), - 'price_prediction': price_pred.squeeze(0).cpu().numpy().tolist() + 'price_prediction': price_pred.squeeze(0).cpu().numpy().tolist(), + 'pivot_price': pivot_price } # Create hidden states dictionary @@ -219,11 +246,31 @@ class EnhancedCNNAdapter: 'features': features_refined.squeeze(0).cpu().numpy().tolist() } + # Calculate inference duration + end_time = datetime.now() + inference_duration = (end_time.timestamp() - inference_start) * 1000 # Convert to milliseconds + + # Update metrics + self.last_inference_time = start_time + self.last_inference_duration = inference_duration + self.inference_count += 1 + + # Store last prediction output for dashboard + self.last_prediction_output = { + 'action': action, + 'confidence': confidence, + 'pivot_price': pivot_price, + 'timestamp': start_time, + 'symbol': base_data.symbol + } + # Create metadata dictionary metadata = { 'model_version': '1.0', - 'timestamp': datetime.now().isoformat(), - 'input_shape': features.shape + 'timestamp': start_time.isoformat(), + 'input_shape': features.shape, + 'inference_duration_ms': inference_duration, + 'inference_count': self.inference_count } # Create ModelOutput @@ -231,7 +278,7 @@ class EnhancedCNNAdapter: model_type='cnn', model_name=self.model_name, symbol=base_data.symbol, - timestamp=datetime.now(), + timestamp=start_time, confidence=confidence, predictions=predictions, hidden_states=hidden_states, @@ -294,6 +341,10 @@ class EnhancedCNNAdapter: Dict[str, float]: Training metrics """ try: + # Track training timing + training_start_time = datetime.now() + training_start = training_start_time.timestamp() + with self.training_lock: # Check if we have enough data if len(self.training_data) < self.batch_size: @@ -378,15 +429,27 @@ class EnhancedCNNAdapter: avg_loss = total_loss / (len(self.training_data) / self.batch_size) accuracy = correct_predictions / total_predictions if total_predictions > 0 else 0.0 + # Calculate training duration + training_end_time = datetime.now() + training_duration = (training_end_time.timestamp() - training_start) * 1000 # Convert to milliseconds + + # Update training metrics + self.last_training_time = training_start_time + self.last_training_duration = training_duration + self.last_training_loss = avg_loss + self.training_count += 1 + # Save checkpoint self._save_checkpoint(avg_loss, accuracy) - logger.info(f"Training completed: loss={avg_loss:.4f}, accuracy={accuracy:.4f}, samples={len(self.training_data)}") + logger.info(f"Training completed: loss={avg_loss:.4f}, accuracy={accuracy:.4f}, samples={len(self.training_data)}, duration={training_duration:.1f}ms") return { 'loss': avg_loss, 'accuracy': accuracy, - 'samples': len(self.training_data) + 'samples': len(self.training_data), + 'duration_ms': training_duration, + 'training_count': self.training_count } except Exception as e: diff --git a/test_cob_audit.py b/test_cob_audit.py deleted file mode 100644 index 7afacce..0000000 --- a/test_cob_audit.py +++ /dev/null @@ -1,87 +0,0 @@ -#!/usr/bin/env python3 -""" -Test COB Integration Status in Enhanced Orchestrator -""" - -import asyncio -import sys -from pathlib import Path -sys.path.append(str(Path('.').absolute())) - -from core.enhanced_orchestrator import EnhancedTradingOrchestrator -from core.data_provider import DataProvider - -async def test_cob_integration(): - print("=" * 60) - print("COB INTEGRATION AUDIT") - print("=" * 60) - - try: - data_provider = DataProvider() - orchestrator = EnhancedTradingOrchestrator( - data_provider=data_provider, - symbols=['ETH/USDT', 'BTC/USDT'], - enhanced_rl_training=True - ) - - print(f"āœ“ Enhanced Orchestrator created") - print(f"Has COB integration attribute: {hasattr(orchestrator, 'cob_integration')}") - print(f"COB integration value: {orchestrator.cob_integration}") - print(f"COB integration type: {type(orchestrator.cob_integration)}") - print(f"COB integration active: {getattr(orchestrator, 'cob_integration_active', 'Not set')}") - - if orchestrator.cob_integration: - print("\n--- COB Integration Details ---") - print(f"COB Integration class: {orchestrator.cob_integration.__class__.__name__}") - - # Check if it has the expected methods - methods_to_check = ['get_statistics', 'get_cob_snapshot', 'add_dashboard_callback', 'start', 'stop'] - for method in methods_to_check: - has_method = hasattr(orchestrator.cob_integration, method) - print(f"Has {method}: {has_method}") - - # Try to get statistics - if hasattr(orchestrator.cob_integration, 'get_statistics'): - try: - stats = orchestrator.cob_integration.get_statistics() - print(f"COB statistics: {stats}") - except Exception as e: - print(f"Error getting COB statistics: {e}") - - # Try to get a snapshot - if hasattr(orchestrator.cob_integration, 'get_cob_snapshot'): - try: - snapshot = orchestrator.cob_integration.get_cob_snapshot('ETH/USDT') - print(f"ETH/USDT snapshot: {snapshot}") - except Exception as e: - print(f"Error getting COB snapshot: {e}") - - # Check if COB integration needs to be started - print(f"\n--- Starting COB Integration ---") - try: - await orchestrator.start_cob_integration() - print("āœ“ COB integration started successfully") - - # Wait a moment and check statistics again - await asyncio.sleep(3) - if hasattr(orchestrator.cob_integration, 'get_statistics'): - stats = orchestrator.cob_integration.get_statistics() - print(f"COB statistics after start: {stats}") - - except Exception as e: - print(f"Error starting COB integration: {e}") - else: - print("\nāŒ COB integration is None - this explains the dashboard issues") - print("The Enhanced Orchestrator failed to initialize COB integration") - - # Check the error flag - if hasattr(orchestrator, '_cob_integration_failed'): - print(f"COB integration failed flag: {orchestrator._cob_integration_failed}") - - except Exception as e: - print(f"Error in COB audit: {e}") - import traceback - traceback.print_exc() - -if __name__ == "__main__": - asyncio.run(test_cob_integration()) \ No newline at end of file diff --git a/test_cob_comparison.py b/tests/cob/test_cob_comparison.py similarity index 100% rename from test_cob_comparison.py rename to tests/cob/test_cob_comparison.py diff --git a/test_cob_data_stability.py b/tests/cob/test_cob_data_stability.py similarity index 52% rename from test_cob_data_stability.py rename to tests/cob/test_cob_data_stability.py index 3f7675e..6a58ac1 100644 --- a/test_cob_data_stability.py +++ b/tests/cob/test_cob_data_stability.py @@ -150,168 +150,63 @@ class COBStabilityTester: else: logger.warning("No data was collected. Cannot generate plot.") - def plot_spectrogram(self): - """Create a bookmap-style visualization showing order book depth over time.""" - if not self.ticks: - logger.warning("No ticks to plot.") + def create_price_heatmap_chart(self): + """Create a visualization with price chart and order book heatmap.""" + if not self.price_data or not self.cob_snapshots: + logger.warning("Insufficient data to plot.") return - logger.info(f"Creating bookmap-style visualization with {len(self.ticks)} data points...") - - # Extract order book data from ticks - time_points = [] - bid_data = [] - ask_data = [] - price_levels = set() - - for tick in self.ticks: - if hasattr(tick, 'raw_data') and tick.raw_data: - cob_data = tick.raw_data - if 'bids' in cob_data and 'asks' in cob_data: - timestamp = tick.timestamp - - # Extract bid levels (green - buy orders) - bids = cob_data['bids'][:20] # Top 20 levels - for bid in bids: - if isinstance(bid, dict) and 'price' in bid and 'size' in bid: - bid_data.append({ - 'time': timestamp, - 'price': bid['price'], - 'size': bid['size'], - 'side': 'bid' - }) - price_levels.add(bid['price']) - - # Extract ask levels (red - sell orders) - asks = cob_data['asks'][:20] # Top 20 levels - for ask in asks: - if isinstance(ask, dict) and 'price' in ask and 'size' in ask: - ask_data.append({ - 'time': timestamp, - 'price': ask['price'], - 'size': ask['size'], - 'side': 'ask' - }) - price_levels.add(ask['price']) - - if not bid_data and not ask_data: - logger.warning("No order book data found in ticks. Cannot create bookmap visualization.") - # Fallback to simple price chart - self._create_simple_price_chart() - return - - logger.info(f"Extracted {len(bid_data)} bid levels and {len(ask_data)} ask levels") - - # Create the bookmap visualization - fig, ax = plt.subplots(figsize=(16, 10)) - - # Combine all data - all_data = bid_data + ask_data - if not all_data: - logger.warning("No order book data to plot") - return - - # Create DataFrames - df = pd.DataFrame(all_data) - df['time'] = pd.to_datetime(df['time']) - - # Create price bins (like in bookmap) - price_min = df['price'].min() - price_max = df['price'].max() - price_range = price_max - price_min - if price_range == 0: - logger.warning("No price variation in data") - return - - # Create time bins - time_min = df['time'].min() - time_max = df['time'].max() - - # Create 2D heatmaps for bids and asks separately - time_bins = pd.date_range(time_min, time_max, periods=100) - price_bins = np.linspace(price_min, price_max, 200) # Higher resolution for price - - # Separate bid and ask data - bid_df = df[df['side'] == 'bid'] - ask_df = df[df['side'] == 'ask'] - - # Create bid heatmap (green) - if not bid_df.empty: - bid_hist, _, _ = np.histogram2d( - bid_df['time'].astype(np.int64) // 10**9, - bid_df['price'], - bins=[time_bins.astype(np.int64) // 10**9, price_bins], - weights=bid_df['size'] - ) - # Plot bids in green (buying pressure) - bid_mask = bid_hist > 0 - pcm_bid = ax.pcolormesh( - time_bins, price_bins, bid_hist.T, - cmap='Greens', alpha=0.7, vmin=0, vmax=bid_hist.max() - ) - - # Create ask heatmap (red) - if not ask_df.empty: - ask_hist, _, _ = np.histogram2d( - ask_df['time'].astype(np.int64) // 10**9, - ask_df['price'], - bins=[time_bins.astype(np.int64) // 10**9, price_bins], - weights=ask_df['size'] - ) - # Plot asks in red (selling pressure) - ask_mask = ask_hist > 0 - pcm_ask = ax.pcolormesh( - time_bins, price_bins, ask_hist.T, - cmap='Reds', alpha=0.7, vmin=0, vmax=ask_hist.max() - ) - - # Add mid price line - mid_prices = [] - mid_times = [] - for tick in self.ticks: - if hasattr(tick, 'raw_data') and tick.raw_data and 'stats' in tick.raw_data: - stats = tick.raw_data['stats'] - if 'mid_price' in stats and stats['mid_price'] > 0: - mid_prices.append(stats['mid_price']) - mid_times.append(tick.timestamp) - - if mid_prices: - ax.plot(pd.to_datetime(mid_times), mid_prices, 'yellow', linewidth=2, alpha=0.8, label='Mid Price') - - # Styling like bookmap - ax.set_facecolor('black') - fig.patch.set_facecolor('black') - - ax.set_title(f'Order Book Depth Map - {self.symbol}\n(Green=Bids/Buy Orders, Red=Asks/Sell Orders)', - color='white', fontsize=14) - ax.set_xlabel('Time', color='white') - ax.set_ylabel('Price (USDT)', color='white') - - # White ticks and labels - ax.tick_params(colors='white') - ax.spines['bottom'].set_color('white') - ax.spines['top'].set_color('white') - ax.spines['right'].set_color('white') - ax.spines['left'].set_color('white') - - # Add colorbar for bid data - if not bid_df.empty: - cbar_bid = fig.colorbar(pcm_bid, ax=ax, location='right', pad=0.02, shrink=0.5) - cbar_bid.set_label('Bid Size (Order Volume)', color='white', labelpad=15) - cbar_bid.ax.yaxis.set_tick_params(color='white') - cbar_bid.ax.yaxis.set_tick_params(labelcolor='white') - - # Format the x-axis to show time properly - fig.autofmt_xdate() - - if mid_prices: - ax.legend(loc='upper left') - + logger.info(f"Creating price and order book heatmap chart...") + + # Prepare data + price_df = pd.DataFrame(self.price_data) + price_df['timestamp'] = pd.to_datetime(price_df['timestamp']) + + # Extract order book data for heatmap + heatmap_data = [] + for snapshot in self.cob_snapshots: + timestamp = snapshot['timestamp'] + for side in ['bids', 'asks']: + for order in snapshot[side]: + bucketed_price = round(order['price'] / self.price_granularity) * self.price_granularity + heatmap_data.append({ + 'time': timestamp, + 'price': bucketed_price, + 'size': order['size'], + 'side': side + }) + + heatmap_df = pd.DataFrame(heatmap_data) + + # Create plot + fig, ax1 = plt.subplots(figsize=(16, 8)) + + # Plot price line + ax1.plot(price_df['timestamp'], price_df['price'], 'cyan', linewidth=1, label='Price') + + # Prepare heatmap + for side, cmap in zip(['bids', 'asks'], ['Greens', 'Reds']): + side_df = heatmap_df[heatmap_df['side'] == side] + if not side_df.empty: + hist, xedges, yedges = np.histogram2d( + side_df['time'].astype(np.int64) // 10**9, + side_df['price'], + bins=[np.unique(side_df['time'].astype(np.int64) // 10**9), np.arange(price_df['price'].min(), price_df['price'].max(), self.price_granularity)], + weights=side_df['size'] + ) + ax1.pcolormesh(pd.to_datetime(xedges, unit='s'), yedges, hist.T, cmap=cmap, alpha=0.5) + + # Enhance plot + ax1.set_title(f'Price Chart with Order Book Heatmap - {self.symbol}') + ax1.set_xlabel('Time') + ax1.set_ylabel('Price (USDT)') + ax1.legend(loc='upper left') + ax1.grid(True, alpha=0.3) + plt.tight_layout() - - plot_filename = f"cob_bookmap_{self.symbol.replace('/', '_')}_{datetime.now():%Y%m%d_%H%M%S}.png" - plt.savefig(plot_filename, facecolor='black', dpi=150) - logger.info(f"Bookmap-style plot saved to {plot_filename}") + plot_filename = f"price_heatmap_chart_{self.symbol.replace('/', '_')}_{datetime.now():%Y%m%d_%H%M%S}.png" + plt.savefig(plot_filename, dpi=150) + logger.info(f"Price and heatmap chart saved to {plot_filename}") plt.show() def _create_simple_price_chart(self): @@ -343,13 +238,37 @@ class COBStabilityTester: plt.show() -async def main(): - tester = COBStabilityTester() +async def main(symbol='ETHUSDT', duration_seconds=15): + """Main function to run the COB test with configurable parameters. + + Args: + symbol: Trading symbol (default: ETHUSDT) + duration_seconds: Test duration in seconds (default: 15) + """ + logger.info(f"Starting COB test with symbol={symbol}, duration={duration_seconds}s") + tester = COBStabilityTester(symbol=symbol, duration_seconds=duration_seconds) await tester.run_test() if __name__ == "__main__": + import sys + + # Parse command line arguments + symbol = 'ETHUSDT' # Default + duration = 15 # Default + + if len(sys.argv) > 1: + symbol = sys.argv[1] + if len(sys.argv) > 2: + try: + duration = int(sys.argv[2]) + except ValueError: + logger.warning(f"Invalid duration '{sys.argv[2]}', using default 15 seconds") + + logger.info(f"Configuration: Symbol={symbol}, Duration={duration}s") + logger.info(f"Granularity: {'1 USD for ETH' if 'ETH' in symbol.upper() else '10 USD for BTC' if 'BTC' in symbol.upper() else '1 USD default'}") + try: - asyncio.run(main()) + asyncio.run(main(symbol, duration)) except KeyboardInterrupt: - logger.info("Test interrupted by user.") \ No newline at end of file + logger.info("Test interrupted by user.") diff --git a/web/clean_dashboard.py b/web/clean_dashboard.py index 2dc74fd..23fe89e 100644 --- a/web/clean_dashboard.py +++ b/web/clean_dashboard.py @@ -2734,7 +2734,14 @@ class CleanTradingDashboard: 'confidence': 0.0, 'last_prediction': 'N/A', 'training_samples': 0, - 'inference_rate': '0.00/s' + 'inference_rate': '0.00/s', + 'last_inference_time': 'Never', + 'last_inference_duration': 0.0, + 'pivot_price': None, + 'suggested_action': 'HOLD', + 'last_training_time': 'Never', + 'last_training_duration': 0.0, + 'last_training_loss': 0.0 } # Get CNN prediction for ETH/USDT @@ -2743,24 +2750,59 @@ class CleanTradingDashboard: # Get model performance metrics model_info = self.cnn_adapter.get_model_info() if hasattr(self.cnn_adapter, 'get_model_info') else {} + # Get inference timing metrics + last_inference_time = getattr(self.cnn_adapter, 'last_inference_time', None) + last_inference_duration = getattr(self.cnn_adapter, 'last_inference_duration', 0.0) + inference_count = getattr(self.cnn_adapter, 'inference_count', 0) + + # Format inference time + if last_inference_time: + inference_time_str = last_inference_time.strftime('%H:%M:%S') + else: + inference_time_str = 'Never' + # Calculate inference rate - inference_times = getattr(self.cnn_adapter, 'inference_times', []) - if len(inference_times) > 0: - avg_inference_time = sum(inference_times[-10:]) / min(len(inference_times), 10) - inference_rate = f"{1.0/avg_inference_time:.2f}/s" if avg_inference_time > 0 else "0.00/s" + if inference_count > 0 and last_inference_duration > 0: + inference_rate = f"{1000.0/last_inference_duration:.2f}/s" # Convert ms to rate else: inference_rate = "0.00/s" + # Get training timing metrics + last_training_time = getattr(self.cnn_adapter, 'last_training_time', None) + last_training_duration = getattr(self.cnn_adapter, 'last_training_duration', 0.0) + last_training_loss = getattr(self.cnn_adapter, 'last_training_loss', 0.0) + training_count = getattr(self.cnn_adapter, 'training_count', 0) + + # Format training time + if last_training_time: + training_time_str = last_training_time.strftime('%H:%M:%S') + else: + training_time_str = 'Never' + # Get training data count training_samples = len(getattr(self.cnn_adapter, 'training_data', [])) - # Format last prediction - if prediction: - last_prediction = f"{prediction['action']} ({prediction['confidence']:.1%})" - current_confidence = prediction['confidence'] + # Get last prediction output details + last_prediction_output = getattr(self.cnn_adapter, 'last_prediction_output', None) + + # Format prediction details + if last_prediction_output: + suggested_action = last_prediction_output.get('action', 'HOLD') + current_confidence = last_prediction_output.get('confidence', 0.0) + pivot_price = last_prediction_output.get('pivot_price', None) + + # Format pivot price + if pivot_price and pivot_price > 0: + pivot_price_str = f"${pivot_price:.2f}" + else: + pivot_price_str = "N/A" + + last_prediction = f"{suggested_action} ({current_confidence:.1%})" else: - last_prediction = "No prediction" + suggested_action = 'HOLD' current_confidence = 0.0 + pivot_price_str = "N/A" + last_prediction = "No prediction" # Get model status if hasattr(self.cnn_adapter, 'model') and self.cnn_adapter.model: @@ -2775,14 +2817,25 @@ class CleanTradingDashboard: return { 'status': status, - 'parameters': model_info.get('parameters', '50.0M'), - 'current_loss': model_info.get('current_loss', 0.0), + 'parameters': '50.0M', # Enhanced CNN parameters + 'current_loss': last_training_loss, 'accuracy': model_info.get('accuracy', 0.0), 'confidence': current_confidence, 'last_prediction': last_prediction, 'training_samples': training_samples, 'inference_rate': inference_rate, - 'last_update': datetime.now().strftime('%H:%M:%S') + 'last_update': datetime.now().strftime('%H:%M:%S'), + + # Enhanced metrics + 'last_inference_time': inference_time_str, + 'last_inference_duration': f"{last_inference_duration:.1f}ms", + 'inference_count': inference_count, + 'pivot_price': pivot_price_str, + 'suggested_action': suggested_action, + 'last_training_time': training_time_str, + 'last_training_duration': f"{last_training_duration:.1f}ms", + 'last_training_loss': f"{last_training_loss:.6f}", + 'training_count': training_count } except Exception as e: @@ -2795,7 +2848,14 @@ class CleanTradingDashboard: 'confidence': 0.0, 'last_prediction': f'Error: {str(e)}', 'training_samples': 0, - 'inference_rate': '0.00/s' + 'inference_rate': '0.00/s', + 'last_inference_time': 'Error', + 'last_inference_duration': '0.0ms', + 'pivot_price': 'N/A', + 'suggested_action': 'HOLD', + 'last_training_time': 'Error', + 'last_training_duration': '0.0ms', + 'last_training_loss': '0.000000' } def _get_training_metrics(self) -> Dict: @@ -2996,29 +3056,27 @@ class CleanTradingDashboard: } loaded_models['dqn'] = dqn_model_info - # 2. CNN Model Status - using orchestrator SSOT + # 2. CNN Model Status - using enhanced CNN adapter data cnn_state = model_states.get('cnn', {}) cnn_timing = get_model_timing_info('CNN') - cnn_active = True - # Get latest CNN prediction - cnn_latest = latest_predictions.get('cnn', {}) - if cnn_latest: - cnn_action = cnn_latest.get('action', 'PATTERN_ANALYSIS') - cnn_confidence = cnn_latest.get('confidence', 0.68) - timestamp_val = cnn_latest.get('timestamp', datetime.now()) - if isinstance(timestamp_val, str): - cnn_timestamp = timestamp_val - elif hasattr(timestamp_val, 'strftime'): - cnn_timestamp = timestamp_val.strftime('%H:%M:%S') - else: - cnn_timestamp = datetime.now().strftime('%H:%M:%S') - cnn_predicted_price = cnn_latest.get('predicted_price', 0) - else: - cnn_action = 'PATTERN_ANALYSIS' - cnn_confidence = 0.68 - cnn_timestamp = datetime.now().strftime('%H:%M:%S') - cnn_predicted_price = 0 + # Get enhanced CNN panel data with detailed metrics + cnn_panel_data = self._update_cnn_model_panel() + cnn_active = cnn_panel_data.get('status') not in ['NOT_AVAILABLE', 'ERROR', 'NOT_LOADED'] + + # Use enhanced CNN data for display + cnn_action = cnn_panel_data.get('suggested_action', 'PATTERN_ANALYSIS') + cnn_confidence = cnn_panel_data.get('confidence', 0.0) + cnn_timestamp = cnn_panel_data.get('last_inference_time', 'Never') + cnn_pivot_price = cnn_panel_data.get('pivot_price', 'N/A') + + # Parse pivot price for prediction + cnn_predicted_price = 0 + if cnn_pivot_price != 'N/A' and cnn_pivot_price.startswith('$'): + try: + cnn_predicted_price = float(cnn_pivot_price[1:]) # Remove $ sign + except: + cnn_predicted_price = 0 cnn_model_info = { 'active': cnn_active, @@ -3028,16 +3086,29 @@ class CleanTradingDashboard: 'action': cnn_action, 'confidence': cnn_confidence, 'predicted_price': cnn_predicted_price, - 'type': cnn_latest.get('type', 'cnn_pivot') if cnn_latest else 'cnn_pivot' + 'pivot_price': cnn_pivot_price, + 'type': 'enhanced_cnn_pivot' }, - 'loss_5ma': cnn_state.get('current_loss'), + 'loss_5ma': float(cnn_panel_data.get('last_training_loss', '0.0').replace('f', '')), 'initial_loss': cnn_state.get('initial_loss'), 'best_loss': cnn_state.get('best_loss'), 'improvement': safe_improvement_calc( cnn_state.get('initial_loss'), - cnn_state.get('current_loss'), - 0.0 # No synthetic default improvement + float(cnn_panel_data.get('last_training_loss', '0.0').replace('f', '')), + 0.0 ), + + # Enhanced timing metrics + 'enhanced_timing': { + 'last_inference_time': cnn_panel_data.get('last_inference_time', 'Never'), + 'last_inference_duration': cnn_panel_data.get('last_inference_duration', '0.0ms'), + 'inference_count': cnn_panel_data.get('inference_count', 0), + 'inference_rate': cnn_panel_data.get('inference_rate', '0.00/s'), + 'last_training_time': cnn_panel_data.get('last_training_time', 'Never'), + 'last_training_duration': cnn_panel_data.get('last_training_duration', '0.0ms'), + 'training_count': cnn_panel_data.get('training_count', 0), + 'training_samples': cnn_panel_data.get('training_samples', 0) + }, 'checkpoint_loaded': cnn_state.get('checkpoint_loaded', False), 'model_type': 'CNN', 'description': 'Williams Market Structure CNN (Data Bus Input)', @@ -5832,6 +5903,44 @@ class CleanTradingDashboard: logger.error(f"Error getting pivot points for {symbol}: {e}") return [] + def _format_cnn_metrics_for_display(self) -> Dict[str, str]: + """Format CNN metrics for dashboard display""" + try: + cnn_panel_data = self._update_cnn_model_panel() + + # Format the metrics for display + formatted_metrics = { + 'status': cnn_panel_data.get('status', 'NOT_AVAILABLE'), + 'parameters': '50.0M', + 'last_inference': f"Inf: {cnn_panel_data.get('last_inference_time', 'Never')} ({cnn_panel_data.get('last_inference_duration', '0.0ms')})", + 'last_training': f"Train: {cnn_panel_data.get('last_training_time', 'Never')} ({cnn_panel_data.get('last_training_duration', '0.0ms')})", + 'inference_rate': cnn_panel_data.get('inference_rate', '0.00/s'), + 'training_samples': str(cnn_panel_data.get('training_samples', 0)), + 'current_loss': cnn_panel_data.get('last_training_loss', '0.000000'), + 'suggested_action': cnn_panel_data.get('suggested_action', 'HOLD'), + 'pivot_price': cnn_panel_data.get('pivot_price', 'N/A'), + 'confidence': f"{cnn_panel_data.get('confidence', 0.0):.1%}", + 'prediction_summary': f"{cnn_panel_data.get('suggested_action', 'HOLD')} @ {cnn_panel_data.get('pivot_price', 'N/A')} ({cnn_panel_data.get('confidence', 0.0):.1%})" + } + + return formatted_metrics + + except Exception as e: + logger.error(f"Error formatting CNN metrics for display: {e}") + return { + 'status': 'ERROR', + 'parameters': '0M', + 'last_inference': 'Inf: Error', + 'last_training': 'Train: Error', + 'inference_rate': '0.00/s', + 'training_samples': '0', + 'current_loss': '0.000000', + 'suggested_action': 'HOLD', + 'pivot_price': 'N/A', + 'confidence': '0.0%', + 'prediction_summary': 'Error' + } + def _start_cnn_prediction_loop(self): """Start CNN real-time prediction loop""" try: