From 9a44ddfa3c3b2d0921ceec03fdcdc5876ad772b3 Mon Sep 17 00:00:00 2001 From: Dobromir Popov Date: Sat, 31 May 2025 01:01:06 +0300 Subject: [PATCH] PROFITABLE! no CNN training; less logging --- core/data_provider.py | 2 +- dataprovider_realtime.py | 2 +- training/williams_market_structure.py | 2 +- web/dashboard.py | 219 ++++++++++++++++++++++++-- 4 files changed, 208 insertions(+), 17 deletions(-) diff --git a/core/data_provider.py b/core/data_provider.py index 99cd675..9ec350a 100644 --- a/core/data_provider.py +++ b/core/data_provider.py @@ -1581,7 +1581,7 @@ class DataProvider: # Convert to sorted list for consistent ordering common_feature_names = sorted(list(common_feature_names)) - logger.info(f"Using {len(common_feature_names)} common features: {common_feature_names}") + # logger.info(f"Using {len(common_feature_names)} common features: {common_feature_names}") # Second pass: create feature channels with common features for tf in timeframes: diff --git a/dataprovider_realtime.py b/dataprovider_realtime.py index 45191c6..60d297e 100644 --- a/dataprovider_realtime.py +++ b/dataprovider_realtime.py @@ -464,7 +464,7 @@ class MultiTimeframeDataInterface: self.dataframes[timeframe] is not None and self.last_updates[timeframe] is not None and (current_time - self.last_updates[timeframe]).total_seconds() < 60): - logger.info(f"Using cached data for {self.symbol} {timeframe}") + #logger.info(f"Using cached data for {self.symbol} {timeframe}") return self.dataframes[timeframe] interval_seconds = self.timeframe_to_seconds.get(timeframe, 3600) diff --git a/training/williams_market_structure.py b/training/williams_market_structure.py index 1f337f4..8819b9d 100644 --- a/training/williams_market_structure.py +++ b/training/williams_market_structure.py @@ -919,7 +919,7 @@ class WilliamsMarketStructure: else: X_predict_batch = X_predict # Or handle error - logger.info(f"CNN Predicting with X_shape: {X_predict_batch.shape}") + # logger.info(f"CNN Predicting with X_shape: {X_predict_batch.shape}") pred_class, pred_proba = self.cnn_model.predict(X_predict_batch) # predict expects batch # pred_class/pred_proba might be arrays if batch_size > 1, or if output is multi-dim diff --git a/web/dashboard.py b/web/dashboard.py index 8656ff5..991dc2b 100644 --- a/web/dashboard.py +++ b/web/dashboard.py @@ -1245,16 +1245,19 @@ class TradingDashboard: risk_level = "Extreme Risk" risk_class = "bg-dark" + # Create CNN monitoring content + try: + cnn_monitoring_content = self._create_cnn_monitoring_content() + except Exception as e: + logger.warning(f"CNN monitoring error: {e}") + cnn_monitoring_content = [html.P("CNN monitoring unavailable", className="text-muted")] + return ( price_text, pnl_text, pnl_class, fees_text, position_text, position_class, trade_count_text, portfolio_text, mexc_status, price_chart, training_metrics, decisions_list, session_perf, closed_trades_table, system_status['icon_class'], system_status['title'], system_status['details'], leverage_text, f"{risk_level}", - # # Model data feed charts - # self._create_model_data_chart('ETH/USDT', '1m'), - # self._create_model_data_chart('ETH/USDT', '1h'), - # self._create_model_data_chart('ETH/USDT', '1d'), - # self._create_model_data_chart('BTC/USDT', '1s') + cnn_monitoring_content ) except Exception as e: @@ -1273,11 +1276,7 @@ class TradingDashboard: "Error: Dashboard error - check logs", [html.P(f"Error: {str(e)}", className="text-danger")], f"{self.leverage_multiplier:.0f}x", "Error", - # Model data feed charts - # self._create_model_data_chart('ETH/USDT', '1m'), - # self._create_model_data_chart('ETH/USDT', '1h'), - # self._create_model_data_chart('ETH/USDT', '1d'), - # self._create_model_data_chart('BTC/USDT', '1s') + [html.P("CNN monitoring unavailable", className="text-danger")] ) # Clear history callback @@ -2380,6 +2379,7 @@ class TradingDashboard: 'entry_price': entry_price, 'exit_price': exit_price, 'size': size, + 'leverage': self.leverage_multiplier, # Store leverage used 'gross_pnl': leveraged_pnl, 'fees': leveraged_fee + self.current_position['fees'], 'fee_type': fee_type, @@ -2541,6 +2541,7 @@ class TradingDashboard: 'entry_price': entry_price, 'exit_price': exit_price, 'size': size, + 'leverage': self.leverage_multiplier, # Store leverage used 'gross_pnl': leveraged_pnl, 'fees': leveraged_fee + self.current_position['fees'], 'fee_type': fee_type, @@ -2758,12 +2759,22 @@ class TradingDashboard: # Format side color side_color = "text-success" if trade['side'] == 'LONG' else "text-danger" - # Format position size + # Calculate leveraged position size in USD position_size = trade.get('size', 0) - size_display = f"{position_size:.4f}" if position_size < 1 else f"{position_size:.2f}" + entry_price = trade.get('entry_price', 0) + leverage_used = trade.get('leverage', self.leverage_multiplier) # Use trade's leverage or current - # Simple total fees display + # Base position value in USD + base_position_usd = position_size * entry_price + # Leveraged position value (this is what we're actually exposed to) + leveraged_position_usd = base_position_usd * leverage_used + + # Display format: show both base crypto amount and leveraged USD value + size_display = f"{position_size:.4f} ETH (${leveraged_position_usd:,.0f}@{leverage_used:.0f}x)" + + # Leverage-adjusted fees display total_fees = trade.get('fees', 0) + # Note: Fees should already be calculated correctly with leverage in the P&L calculation table_rows.append( html.Tr([ @@ -2786,7 +2797,7 @@ class TradingDashboard: html.Tr([ html.Th("ID", className="small"), html.Th("Side", className="small"), - html.Th("Size", className="small"), + html.Th("Position Size", className="small"), html.Th("Entry", className="small"), html.Th("Exit", className="small"), html.Th("Total Fees", className="small"), @@ -3405,6 +3416,186 @@ class TradingDashboard: logger.error(f"Error getting 1-second bars: {e}") return pd.DataFrame() + def _create_cnn_monitoring_content(self) -> List: + """Create CNN monitoring and prediction analysis content""" + try: + # Get CNN monitoring data + if CNN_MONITORING_AVAILABLE: + cnn_data = get_cnn_dashboard_data() + else: + cnn_data = {'statistics': {'total_predictions_logged': 0}} + + components = [] + + # CNN Statistics Overview + stats = cnn_data.get('statistics', {}) + components.append(html.Div([ + html.H6([ + html.I(className="fas fa-chart-bar me-2"), + "CNN Performance Overview" + ], className="mb-2"), + html.Div([ + html.Div([ + html.Strong(f"{stats.get('total_predictions_logged', 0):,}"), + html.Br(), + html.Small("Total Predictions", className="text-muted") + ], className="text-center", style={"flex": "1"}), + html.Div([ + html.Strong(f"{stats.get('avg_prediction_latency_ms', 0):.1f}ms"), + html.Br(), + html.Small("Avg Latency", className="text-muted") + ], className="text-center", style={"flex": "1"}), + html.Div([ + html.Strong(f"{stats.get('avg_confidence', 0)*100:.1f}%"), + html.Br(), + html.Small("Avg Confidence", className="text-muted") + ], className="text-center", style={"flex": "1"}), + html.Div([ + html.Strong(f"{len(stats.get('active_models', []))}"), + html.Br(), + html.Small("Active Models", className="text-muted") + ], className="text-center", style={"flex": "1"}) + ], style={"display": "flex", "gap": "10px", "marginBottom": "15px"}) + ])) + + # Recent Predictions Table + recent_predictions = cnn_data.get('recent_predictions', []) + if recent_predictions: + components.append(html.Div([ + html.H6([ + html.I(className="fas fa-list-alt me-2"), + "Recent CNN Predictions" + ], className="mb-2"), + self._create_cnn_predictions_table(recent_predictions[-10:]) # Last 10 predictions + ])) + else: + components.append(html.Div([ + html.H6("Recent Predictions", className="mb-2"), + html.P("No recent predictions available", className="text-muted") + ])) + + # Model Performance Comparison + model_stats = cnn_data.get('model_performance', {}) + if model_stats: + components.append(html.Div([ + html.H6([ + html.I(className="fas fa-trophy me-2"), + "Model Performance Comparison" + ], className="mb-2"), + self._create_model_performance_table(model_stats) + ])) + + return components + + except Exception as e: + logger.error(f"Error creating CNN monitoring content: {e}") + return [html.P(f"Error loading CNN monitoring: {str(e)}", className="text-danger")] + + def _create_cnn_predictions_table(self, predictions: List[Dict]) -> html.Table: + """Create table showing recent CNN predictions""" + try: + if not predictions: + return html.P("No predictions available", className="text-muted") + + # Table headers + headers = ["Time", "Model", "Symbol", "Action", "Confidence", "Latency", "Price Context"] + + # Create rows + rows = [] + for pred in reversed(predictions): # Most recent first + try: + timestamp = pred.get('timestamp', '') + if isinstance(timestamp, str): + # Format timestamp for display + from datetime import datetime + dt = datetime.fromisoformat(timestamp.replace('Z', '+00:00')) + time_str = dt.strftime('%H:%M:%S') + else: + time_str = str(timestamp)[-8:] # Last 8 chars for time + + model_name = pred.get('model_name', 'Unknown')[:12] # Truncate long names + symbol = pred.get('symbol', '') + action_name = pred.get('action_name', 'HOLD') + confidence = pred.get('confidence', 0) * 100 + latency = pred.get('prediction_latency_ms', 0) + current_price = pred.get('current_price', 0) + + # Action styling + if action_name == 'BUY': + action_badge = html.Span(action_name, className="badge bg-success text-white") + elif action_name == 'SELL': + action_badge = html.Span(action_name, className="badge bg-danger text-white") + else: + action_badge = html.Span(action_name, className="badge bg-secondary") + + # Confidence styling + if confidence > 70: + conf_class = "text-success fw-bold" + elif confidence > 50: + conf_class = "text-warning" + else: + conf_class = "text-muted" + + row = html.Tr([ + html.Td(time_str, className="small"), + html.Td(model_name, className="small"), + html.Td(symbol, className="small"), + html.Td(action_badge), + html.Td(f"{confidence:.1f}%", className=f"small {conf_class}"), + html.Td(f"{latency:.1f}ms", className="small text-muted"), + html.Td(f"${current_price:.2f}" if current_price else "N/A", className="small") + ]) + rows.append(row) + except Exception as e: + logger.warning(f"Error processing prediction row: {e}") + continue + + return html.Table([ + html.Thead([ + html.Tr([html.Th(h, className="small") for h in headers]) + ]), + html.Tbody(rows) + ], className="table table-sm table-striped") + + except Exception as e: + logger.error(f"Error creating CNN predictions table: {e}") + return html.P(f"Error creating predictions table: {str(e)}", className="text-danger") + + def _create_model_performance_table(self, model_stats: Dict) -> html.Table: + """Create table showing model performance metrics""" + try: + if not model_stats: + return html.P("No model performance data available", className="text-muted") + + headers = ["Model", "Predictions", "Avg Confidence", "Avg Latency", "Memory Usage"] + rows = [] + + for model_name, stats in model_stats.items(): + prediction_count = stats.get('prediction_count', 0) + avg_confidence = stats.get('avg_confidence', 0) * 100 + avg_latency = stats.get('avg_latency_ms', 0) + memory_usage = stats.get('avg_memory_usage_mb', 0) + + row = html.Tr([ + html.Td(model_name[:15], className="small"), # Truncate long names + html.Td(f"{prediction_count:,}", className="small"), + html.Td(f"{avg_confidence:.1f}%", className="small"), + html.Td(f"{avg_latency:.1f}ms", className="small"), + html.Td(f"{memory_usage:.0f}MB" if memory_usage else "N/A", className="small") + ]) + rows.append(row) + + return html.Table([ + html.Thead([ + html.Tr([html.Th(h, className="small") for h in headers]) + ]), + html.Tbody(rows) + ], className="table table-sm table-striped") + + except Exception as e: + logger.error(f"Error creating model performance table: {e}") + return html.P(f"Error creating performance table: {str(e)}", className="text-danger") + def _create_training_metrics(self) -> List: """Create comprehensive model training metrics display with enhanced RL integration""" try: