popov/gogo2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

main cleanup

Browse Source
This commit is contained in:
Dobromir Popov
2025-09-30 23:56:36 +03:00
parent 468a2c2a66
commit 608da8233f
52 changed files with 5308 additions and 9985 deletions
Download Patch File Download Diff File Expand all files Collapse all files

399
web/clean_dashboard.py
View File

@@ -1,6 +1,21 @@
"""
Clean Trading Dashboard - Modular Implementation
CRITICAL POLICY: NO SYNTHETIC DATA ALLOWED
This module MUST ONLY use real market data from exchanges.
NEVER use:
- np.random.* for any data generation
- Mock/fake/synthetic data
- Placeholder values that simulate real data
If data is unavailable:
- Return None, 0, or empty collections
- Log clear error messages
- Raise exceptions if critical
See: reports/REAL_MARKET_DATA_POLICY.md
This dashboard is fully integrated with the Universal Data Stream architecture
and receives the standardized 5 timeseries format:
@@ -78,6 +93,9 @@ from core.trading_executor import TradingExecutor
from web.layout_manager import DashboardLayoutManager
from web.component_manager import DashboardComponentManager
# Import backtest training panel
from core.backtest_training_panel import BacktestTrainingPanel
try:
from core.cob_integration import COBIntegration
@@ -146,6 +164,12 @@ class CleanTradingDashboard:
trading_executor=self.trading_executor
)
self.component_manager = DashboardComponentManager()
# Initialize backtest training panel
self.backtest_training_panel = BacktestTrainingPanel(
data_provider=self.data_provider,
orchestrator=self.orchestrator
)
# Initialize Universal Data Adapter access through orchestrator
if UNIVERSAL_DATA_AVAILABLE:
@@ -427,7 +451,7 @@ class CleanTradingDashboard:
# Get recent predictions (last 24 hours)
predictions = []
# Mock data for now - replace with actual database query
# Query real prediction data from database
import sqlite3
try:
with sqlite3.connect(db.db_path) as conn:
@@ -1181,6 +1205,255 @@ class CleanTradingDashboard:
logger.error(f"Error in chained inference callback: {e}")
return f"❌ Error: {str(e)}"
# Backtest Training Panel Callbacks
self._setup_backtest_training_callbacks()
def _create_candlestick_chart(self, stats):
"""Create mini candlestick chart for visualization"""
try:
import plotly.graph_objects as go
from datetime import datetime
candlestick_data = stats.get('candlestick_data', [])
if not candlestick_data:
# Empty chart
fig = go.Figure()
fig.update_layout(
title="No Data Available",
paper_bgcolor='rgba(0,0,0,0)',
plot_bgcolor='rgba(0,0,0,0)',
font_color='white',
height=200
)
return fig
# Create candlestick chart
fig = go.Figure(data=[
go.Candlestick(
x=[d.get('timestamp', datetime.now()) for d in candlestick_data],
open=[d['open'] for d in candlestick_data],
high=[d['high'] for d in candlestick_data],
low=[d['low'] for d in candlestick_data],
close=[d['close'] for d in candlestick_data],
name='ETH/USDT'
)
])
fig.update_layout(
title="Recent Price Action",
yaxis_title="Price (USDT)",
xaxis_rangeslider_visible=False,
paper_bgcolor='rgba(0,0,0,0)',
plot_bgcolor='rgba(31,41,55,0.5)',
font_color='white',
height=200,
margin=dict(l=10, r=10, t=40, b=10)
)
fig.update_xaxes(showgrid=False, color='white')
fig.update_yaxes(showgrid=True, gridcolor='rgba(255,255,255,0.1)', color='white')
return fig
except Exception as e:
logger.error(f"Error creating candlestick chart: {e}")
return go.Figure()
def _create_best_predictions_display(self, stats):
"""Create display for best predictions"""
try:
best_predictions = stats.get('recent_predictions', [])
if not best_predictions:
return [html.Div("No predictions yet", className="text-muted small")]
prediction_items = []
for i, pred in enumerate(best_predictions[:5]): # Show top 5
accuracy_color = "green" if pred.get('accuracy', 0) > 0.6 else "orange" if pred.get('accuracy', 0) > 0.5 else "red"
prediction_item = html.Div([
html.Div([
html.Span(f"{pred.get('horizon', '?')}m ", className="fw-bold text-light"),
html.Span(".1%", style={"color": accuracy_color}, className="small"),
html.Span(f" conf: {pred.get('confidence', 0):.2f}", className="text-muted small ms-2")
], className="d-flex justify-content-between"),
html.Div([
html.Span(f"Pred: {pred.get('predicted_range', 'N/A')}", className="text-info small"),
html.Span(f" {pred.get('profit_potential', 'N/A')}", className="text-success small ms-2")
], className="mt-1")
], className="mb-2 p-2 bg-secondary rounded")
prediction_items.append(prediction_item)
return prediction_items
except Exception as e:
logger.error(f"Error creating best predictions display: {e}")
return [html.Div("Error loading predictions", className="text-danger small")]
@self.app.callback(
Output("backtest-training-state", "data"),
[Input("backtest-start-training-btn", "n_clicks"),
Input("backtest-stop-training-btn", "n_clicks"),
Input("backtest-run-backtest-btn", "n_clicks")],
[State("backtest-training-duration-slider", "value"),
State("backtest-training-state", "data")]
)
def handle_backtest_training_controls(start_clicks, stop_clicks, backtest_clicks, duration, current_state):
"""Handle backtest training control button clicks"""
ctx = dash.callback_context
if not ctx.triggered:
return current_state
button_id = ctx.triggered[0]["prop_id"].split(".")[0]
if button_id == "backtest-start-training-btn":
self.backtest_training_panel.start_training(duration)
logger.info(f"Backtest training started for {duration} hours")
elif button_id == "backtest-stop-training-btn":
self.backtest_training_panel.stop_training()
logger.info("Backtest training stopped")
elif button_id == "backtest-run-backtest-btn":
self.backtest_training_panel._run_backtest()
logger.info("Manual backtest executed")
return self.backtest_training_panel.get_training_stats()
def _setup_backtest_training_callbacks(self):
"""Setup callbacks for the backtest training panel"""
@self.app.callback(
[Output("backtest-training-status", "children"),
Output("backtest-current-accuracy", "children"),
Output("backtest-training-cycles", "children"),
Output("backtest-training-progress-bar", "style"),
Output("backtest-progress-text", "children"),
Output("backtest-gpu-status", "children"),
Output("backtest-model-status", "children"),
Output("backtest-accuracy-chart", "figure"),
Output("backtest-candlestick-chart", "figure"),
Output("backtest-best-predictions", "children")],
[Input("backtest-training-update-interval", "n_intervals"),
State("backtest-training-duration-slider", "value")]
)
def update_backtest_training_status(n_intervals, duration_hours):
"""Update backtest training panel status"""
try:
stats = self.backtest_training_panel.get_training_stats()
# Training status
status = html.Span(
"Active" if self.backtest_training_panel.training_active else "Inactive",
style={"color": "green" if self.backtest_training_panel.training_active else "red"}
)
# Current accuracy
accuracy = f"{stats['current_accuracy']:.2f}%"
# Training cycles
cycles = str(stats['training_cycles'])
# Progress
progress_percentage = 0
progress_text = "Ready to start"
progress_style = {
"width": "0%",
"height": "20px",
"backgroundColor": "#007bff",
"borderRadius": "4px",
"transition": "width 0.3s ease"
}
if self.backtest_training_panel.training_active and stats['start_time']:
elapsed = (datetime.now() - stats['start_time']).total_seconds() / 3600
# Progress based on selected training duration
progress_percentage = min(100, (elapsed / max(1, duration_hours)) * 100)
progress_text = ".1f"
progress_style["width"] = f"{progress_percentage}%"
# GPU/NPU status with detailed info
gpu_available = self.backtest_training_panel.gpu_available
npu_available = self.backtest_training_panel.npu_available
gpu_status = []
if gpu_available:
gpu_type = getattr(self.backtest_training_panel, 'gpu_type', 'GPU')
gpu_status.append(html.Span(f"{gpu_type} ✓", style={"color": "green"}))
else:
gpu_status.append(html.Span("GPU ✗", style={"color": "red"}))
if npu_available:
gpu_status.append(html.Span(" NPU ✓", style={"color": "green"}))
else:
gpu_status.append(html.Span(" NPU ✗", style={"color": "red"}))
# Model status
model_status = self.backtest_training_panel._get_model_status()
# Accuracy chart
chart = self.backtest_training_panel.update_accuracy_chart()
# Candlestick chart
candlestick_chart = self._create_candlestick_chart(stats)
# Best predictions display
best_predictions = self._create_best_predictions_display(stats)
return status, accuracy, cycles, progress_style, progress_text, gpu_status, model_status, chart, candlestick_chart, best_predictions
except Exception as e:
logger.error(f"Error updating backtest training status: {e}")
return [html.Span("Error", style={"color": "red"})] * 10
@self.app.callback(
Output("backtest-training-state", "data"),
[Input("backtest-start-training-btn", "n_clicks"),
Input("backtest-stop-training-btn", "n_clicks"),
Input("backtest-run-backtest-btn", "n_clicks")],
[State("backtest-training-duration-slider", "value"),
State("backtest-training-state", "data")]
)
def handle_backtest_training_controls(start_clicks, stop_clicks, backtest_clicks, duration, current_state):
"""Handle backtest training control button clicks"""
ctx = dash.callback_context
if not ctx.triggered:
return current_state
button_id = ctx.triggered[0]["prop_id"].split(".")[0]
if button_id == "backtest-start-training-btn":
self.backtest_training_panel.start_training(duration)
logger.info(f"Backtest training started for {duration} hours")
elif button_id == "backtest-stop-training-btn":
self.backtest_training_panel.stop_training()
logger.info("Backtest training stopped")
elif button_id == "backtest-run-backtest-btn":
self.backtest_training_panel._run_backtest()
logger.info("Manual backtest executed")
return self.backtest_training_panel.get_training_stats()
# Add interval for backtest training updates
self.app.layout.children.append(
dcc.Interval(
id="backtest-training-update-interval",
interval=5000, # Update every 5 seconds
n_intervals=0
)
)
# Add store for backtest training state
self.app.layout.children.append(
dcc.Store(id="backtest-training-state", data=self.backtest_training_panel.get_training_stats())
)
def _get_real_model_performance_data(self) -> Dict[str, Any]:
"""Get real model performance data from orchestrator"""
try:
@@ -1779,6 +2052,9 @@ class CleanTradingDashboard:
# ADD TRADES TO MAIN CHART
self._add_trades_to_chart(fig, symbol, df_main, row=1)
# ADD PIVOT POINTS TO MAIN CHART
self._add_pivot_points_to_chart(fig, symbol, df_main, row=1)
# Mini 1-second chart (if available)
if has_mini_chart and ws_data_1s is not None:
@@ -2856,7 +3132,107 @@ class CleanTradingDashboard:
except Exception as e:
logger.warning(f"Error adding trades to chart: {e}")
def _add_pivot_points_to_chart(self, fig: go.Figure, symbol: str, df_main: pd.DataFrame, row: int = 1):
"""Add nested pivot points to the chart"""
try:
# Get pivot bounds from data provider
if not hasattr(self, 'data_provider') or not self.data_provider:
return
pivot_bounds = self.data_provider.get_pivot_bounds(symbol)
if not pivot_bounds or not hasattr(pivot_bounds, 'pivot_support_levels'):
return
support_levels = pivot_bounds.pivot_support_levels
resistance_levels = pivot_bounds.pivot_resistance_levels
if not support_levels and not resistance_levels:
return
# Get chart time range for pivot display
chart_start = df_main.index.min()
chart_end = df_main.index.max()
# Define colors for different pivot levels
pivot_colors = {
'support': ['rgba(0, 255, 0, 0.3)', 'rgba(0, 200, 0, 0.4)', 'rgba(0, 150, 0, 0.5)'],
'resistance': ['rgba(255, 0, 0, 0.3)', 'rgba(200, 0, 0, 0.4)', 'rgba(150, 0, 0, 0.5)']
}
# Add support levels
for i, support_price in enumerate(support_levels[-5:]): # Show last 5 support levels
color_idx = min(i, len(pivot_colors['support']) - 1)
fig.add_trace(
go.Scatter(
x=[chart_start, chart_end],
y=[support_price, support_price],
mode='lines',
line=dict(
color=pivot_colors['support'][color_idx],
width=2,
dash='dot'
),
name=f'Support L{i+1}: ${support_price:.2f}',
showlegend=True,
hovertemplate=f"Support Level {i+1}: ${{y:.2f}}<extra></extra>"
),
row=row, col=1
)
# Add resistance levels
for i, resistance_price in enumerate(resistance_levels[-5:]): # Show last 5 resistance levels
color_idx = min(i, len(pivot_colors['resistance']) - 1)
fig.add_trace(
go.Scatter(
x=[chart_start, chart_end],
y=[resistance_price, resistance_price],
mode='lines',
line=dict(
color=pivot_colors['resistance'][color_idx],
width=2,
dash='dot'
),
name=f'Resistance L{i+1}: ${resistance_price:.2f}',
showlegend=True,
hovertemplate=f"Resistance Level {i+1}: ${{y:.2f}}<extra></extra>"
),
row=row, col=1
)
# Add pivot context annotation if available
if hasattr(pivot_bounds, 'pivot_context') and pivot_bounds.pivot_context:
context = pivot_bounds.pivot_context
if isinstance(context, dict) and 'trend_direction' in context:
trend = context.get('trend_direction', 'UNKNOWN')
strength = context.get('trend_strength', 0.0)
nested_levels = context.get('nested_levels', 0)
# Add trend annotation
trend_color = {
'UPTREND': 'green',
'DOWNTREND': 'red',
'SIDEWAYS': 'orange'
}.get(trend, 'gray')
fig.add_annotation(
xref="paper", yref="paper",
x=0.02, y=0.98,
text=f"Trend: {trend} ({strength:.1%}) | Pivots: {nested_levels} levels",
showarrow=False,
bgcolor="rgba(0,0,0,0.7)",
bordercolor=trend_color,
borderwidth=1,
borderpad=4,
font=dict(color="white", size=10),
row=row, col=1
)
logger.debug(f"Added {len(support_levels)} support and {len(resistance_levels)} resistance levels to chart")
except Exception as e:
logger.warning(f"Error adding pivot points to chart: {e}")
def _get_price_at_time(self, df: pd.DataFrame, timestamp) -> Optional[float]:
"""Get price from dataframe at specific timestamp"""
try:
@@ -2924,10 +3300,11 @@ class CleanTradingDashboard:
if 'volume' in df.columns and df['volume'].sum() > 0:
df_resampled['volume'] = df['volume'].resample('1s').sum()
else:
# Use tick count as volume proxy with some randomization for variety
import random
# CRITICAL: NO SYNTHETIC DATA - If volume unavailable, set to 0
# NEVER use random.randint() or any synthetic data generation
tick_counts = df[price_col].resample('1s').count()
df_resampled['volume'] = tick_counts * (50 + random.randint(0, 100))
df_resampled['volume'] = 0 # No volume data available
logger.warning(f"Volume data unavailable for 1s timeframe {symbol} - using 0 (NEVER synthetic)")
# For 1m timeframe, volume is already in the raw data
# Remove any NaN rows and limit to max bars
@@ -7834,9 +8211,13 @@ class CleanTradingDashboard:
price_change = (next_price - current_price) / current_price if current_price > 0 else 0
cumulative_imbalance = current_data.get('cumulative_imbalance', {})
# TODO(Guideline: no synthetic data) Replace the random baseline with real orchestrator features.
# TODO(Guideline: no synthetic data) Replace the random baseline with real orchestrator features.
features = np.random.randn(100)
# CRITICAL: Extract REAL features from orchestrator - NEVER use np.random or synthetic data
if not self.orchestrator or not hasattr(self.orchestrator, 'extract_features'):
logger.error("CRITICAL: Cannot train CNN - orchestrator feature extraction unavailable. NEVER use synthetic data.")
continue
# Build real feature vector from actual market data
features = np.zeros(100)
features[0] = current_price / 10000
features[1] = price_change
features[2] = current_data.get('volume', 0) / 1000000
@@ -7845,6 +8226,8 @@ class CleanTradingDashboard:
features[4] = cumulative_imbalance.get('5s', 0.0)
features[5] = cumulative_imbalance.get('15s', 0.0)
features[6] = cumulative_imbalance.get('60s', 0.0)
# Leave remaining features as 0.0 until proper feature extraction is implemented
# NEVER fill with random values
if price_change > 0.001: target = 2
elif price_change < -0.001: target = 0
else: target = 1