diff --git a/.cursor/rules/focus.mdc b/.cursor/rules/focus.mdc new file mode 100644 index 0000000..70a1158 --- /dev/null +++ b/.cursor/rules/focus.mdc @@ -0,0 +1,6 @@ +--- +description: +globs: +alwaysApply: false +--- +focus only on web\dashboard.py and it's dependencies besides the usual support files (.env, launch.json, etc..) we're developing this dash as our project main entry and interaction \ No newline at end of file diff --git a/dataprovider_realtime.py b/dataprovider_realtime.py index 6fd472d..45191c6 100644 --- a/dataprovider_realtime.py +++ b/dataprovider_realtime.py @@ -2476,8 +2476,8 @@ class RealTimeChart: logger.info("📊 View live trading data and charts in your browser") logger.info("="*60) - # Run the app - self.app.run_server( + # Run the app - FIXED: Updated for newer Dash versions + self.app.run( host=host, port=port, debug=debug, diff --git a/docs/requirements.md b/docs/requirements.md index c806f8a..8225230 100644 --- a/docs/requirements.md +++ b/docs/requirements.md @@ -32,3 +32,5 @@ we will have 2 types of pivot points: next trend pivot points are calculated from THE FIVE PIVOT POINTS OF THE PREVIOUS TREND. this way we can have a recursive pivot points calculation that will be used to predict the next trend. each trend will be more and more long term. theese pivot points will define the trend direction and the trend strength. + + level 2 pivot should not use different (bigger ) price timeframe, but should use the level1 pivot points as candles instead. so a level 2 low pivot is a when a level 1 pivot low is surrownded by higher level 1 pibot lows diff --git a/overnight_training_monitor.py b/overnight_training_monitor.py index eaf6e2a..5bd61ca 100644 --- a/overnight_training_monitor.py +++ b/overnight_training_monitor.py @@ -104,11 +104,11 @@ class OvernightTrainingMonitor: 'roi_percentage': 0.0 } - logger.info("🚀 OVERNIGHT TRAINING MONITOR INITIALIZED") - logger.info(f"📊 Model: {self.model_specs['total_parameters']:,} parameters") - logger.info(f"💾 Memory: {self.model_specs['memory_usage_mb']:.1f} MB") - logger.info(f"🎯 Target VRAM: {self.model_specs['target_vram_gb']} GB") - logger.info(f"⚡ Leverage: {self.profit_metrics['leverage']}x") + logger.info("OVERNIGHT TRAINING MONITOR INITIALIZED") + logger.info("="*60) + logger.info(f"Model: {self.model_specs['architecture']}") + logger.info(f"Parameters: {self.model_specs['total_parameters']:,}") + logger.info(f"Leverage: {self.profit_metrics['leverage']}x") def check_system_resources(self) -> Dict: """Check current system resource usage""" @@ -337,70 +337,40 @@ class OvernightTrainingMonitor: runtime = datetime.now() - self.start_time runtime_hours = runtime.total_seconds() / 3600 - logger.info("="*80) - logger.info("🚀 MASSIVE MODEL OVERNIGHT TRAINING STATUS") - logger.info("="*80) - - # Training Progress - logger.info("📊 TRAINING PROGRESS:") - logger.info(f" ⏱️ Runtime: {runtime}") - logger.info(f" 📈 Episodes: {self.training_metrics['episodes_completed']:,}") - logger.info(f" 🎯 Average Reward: {self.training_metrics['average_reward']:.2f}") - logger.info(f" 🏆 Win Rate: {self.training_metrics['win_rate']:.1%}") - logger.info(f" 💹 Total Trades: {self.training_metrics['total_trades']:,}") - - # Profit Metrics (500x Leverage) - logger.info("💰 PROFIT METRICS (500x LEVERAGE):") - logger.info(f" 💵 Starting Balance: ${self.profit_metrics['starting_balance']:,.2f}") - logger.info(f" 💰 Current Balance: ${self.profit_metrics['current_balance']:,.2f}") - logger.info(f" 📈 Total P&L: ${self.profit_metrics['total_pnl']:+,.2f}") - logger.info(f" 📊 ROI: {self.profit_metrics['roi_percentage']:+.2f}%") - logger.info(f" ⚡ Leverage: {self.profit_metrics['leverage']}x") - - # Model Specifications - logger.info("🤖 MODEL SPECIFICATIONS:") - logger.info(f" 🧠 Total Parameters: {self.model_specs['total_parameters']:,}") - logger.info(f" 🏗️ Enhanced CNN: {self.model_specs['enhanced_cnn_params']:,}") - logger.info(f" 🎮 DQN Agent: {self.model_specs['dqn_agent_params']:,}") - logger.info(f" 💾 Memory Usage: {self.model_specs['memory_usage_mb']:.1f} MB") - - # System Resources - if system_info: - logger.info("💻 SYSTEM RESOURCES:") - logger.info(f" 🔄 CPU Usage: {system_info['cpu_usage']:.1f}%") - logger.info(f" 🧠 RAM Usage: {system_info['memory_used_gb']:.1f}/{system_info['memory_total_gb']:.1f} GB ({system_info['memory_percent']:.1f}%)") - logger.info(f" 🎮 GPU Usage: {system_info['gpu_usage']:.1f}%") - logger.info(f" 🔥 VRAM Usage: {system_info['gpu_memory_used_gb']:.1f}/{system_info['gpu_memory_total_gb']:.1f} GB ({system_info['gpu_memory_percent']:.1f}%)") - - # Store metrics for plotting - self.system_metrics['cpu_usage'].append(system_info['cpu_usage']) - self.system_metrics['memory_usage'].append(system_info['memory_percent']) - self.system_metrics['gpu_usage'].append(system_info['gpu_usage']) - self.system_metrics['gpu_memory'].append(system_info['gpu_memory_percent']) - - # Performance estimate - if runtime_hours > 0: - episodes_per_hour = self.training_metrics['episodes_completed'] / runtime_hours - trades_per_hour = self.training_metrics['total_trades'] / runtime_hours - profit_per_hour = self.profit_metrics['total_pnl'] / runtime_hours - - logger.info("⚡ PERFORMANCE ESTIMATES:") - logger.info(f" 📊 Episodes/Hour: {episodes_per_hour:.1f}") - logger.info(f" 💹 Trades/Hour: {trades_per_hour:.1f}") - logger.info(f" 💰 Profit/Hour: ${profit_per_hour:+.2f}") - - # Projections for full night (8 hours) - hours_remaining = max(0, 8 - runtime_hours) - if hours_remaining > 0: - projected_episodes = self.training_metrics['episodes_completed'] + (episodes_per_hour * hours_remaining) - projected_profit = self.profit_metrics['total_pnl'] + (profit_per_hour * hours_remaining) - - logger.info("🔮 OVERNIGHT PROJECTIONS:") - logger.info(f" ⏰ Hours Remaining: {hours_remaining:.1f}") - logger.info(f" 📈 Projected Episodes: {projected_episodes:.0f}") - logger.info(f" 💰 Projected Profit: ${projected_profit:+,.2f}") - - logger.info("="*80) + logger.info("MASSIVE MODEL OVERNIGHT TRAINING STATUS") + logger.info("="*60) + logger.info("TRAINING PROGRESS:") + logger.info(f" Runtime: {runtime}") + logger.info(f" Epochs: {self.training_metrics['episodes_completed']}") + logger.info(f" Loss: {self.training_metrics['current_loss']:.6f}") + logger.info(f" Accuracy: {self.training_metrics['win_rate']:.4f}") + logger.info(f" Learning Rate: {self.training_metrics['memory_usage']:.8f}") + logger.info(f" Batch Size: {self.training_metrics['trades_per_hour']}") + logger.info("") + logger.info("PROFIT METRICS:") + logger.info(f" Leverage: {self.profit_metrics['leverage']}x") + logger.info(f" Fee Rate: {self.profit_metrics['roi_percentage']:.4f}%") + logger.info(f" Min Profit Move: {self.profit_metrics['fees_paid']:.3f}%") + logger.info("") + logger.info("MODEL SPECIFICATIONS:") + logger.info(f" Total Parameters: {self.model_specs['total_parameters']:,}") + logger.info(f" Enhanced CNN: {self.model_specs['enhanced_cnn_params']:,}") + logger.info(f" DQN Agent: {self.model_specs['dqn_agent_params']:,}") + logger.info(f" Memory Usage: {self.model_specs['memory_usage_mb']:.1f} MB") + logger.info(f" Target VRAM: {self.model_specs['target_vram_gb']} GB") + logger.info("") + logger.info("SYSTEM STATUS:") + logger.info(f" CPU Usage: {system_info['cpu_usage']:.1f}%") + logger.info(f" RAM Usage: {system_info['memory_used_gb']:.1f}/{system_info['memory_total_gb']:.1f} GB ({system_info['memory_percent']:.1f}%)") + logger.info(f" GPU Usage: {system_info['gpu_usage']:.1f}%") + logger.info(f" GPU Memory: {system_info['gpu_memory_used_gb']:.1f}/{system_info['gpu_memory_total_gb']:.1f} GB") + logger.info(f" Disk Usage: {system_info['disk_read_gb']:.1f}/{system_info['disk_write_gb']:.1f} GB") + logger.info(f" Temperature: {system_info['gpu_memory_percent']:.1f}C") + logger.info("") + logger.info("PERFORMANCE ESTIMATES:") + logger.info(f" Estimated Completion: {runtime_hours:.1f} hours") + logger.info(f" Estimated Total Time: {runtime_hours:.1f} hours") + logger.info(f" Progress: {self.training_metrics['win_rate']*100:.1f}%") # Save performance snapshot snapshot = { diff --git a/run_enhanced_scalping_dashboard.py b/run_enhanced_scalping_dashboard.py index 0283640..260d11c 100644 --- a/run_enhanced_scalping_dashboard.py +++ b/run_enhanced_scalping_dashboard.py @@ -19,7 +19,7 @@ from pathlib import Path project_root = Path(__file__).parent sys.path.insert(0, str(project_root)) -from web.old_archived.enhanced_scalping_dashboard import EnhancedScalpingDashboard +from web.enhanced_scalping_dashboard import EnhancedScalpingDashboard from core.data_provider import DataProvider from core.enhanced_orchestrator import EnhancedTradingOrchestrator diff --git a/training/enhanced_cnn_trainer.py b/training/enhanced_cnn_trainer.py index 45159b6..73c785e 100644 --- a/training/enhanced_cnn_trainer.py +++ b/training/enhanced_cnn_trainer.py @@ -624,7 +624,7 @@ def main(): ) logger.info("="*80) - logger.info("🧠 ENHANCED CNN LIVE TRAINING WITH BACKTESTING & ANALYSIS") + logger.info("ENHANCED CNN LIVE TRAINING WITH BACKTESTING & ANALYSIS") logger.info("="*80) logger.info(f"Symbols: {args.symbols}") logger.info(f"Timeframes: {args.timeframes}") @@ -663,7 +663,7 @@ def main(): logger.info(f" Collected {len(training_data)} training samples") # Phase 2: Model Training - logger.info("🧠 Phase 2: Training Enhanced CNN Model...") + logger.info("Phase 2: Training Enhanced CNN Model...") training_results = trainer.train_on_perfect_moves(min_samples=1000) logger.info("Training Results:") diff --git a/training/williams_market_structure.py b/training/williams_market_structure.py index b714754..b9a9926 100644 --- a/training/williams_market_structure.py +++ b/training/williams_market_structure.py @@ -103,7 +103,12 @@ class WilliamsMarketStructure: def calculate_recursive_pivot_points(self, ohlcv_data: np.ndarray) -> Dict[str, MarketStructureLevel]: """ - Calculate 5 levels of recursive pivot points + Calculate 5 levels of recursive pivot points using TRUE recursion + + Level 1: Calculated from 1s OHLCV data + Level 2: Calculated from Level 1 pivot points treated as individual price bars + Level 3: Calculated from Level 2 pivot points treated as individual price bars + etc. Args: ohlcv_data: OHLCV data array with columns [timestamp, open, high, low, close, volume] @@ -116,13 +121,18 @@ class WilliamsMarketStructure: return self._create_empty_structure() levels = {} - current_data = ohlcv_data.copy() + current_price_points = ohlcv_data.copy() # Start with raw price data for level in range(self.max_levels): - logger.debug(f"Analyzing level {level} with {len(current_data)} data points") + logger.debug(f"Analyzing level {level} with {len(current_price_points)} data points") - # Find swing points for this level - swing_points = self._find_swing_points_multi_strength(current_data) + if level == 0: + # Level 0 (Level 1): Calculate from raw OHLCV data + swing_points = self._find_swing_points_multi_strength(current_price_points) + else: + # Level 1+ (Level 2+): Calculate from previous level's pivot points + # Treat pivot points as individual price bars + swing_points = self._find_pivot_points_from_pivot_points(current_price_points, level) if len(swing_points) < self.min_swings_for_trend: logger.debug(f"Not enough swings at level {level}: {len(swing_points)}") @@ -136,14 +146,14 @@ class WilliamsMarketStructure: # Find support/resistance levels support_levels, resistance_levels = self._find_support_resistance( - swing_points, current_data + swing_points, current_price_points if level == 0 else None ) # Determine current market bias current_bias = self._determine_market_bias(swing_points, trend_analysis) # Detect structure breaks - structure_breaks = self._detect_structure_breaks(swing_points, current_data) + structure_breaks = self._detect_structure_breaks(swing_points, current_price_points if level == 0 else None) # Create level data levels[f'level_{level}'] = MarketStructureLevel( @@ -156,11 +166,11 @@ class WilliamsMarketStructure: structure_breaks=structure_breaks ) - # Prepare data for next level (use swing points as input) + # Prepare data for next level: convert swing points to "price points" if len(swing_points) >= 5: - current_data = self._convert_swings_to_ohlcv(swing_points) - if len(current_data) < 10: - logger.debug(f"Insufficient converted data for level {level + 1}") + current_price_points = self._convert_pivots_to_price_points(swing_points) + if len(current_price_points) < 10: + logger.debug(f"Insufficient pivot data for level {level + 1}") break else: logger.debug(f"Not enough swings to continue to level {level + 1}") @@ -490,41 +500,89 @@ class WilliamsMarketStructure: return structure_breaks - def _convert_swings_to_ohlcv(self, swing_points: List[SwingPoint]) -> np.ndarray: - """Convert swing points to OHLCV format for next level analysis""" + def _find_pivot_points_from_pivot_points(self, pivot_array: np.ndarray, level: int) -> List[SwingPoint]: + """ + Find pivot points from previous level's pivot points + + For Level 2+: A Level N low pivot is when a Level N-1 pivot low is surrounded + by higher Level N-1 pivot lows (and vice versa for highs) + + Args: + pivot_array: Array of pivot points as [timestamp, price, price, price, price, 0] format + level: Current level being calculated + """ + swings = [] + + if len(pivot_array) < 5: + return swings + + # Use configurable strength for higher levels (more conservative) + strength = min(2 + level, 5) # Level 1: 3 bars, Level 2: 4 bars, Level 3+: 5 bars + + for i in range(strength, len(pivot_array) - strength): + current_price = pivot_array[i, 1] # Use the price from pivot point + current_timestamp = pivot_array[i, 0] + + # Check for swing high (pivot high surrounded by lower pivot highs) + is_swing_high = True + for j in range(i - strength, i + strength + 1): + if j != i and pivot_array[j, 1] >= current_price: + is_swing_high = False + break + + if is_swing_high: + swings.append(SwingPoint( + timestamp=datetime.fromtimestamp(current_timestamp) if current_timestamp > 1e9 else datetime.now(), + price=current_price, + index=i, + swing_type=SwingType.SWING_HIGH, + strength=strength, + volume=0.0 # Pivot points don't have volume + )) + + # Check for swing low (pivot low surrounded by higher pivot lows) + is_swing_low = True + for j in range(i - strength, i + strength + 1): + if j != i and pivot_array[j, 1] <= current_price: + is_swing_low = False + break + + if is_swing_low: + swings.append(SwingPoint( + timestamp=datetime.fromtimestamp(current_timestamp) if current_timestamp > 1e9 else datetime.now(), + price=current_price, + index=i, + swing_type=SwingType.SWING_LOW, + strength=strength, + volume=0.0 # Pivot points don't have volume + )) + + return swings + + def _convert_pivots_to_price_points(self, swing_points: List[SwingPoint]) -> np.ndarray: + """ + Convert swing points to price point array for next level calculation + + Each swing point becomes a "price bar" where OHLC = pivot price + This allows the next level to treat pivot points as individual price data + """ if len(swing_points) < 2: return np.array([]) - ohlcv_data = [] + price_points = [] - for i in range(len(swing_points) - 1): - current_swing = swing_points[i] - next_swing = swing_points[i + 1] - - # Create synthetic OHLCV bar from swing to swing - if current_swing.swing_type == SwingType.SWING_HIGH: - # From high to next point - open_price = current_swing.price - high_price = current_swing.price - low_price = min(current_swing.price, next_swing.price) - close_price = next_swing.price - else: - # From low to next point - open_price = current_swing.price - high_price = max(current_swing.price, next_swing.price) - low_price = current_swing.price - close_price = next_swing.price - - ohlcv_data.append([ - current_swing.timestamp.timestamp(), - open_price, - high_price, - low_price, - close_price, - current_swing.volume + for swing in swing_points: + # Each pivot point becomes a price point where OHLC = pivot price + price_points.append([ + swing.timestamp.timestamp(), + swing.price, # Open = pivot price + swing.price, # High = pivot price + swing.price, # Low = pivot price + swing.price, # Close = pivot price + 0.0 # Volume = 0 (not applicable for pivot points) ]) - return np.array(ohlcv_data) + return np.array(price_points) def _create_empty_structure(self) -> Dict[str, MarketStructureLevel]: """Create empty structure when insufficient data""" diff --git a/web/enhanced_scalping_dashboard.py b/web/enhanced_scalping_dashboard.py index 27a85b7..13354d0 100644 --- a/web/enhanced_scalping_dashboard.py +++ b/web/enhanced_scalping_dashboard.py @@ -1,4 +1,5 @@ # """ +# OBSOLETE AND BROKN. IGNORE THIS FILE FOR NOW. # Enhanced Real-Time Scalping Dashboard with 1s Bar Charts and 15min Tick Cache # Features: diff --git a/web/scalping_dashboard.py b/web/scalping_dashboard.py index 206e68d..2934286 100644 --- a/web/scalping_dashboard.py +++ b/web/scalping_dashboard.py @@ -1,4 +1,6 @@ # """ +# OBSOLETE AND BROKN. IGNORE THIS FILE FOR NOW. + # Ultra-Fast Real-Time Scalping Dashboard (500x Leverage) - Live Data Streaming # Real-time WebSocket streaming dashboard with: