diff --git a/core/data_provider.py b/core/data_provider.py index 4afaac1..fddc3c8 100644 --- a/core/data_provider.py +++ b/core/data_provider.py @@ -1042,10 +1042,10 @@ class DataProvider: def _fetch_from_mexc(self, symbol: str, timeframe: str, limit: int) -> Optional[pd.DataFrame]: """Fetch data from MEXC API (fallback data source when Binance is unavailable)""" try: - # MEXC doesn't support 1s intervals + # For 1s timeframe, generate from WebSocket tick data if timeframe == '1s': - logger.warning(f"MEXC doesn't support 1s intervals, skipping {symbol}") - return None + logger.info(f"Generating 1s candles from WebSocket ticks for {symbol}") + return self._generate_1s_candles_from_ticks(symbol, limit) # Convert symbol format mexc_symbol = symbol.replace('/', '').upper() @@ -1096,11 +1096,105 @@ class DataProvider: logger.error(f"MEXC: Error fetching data: {e}") return None + def _generate_1s_candles_from_ticks(self, symbol: str, limit: int = 1000) -> Optional[pd.DataFrame]: + """Generate 1-second OHLCV candles from WebSocket tick data""" + try: + # Get recent ticks from COB data + recent_ticks = self.get_cob_raw_ticks(symbol, count=limit * 10) # Get more ticks than needed + + if not recent_ticks: + logger.warning(f"No tick data available for {symbol}, cannot generate 1s candles") + return None + + # Group ticks by second and create OHLCV candles + candles = [] + current_second = None + current_candle = None + + for tick in recent_ticks: + # Extract timestamp and price from tick + if isinstance(tick, dict): + timestamp = tick.get('timestamp') + price = tick.get('price', tick.get('mid_price', 0)) + volume = tick.get('volume', 1.0) # Default volume if not available + else: + continue + + if not timestamp or not price or price <= 0: + continue + + # Convert timestamp to datetime if needed + if isinstance(timestamp, (int, float)): + tick_time = datetime.fromtimestamp(timestamp) + elif isinstance(timestamp, datetime): + tick_time = timestamp + else: + continue + + # Round to second + tick_second = tick_time.replace(microsecond=0) + + # Start new candle if second changed + if current_second != tick_second: + # Save previous candle if exists + if current_candle: + candles.append(current_candle) + + # Start new candle + current_second = tick_second + current_candle = { + 'timestamp': tick_second, + 'open': price, + 'high': price, + 'low': price, + 'close': price, + 'volume': volume + } + else: + # Update current candle + current_candle['high'] = max(current_candle['high'], price) + current_candle['low'] = min(current_candle['low'], price) + current_candle['close'] = price + current_candle['volume'] += volume + + # Add final candle + if current_candle: + candles.append(current_candle) + + if not candles: + logger.warning(f"No valid candles generated for {symbol}") + return None + + # Convert to DataFrame + df = pd.DataFrame(candles) + df = df.sort_values('timestamp').reset_index(drop=True) + + # Limit to requested number + if len(df) > limit: + df = df.tail(limit) + + logger.info(f"Generated {len(df)} 1s candles from {len(recent_ticks)} ticks for {symbol}") + return df + + except Exception as e: + logger.error(f"Error generating 1s candles from ticks for {symbol}: {e}") + return None + def _fetch_from_binance(self, symbol: str, timeframe: str, limit: int) -> Optional[pd.DataFrame]: """Fetch data from Binance API with robust rate limiting and error handling""" try: from .api_rate_limiter import get_rate_limiter + # For 1s timeframe, try to generate from WebSocket ticks first + if timeframe == '1s': + logger.info(f"Attempting to generate 1s candles from WebSocket ticks for {symbol}") + generated_df = self._generate_1s_candles_from_ticks(symbol, limit) + if generated_df is not None and not generated_df.empty: + logger.info(f"Successfully generated 1s candles from WebSocket ticks for {symbol}") + return generated_df + else: + logger.warning(f"Failed to generate 1s candles from ticks for {symbol}, trying Binance API") + # Convert symbol format binance_symbol = symbol.replace('/', '').upper() @@ -1174,6 +1268,14 @@ class DataProvider: try: logger.info(f"FALLBACK: Attempting to get real cached data for {symbol} {timeframe}") + # For 1s timeframe, try generating from WebSocket ticks first + if timeframe == '1s': + logger.info(f"FALLBACK: Attempting to generate 1s candles from WebSocket ticks for {symbol}") + generated_data = self._generate_1s_candles_from_ticks(symbol, limit) + if generated_data is not None and not generated_data.empty: + logger.info(f"FALLBACK: Generated 1s candles from WebSocket ticks for {symbol}: {len(generated_data)} bars") + return generated_data + # ONLY try cached data cached_data = self._load_from_cache(symbol, timeframe) if cached_data is not None and not cached_data.empty: diff --git a/data/trading_system.db b/data/trading_system.db index 0b2f733..0aa8b65 100644 Binary files a/data/trading_system.db and b/data/trading_system.db differ