fifo n1 que
This commit is contained in:
Dobromir Popov
@ -16,6 +16,7 @@ import logging
|
||||
import time
|
||||
import threading
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from datetime import datetime, timedelta
|
||||
from typing import Dict, List, Optional, Any, Tuple, Union
|
||||
from dataclasses import dataclass, field
|
||||
@ -178,6 +179,23 @@ class TradingOrchestrator:
|
||||
self.fusion_decisions_count: int = 0
|
||||
self.fusion_training_data: List[Any] = [] # Store training examples for decision model
|
||||
|
||||
# FIFO Data Queues - Ensure consistent data availability across different refresh rates
|
||||
self.data_queues = {
|
||||
'ohlcv_1s': {symbol: deque(maxlen=500) for symbol in [self.symbol] + self.ref_symbols},
|
||||
'ohlcv_1m': {symbol: deque(maxlen=300) for symbol in [self.symbol] + self.ref_symbols},
|
||||
'ohlcv_1h': {symbol: deque(maxlen=300) for symbol in [self.symbol] + self.ref_symbols},
|
||||
'ohlcv_1d': {symbol: deque(maxlen=300) for symbol in [self.symbol] + self.ref_symbols},
|
||||
'technical_indicators': {symbol: deque(maxlen=100) for symbol in [self.symbol] + self.ref_symbols},
|
||||
'cob_data': {symbol: deque(maxlen=50) for symbol in [self.symbol]}, # COB only for primary symbol
|
||||
'model_predictions': {symbol: deque(maxlen=20) for symbol in [self.symbol]}
|
||||
}
|
||||
|
||||
# Data queue locks for thread safety
|
||||
self.data_queue_locks = {
|
||||
data_type: {symbol: threading.Lock() for symbol in queue_dict.keys()}
|
||||
for data_type, queue_dict in self.data_queues.items()
|
||||
}
|
||||
|
||||
# COB Integration - Real-time market microstructure data
|
||||
self.cob_integration = None # Will be set to COBIntegration instance if available
|
||||
self.latest_cob_data: Dict[str, Any] = {} # {symbol: COBSnapshot}
|
||||
@ -241,6 +259,13 @@ class TradingOrchestrator:
|
||||
self.data_provider.start_centralized_data_collection()
|
||||
logger.info("Centralized data collection started - all models and dashboard will receive data")
|
||||
|
||||
# Initialize FIFO data queue integration
|
||||
self._initialize_data_queue_integration()
|
||||
|
||||
# Log initial queue status
|
||||
logger.info("FIFO data queues initialized")
|
||||
self.log_queue_status(detailed=False)
|
||||
|
||||
# Initialize database cleanup task
|
||||
self._schedule_database_cleanup()
|
||||
|
||||
@ -1976,85 +2001,118 @@ class TradingOrchestrator:
|
||||
return 50.0
|
||||
|
||||
async def _get_cnn_predictions(self, model: CNNModelInterface, symbol: str) -> List[Prediction]:
|
||||
"""Get predictions from CNN model for all timeframes with enhanced COB features"""
|
||||
"""Get predictions from CNN model using FIFO queue data"""
|
||||
predictions = []
|
||||
try:
|
||||
timeframes = getattr(self.config, 'timeframes', ['1m','5m','15m','1h'])
|
||||
for timeframe in timeframes:
|
||||
# 1) build or fetch your feature matrix (and optionally augment with COB)…
|
||||
feature_matrix = self.data_provider.get_feature_matrix(
|
||||
symbol=symbol,
|
||||
timeframes=[timeframe],
|
||||
window_size=getattr(model, 'window_size', 20)
|
||||
)
|
||||
if feature_matrix is None:
|
||||
continue
|
||||
|
||||
# …apply COB‐augmentation here (omitted for brevity)—
|
||||
enhanced_features = self._augment_with_cob(feature_matrix, symbol)
|
||||
|
||||
# 2) Initialize these before we call the model
|
||||
action_probs, confidence = None, None
|
||||
|
||||
# 3) Try the actual model inference
|
||||
# Use FIFO queue data instead of direct data provider calls
|
||||
base_data = self.build_base_data_input(symbol)
|
||||
if not base_data:
|
||||
logger.warning(f"Cannot build BaseDataInput for CNN prediction: {symbol}")
|
||||
return predictions
|
||||
|
||||
# Use CNN adapter if available
|
||||
if hasattr(self, 'cnn_adapter') and self.cnn_adapter:
|
||||
try:
|
||||
# if your model has an .act() that returns (probs, conf)
|
||||
if hasattr(model.model, 'act'):
|
||||
# Flatten / reshape enhanced_features as needed…
|
||||
x = self._prepare_cnn_input(enhanced_features)
|
||||
result = self.cnn_adapter.predict(base_data)
|
||||
if result:
|
||||
prediction = Prediction(
|
||||
action=result.action,
|
||||
confidence=result.confidence,
|
||||
probabilities=result.predictions,
|
||||
timeframe="multi", # Multi-timeframe prediction
|
||||
timestamp=datetime.now(),
|
||||
model_name="enhanced_cnn",
|
||||
metadata={
|
||||
'feature_size': len(base_data.get_feature_vector()),
|
||||
'data_sources': ['ohlcv_1s', 'ohlcv_1m', 'ohlcv_1h', 'ohlcv_1d', 'btc', 'cob', 'indicators']
|
||||
}
|
||||
)
|
||||
predictions.append(prediction)
|
||||
|
||||
# Debugging: Print the type and content of x before passing to act()
|
||||
logger.debug(f"CNN input (x) type: {type(x)}, shape: {x.shape}, content sample: {x.flatten()[:5]}...")
|
||||
|
||||
action_idx, confidence, action_probs = model.model.act(x, explore=False)
|
||||
# Store prediction in queue for future use
|
||||
self.update_data_queue('model_predictions', symbol, result)
|
||||
|
||||
# Debugging: Print the type and content of the unpacked values
|
||||
logger.debug(f"CNN act() returned: action_idx={action_idx} (type={type(action_idx)}), confidence={confidence} (type={type(confidence)}), action_probs={action_probs[:5]}... (type={type(action_probs)})")
|
||||
else:
|
||||
# fallback to generic predict
|
||||
result = model.predict(enhanced_features)
|
||||
if isinstance(result, tuple) and len(result)==2:
|
||||
action_probs, confidence = result
|
||||
else:
|
||||
action_probs = result
|
||||
confidence = 0.7
|
||||
except Exception as e:
|
||||
logger.warning(f"CNN inference failed for {symbol}@{timeframe}: {e}")
|
||||
continue # skip this timeframe entirely
|
||||
|
||||
# 4) If we still don't have valid probs, skip
|
||||
if action_probs is None:
|
||||
continue
|
||||
|
||||
# 5) Build your Prediction
|
||||
action_names = ['SELL','HOLD','BUY']
|
||||
best_idx = int(np.argmax(action_probs))
|
||||
best_action = action_names[best_idx]
|
||||
pred = Prediction(
|
||||
action=best_action,
|
||||
confidence=float(confidence),
|
||||
probabilities={n: float(p) for n,p in zip(action_names, action_probs)},
|
||||
timeframe=timeframe,
|
||||
timestamp=datetime.now(),
|
||||
model_name=model.name,
|
||||
metadata={
|
||||
'feature_shape': str(enhanced_features.shape),
|
||||
'cob_enhanced': enhanced_features is not feature_matrix
|
||||
}
|
||||
)
|
||||
predictions.append(pred)
|
||||
|
||||
# …and capture for the dashboard if you like…
|
||||
current_price = self._get_current_price(symbol)
|
||||
if current_price is not None:
|
||||
predicted_price = current_price * (1 + (0.01 * (confidence if best_action=='BUY' else -confidence if best_action=='SELL' else 0)))
|
||||
self.capture_cnn_prediction(
|
||||
symbol,
|
||||
direction=best_idx,
|
||||
confidence=confidence,
|
||||
current_price=current_price,
|
||||
predicted_price=predicted_price
|
||||
logger.error(f"Error using CNN adapter: {e}")
|
||||
|
||||
# Fallback to legacy CNN prediction if adapter fails
|
||||
if not predictions:
|
||||
timeframes = getattr(self.config, 'timeframes', ['1m','5m','15m','1h'])
|
||||
for timeframe in timeframes:
|
||||
# 1) build or fetch your feature matrix (and optionally augment with COB)…
|
||||
feature_matrix = self.data_provider.get_feature_matrix(
|
||||
symbol=symbol,
|
||||
timeframes=[timeframe],
|
||||
window_size=getattr(model, 'window_size', 20)
|
||||
)
|
||||
if feature_matrix is None:
|
||||
continue
|
||||
|
||||
# …apply COB‐augmentation here (omitted for brevity)—
|
||||
enhanced_features = self._augment_with_cob(feature_matrix, symbol)
|
||||
|
||||
# 2) Initialize these before we call the model
|
||||
action_probs, confidence = None, None
|
||||
|
||||
# 3) Try the actual model inference
|
||||
try:
|
||||
# if your model has an .act() that returns (probs, conf)
|
||||
if hasattr(model.model, 'act'):
|
||||
# Flatten / reshape enhanced_features as needed…
|
||||
x = self._prepare_cnn_input(enhanced_features)
|
||||
|
||||
# Debugging: Print the type and content of x before passing to act()
|
||||
logger.debug(f"CNN input (x) type: {type(x)}, shape: {x.shape}, content sample: {x.flatten()[:5]}...")
|
||||
|
||||
action_idx, confidence, action_probs = model.model.act(x, explore=False)
|
||||
|
||||
# Debugging: Print the type and content of the unpacked values
|
||||
logger.debug(f"CNN act() returned: action_idx={action_idx} (type={type(action_idx)}), confidence={confidence} (type={type(confidence)}), action_probs={action_probs[:5]}... (type={type(action_probs)})")
|
||||
else:
|
||||
# fallback to generic predict
|
||||
result = model.predict(enhanced_features)
|
||||
if isinstance(result, tuple) and len(result)==2:
|
||||
action_probs, confidence = result
|
||||
else:
|
||||
action_probs = result
|
||||
confidence = 0.7
|
||||
except Exception as e:
|
||||
logger.warning(f"CNN inference failed for {symbol}@{timeframe}: {e}")
|
||||
continue # skip this timeframe entirely
|
||||
|
||||
# 4) If we still don't have valid probs, skip
|
||||
if action_probs is None:
|
||||
continue
|
||||
|
||||
# 5) Build your Prediction
|
||||
action_names = ['SELL','HOLD','BUY']
|
||||
best_idx = int(np.argmax(action_probs))
|
||||
best_action = action_names[best_idx]
|
||||
pred = Prediction(
|
||||
action=best_action,
|
||||
confidence=float(confidence),
|
||||
probabilities={n: float(p) for n,p in zip(action_names, action_probs)},
|
||||
timeframe=timeframe,
|
||||
timestamp=datetime.now(),
|
||||
model_name=model.name,
|
||||
metadata={
|
||||
'feature_shape': str(enhanced_features.shape),
|
||||
'cob_enhanced': enhanced_features is not feature_matrix
|
||||
}
|
||||
)
|
||||
predictions.append(pred)
|
||||
|
||||
# …and capture for the dashboard if you like…
|
||||
current_price = self._get_current_price(symbol)
|
||||
if current_price is not None:
|
||||
predicted_price = current_price * (1 + (0.01 * (confidence if best_action=='BUY' else -confidence if best_action=='SELL' else 0)))
|
||||
self.capture_cnn_prediction(
|
||||
symbol,
|
||||
direction=best_idx,
|
||||
confidence=confidence,
|
||||
current_price=current_price,
|
||||
predicted_price=predicted_price
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"Orch: Error getting CNN predictions: {e}")
|
||||
return predictions
|
||||
@ -2073,8 +2131,17 @@ class TradingOrchestrator:
|
||||
arr = arr[:300]
|
||||
return arr.reshape(1,-1)
|
||||
async def _get_rl_prediction(self, model: RLAgentInterface, symbol: str) -> Optional[Prediction]:
|
||||
"""Get prediction from RL agent"""
|
||||
"""Get prediction from RL agent using FIFO queue data"""
|
||||
try:
|
||||
# Use FIFO queue data to build consistent state
|
||||
base_data = self.build_base_data_input(symbol)
|
||||
if not base_data:
|
||||
logger.warning(f"Cannot build BaseDataInput for RL prediction: {symbol}")
|
||||
return None
|
||||
|
||||
# Convert BaseDataInput to RL state format
|
||||
state_features = base_data.get_feature_vector()
|
||||
|
||||
# Get current state for RL agent
|
||||
state = self._get_rl_state(symbol)
|
||||
if state is None:
|
||||
@ -3630,4 +3697,661 @@ class TradingOrchestrator:
|
||||
|
||||
This is much faster than loading the entire checkpoint just to get metadata
|
||||
"""
|
||||
return self.db_manager.get_best_checkpoint_metadata(model_name)
|
||||
return self.db_manager.get_best_checkpoint_metadata(model_name)
|
||||
|
||||
# === FIFO DATA QUEUE MANAGEMENT ===
|
||||
|
||||
def update_data_queue(self, data_type: str, symbol: str, data: Any) -> bool:
|
||||
"""
|
||||
Update FIFO data queue with new data
|
||||
|
||||
Args:
|
||||
data_type: Type of data ('ohlcv_1s', 'ohlcv_1m', etc.)
|
||||
symbol: Trading symbol
|
||||
data: New data to add
|
||||
|
||||
Returns:
|
||||
bool: True if successful
|
||||
"""
|
||||
try:
|
||||
if data_type not in self.data_queues:
|
||||
logger.warning(f"Unknown data type: {data_type}")
|
||||
return False
|
||||
|
||||
if symbol not in self.data_queues[data_type]:
|
||||
logger.warning(f"Unknown symbol for {data_type}: {symbol}")
|
||||
return False
|
||||
|
||||
# Thread-safe queue update
|
||||
with self.data_queue_locks[data_type][symbol]:
|
||||
self.data_queues[data_type][symbol].append(data)
|
||||
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error updating data queue {data_type}/{symbol}: {e}")
|
||||
return False
|
||||
|
||||
def get_latest_data(self, data_type: str, symbol: str, count: int = 1) -> List[Any]:
|
||||
"""
|
||||
Get latest data from FIFO queue
|
||||
|
||||
Args:
|
||||
data_type: Type of data
|
||||
symbol: Trading symbol
|
||||
count: Number of latest items to retrieve
|
||||
|
||||
Returns:
|
||||
List of latest data items
|
||||
"""
|
||||
try:
|
||||
if data_type not in self.data_queues or symbol not in self.data_queues[data_type]:
|
||||
return []
|
||||
|
||||
with self.data_queue_locks[data_type][symbol]:
|
||||
queue = self.data_queues[data_type][symbol]
|
||||
if len(queue) == 0:
|
||||
return []
|
||||
|
||||
# Get last 'count' items
|
||||
return list(queue)[-count:] if count > 1 else [queue[-1]]
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting latest data {data_type}/{symbol}: {e}")
|
||||
return []
|
||||
|
||||
def get_queue_data(self, data_type: str, symbol: str, max_items: int = None) -> List[Any]:
|
||||
"""
|
||||
Get all data from FIFO queue
|
||||
|
||||
Args:
|
||||
data_type: Type of data
|
||||
symbol: Trading symbol
|
||||
max_items: Maximum number of items to return (None for all)
|
||||
|
||||
Returns:
|
||||
List of data items
|
||||
"""
|
||||
try:
|
||||
if data_type not in self.data_queues or symbol not in self.data_queues[data_type]:
|
||||
return []
|
||||
|
||||
with self.data_queue_locks[data_type][symbol]:
|
||||
queue = self.data_queues[data_type][symbol]
|
||||
data_list = list(queue)
|
||||
|
||||
if max_items and len(data_list) > max_items:
|
||||
return data_list[-max_items:]
|
||||
|
||||
return data_list
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting queue data {data_type}/{symbol}: {e}")
|
||||
return []
|
||||
|
||||
def get_queue_status(self) -> Dict[str, Dict[str, int]]:
|
||||
"""Get status of all data queues"""
|
||||
status = {}
|
||||
|
||||
for data_type, symbol_queues in self.data_queues.items():
|
||||
status[data_type] = {}
|
||||
for symbol, queue in symbol_queues.items():
|
||||
with self.data_queue_locks[data_type][symbol]:
|
||||
status[data_type][symbol] = len(queue)
|
||||
|
||||
return status
|
||||
|
||||
def get_detailed_queue_status(self) -> Dict[str, Any]:
|
||||
"""Get detailed status of all data queues with timestamps and data info"""
|
||||
detailed_status = {}
|
||||
|
||||
for data_type, symbol_queues in self.data_queues.items():
|
||||
detailed_status[data_type] = {}
|
||||
for symbol, queue in symbol_queues.items():
|
||||
with self.data_queue_locks[data_type][symbol]:
|
||||
queue_list = list(queue)
|
||||
queue_info = {
|
||||
'count': len(queue_list),
|
||||
'max_size': queue.maxlen,
|
||||
'usage_percent': (len(queue_list) / queue.maxlen * 100) if queue.maxlen else 0,
|
||||
'oldest_timestamp': None,
|
||||
'newest_timestamp': None,
|
||||
'data_type_info': None
|
||||
}
|
||||
|
||||
if queue_list:
|
||||
# Try to get timestamps from data
|
||||
try:
|
||||
if hasattr(queue_list[0], 'timestamp'):
|
||||
queue_info['oldest_timestamp'] = queue_list[0].timestamp.isoformat()
|
||||
queue_info['newest_timestamp'] = queue_list[-1].timestamp.isoformat()
|
||||
|
||||
# Add data type specific info
|
||||
if data_type.startswith('ohlcv_'):
|
||||
if hasattr(queue_list[-1], 'close'):
|
||||
queue_info['data_type_info'] = f"latest_price={queue_list[-1].close:.2f}"
|
||||
elif data_type == 'technical_indicators':
|
||||
if isinstance(queue_list[-1], dict):
|
||||
indicators = list(queue_list[-1].keys())[:3] # First 3 indicators
|
||||
queue_info['data_type_info'] = f"indicators={indicators}"
|
||||
elif data_type == 'cob_data':
|
||||
queue_info['data_type_info'] = "cob_snapshot"
|
||||
elif data_type == 'model_predictions':
|
||||
if hasattr(queue_list[-1], 'action'):
|
||||
queue_info['data_type_info'] = f"latest_action={queue_list[-1].action}"
|
||||
except Exception as e:
|
||||
queue_info['data_type_info'] = f"error_getting_info: {e}"
|
||||
|
||||
detailed_status[data_type][symbol] = queue_info
|
||||
|
||||
return detailed_status
|
||||
|
||||
def log_queue_status(self, detailed: bool = False):
|
||||
"""Log current queue status for debugging"""
|
||||
if detailed:
|
||||
status = self.get_detailed_queue_status()
|
||||
logger.info("=== Detailed Queue Status ===")
|
||||
for data_type, symbols in status.items():
|
||||
logger.info(f"{data_type}:")
|
||||
for symbol, info in symbols.items():
|
||||
logger.info(f" {symbol}: {info['count']}/{info['max_size']} ({info['usage_percent']:.1f}%) - {info.get('data_type_info', 'no_info')}")
|
||||
else:
|
||||
status = self.get_queue_status()
|
||||
logger.info("=== Queue Status ===")
|
||||
for data_type, symbols in status.items():
|
||||
symbol_counts = [f"{symbol}:{count}" for symbol, count in symbols.items()]
|
||||
logger.info(f"{data_type}: {', '.join(symbol_counts)}")
|
||||
|
||||
def ensure_minimum_data(self, data_type: str, symbol: str, min_count: int) -> bool:
|
||||
"""
|
||||
Check if queue has minimum required data
|
||||
|
||||
Args:
|
||||
data_type: Type of data
|
||||
symbol: Trading symbol
|
||||
min_count: Minimum required items
|
||||
|
||||
Returns:
|
||||
bool: True if minimum data available
|
||||
"""
|
||||
try:
|
||||
if data_type not in self.data_queues or symbol not in self.data_queues[data_type]:
|
||||
return False
|
||||
|
||||
with self.data_queue_locks[data_type][symbol]:
|
||||
return len(self.data_queues[data_type][symbol]) >= min_count
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error checking minimum data {data_type}/{symbol}: {e}")
|
||||
return False
|
||||
|
||||
def build_base_data_input(self, symbol: str) -> Optional[Any]:
|
||||
"""
|
||||
Build BaseDataInput from FIFO queues with consistent data
|
||||
|
||||
Args:
|
||||
symbol: Trading symbol
|
||||
|
||||
Returns:
|
||||
BaseDataInput with consistent data structure
|
||||
"""
|
||||
try:
|
||||
from core.data_models import BaseDataInput
|
||||
|
||||
# Check minimum data requirements
|
||||
min_requirements = {
|
||||
'ohlcv_1s': 100,
|
||||
'ohlcv_1m': 50,
|
||||
'ohlcv_1h': 20,
|
||||
'ohlcv_1d': 10
|
||||
}
|
||||
|
||||
# Verify we have minimum data for all timeframes with fallback strategy
|
||||
missing_data = []
|
||||
for data_type, min_count in min_requirements.items():
|
||||
if not self.ensure_minimum_data(data_type, symbol, min_count):
|
||||
# Get actual count for better logging
|
||||
actual_count = 0
|
||||
if data_type in self.data_queues and symbol in self.data_queues[data_type]:
|
||||
with self.data_queue_locks[data_type][symbol]:
|
||||
actual_count = len(self.data_queues[data_type][symbol])
|
||||
|
||||
missing_data.append((data_type, actual_count, min_count))
|
||||
|
||||
# If we're missing critical 1s data, try to use 1m data as fallback
|
||||
if missing_data:
|
||||
critical_missing = [d for d in missing_data if d[0] in ['ohlcv_1s', 'ohlcv_1h']]
|
||||
if critical_missing:
|
||||
logger.warning(f"Missing critical data for {symbol}: {critical_missing}")
|
||||
|
||||
# Try fallback strategy: use available data with padding
|
||||
if self._try_fallback_data_strategy(symbol, missing_data):
|
||||
logger.info(f"Successfully applied fallback data strategy for {symbol}")
|
||||
else:
|
||||
for data_type, actual_count, min_count in missing_data:
|
||||
logger.warning(f"Insufficient {data_type} data for {symbol}: have {actual_count}, need {min_count}")
|
||||
return None
|
||||
|
||||
# Get BTC data (reference symbol)
|
||||
btc_symbol = 'BTC/USDT'
|
||||
if not self.ensure_minimum_data('ohlcv_1s', btc_symbol, 100):
|
||||
# Get actual BTC data count for logging
|
||||
btc_count = 0
|
||||
if 'ohlcv_1s' in self.data_queues and btc_symbol in self.data_queues['ohlcv_1s']:
|
||||
with self.data_queue_locks['ohlcv_1s'][btc_symbol]:
|
||||
btc_count = len(self.data_queues['ohlcv_1s'][btc_symbol])
|
||||
|
||||
logger.warning(f"Insufficient BTC data for reference: have {btc_count}, need 100, using ETH data as fallback")
|
||||
# Use ETH data as fallback
|
||||
btc_data = self.get_queue_data('ohlcv_1s', symbol, 300)
|
||||
else:
|
||||
btc_data = self.get_queue_data('ohlcv_1s', btc_symbol, 300)
|
||||
|
||||
# Build BaseDataInput with queue data
|
||||
base_data = BaseDataInput(
|
||||
symbol=symbol,
|
||||
timestamp=datetime.now(),
|
||||
ohlcv_1s=self.get_queue_data('ohlcv_1s', symbol, 300),
|
||||
ohlcv_1m=self.get_queue_data('ohlcv_1m', symbol, 300),
|
||||
ohlcv_1h=self.get_queue_data('ohlcv_1h', symbol, 300),
|
||||
ohlcv_1d=self.get_queue_data('ohlcv_1d', symbol, 300),
|
||||
btc_ohlcv_1s=btc_data,
|
||||
technical_indicators=self._get_latest_indicators(symbol),
|
||||
cob_data=self._get_latest_cob_data(symbol),
|
||||
last_predictions=self._get_recent_model_predictions(symbol)
|
||||
)
|
||||
|
||||
# Validate the data
|
||||
if not base_data.validate():
|
||||
logger.warning(f"BaseDataInput validation failed for {symbol}")
|
||||
return None
|
||||
|
||||
return base_data
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error building BaseDataInput for {symbol}: {e}")
|
||||
return None
|
||||
|
||||
def _get_latest_indicators(self, symbol: str) -> Dict[str, float]:
|
||||
"""Get latest technical indicators from queue"""
|
||||
try:
|
||||
indicators_data = self.get_latest_data('technical_indicators', symbol, 1)
|
||||
if indicators_data:
|
||||
return indicators_data[0]
|
||||
return {}
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting indicators for {symbol}: {e}")
|
||||
return {}
|
||||
|
||||
def _get_latest_cob_data(self, symbol: str) -> Optional[Any]:
|
||||
"""Get latest COB data from queue"""
|
||||
try:
|
||||
cob_data = self.get_latest_data('cob_data', symbol, 1)
|
||||
if cob_data:
|
||||
return cob_data[0]
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting COB data for {symbol}: {e}")
|
||||
return None
|
||||
|
||||
def _get_recent_model_predictions(self, symbol: str) -> Dict[str, Any]:
|
||||
"""Get recent model predictions from queue"""
|
||||
try:
|
||||
predictions_data = self.get_latest_data('model_predictions', symbol, 5)
|
||||
|
||||
# Convert to dict format expected by BaseDataInput
|
||||
predictions_dict = {}
|
||||
for i, pred in enumerate(predictions_data):
|
||||
predictions_dict[f"model_{i}"] = pred
|
||||
|
||||
return predictions_dict
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting model predictions for {symbol}: {e}")
|
||||
return {}
|
||||
|
||||
def _initialize_data_queue_integration(self):
|
||||
"""Initialize integration between data provider and FIFO queues"""
|
||||
try:
|
||||
# Register callbacks with data provider to populate FIFO queues
|
||||
if hasattr(self.data_provider, 'register_data_callback'):
|
||||
# Register for different data types
|
||||
self.data_provider.register_data_callback('ohlcv', self._on_ohlcv_data)
|
||||
self.data_provider.register_data_callback('technical_indicators', self._on_indicators_data)
|
||||
self.data_provider.register_data_callback('cob', self._on_cob_data)
|
||||
logger.info("Data provider callbacks registered for FIFO queues")
|
||||
else:
|
||||
# Fallback: Start a background thread to poll data
|
||||
self._start_data_polling_thread()
|
||||
logger.info("Started data polling thread for FIFO queues")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error initializing data queue integration: {e}")
|
||||
|
||||
def _on_ohlcv_data(self, symbol: str, timeframe: str, data: Any):
|
||||
"""Callback for new OHLCV data"""
|
||||
try:
|
||||
data_type = f'ohlcv_{timeframe}'
|
||||
if data_type in self.data_queues and symbol in self.data_queues[data_type]:
|
||||
self.update_data_queue(data_type, symbol, data)
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing OHLCV data callback: {e}")
|
||||
|
||||
def _on_indicators_data(self, symbol: str, indicators: Dict[str, float]):
|
||||
"""Callback for new technical indicators"""
|
||||
try:
|
||||
self.update_data_queue('technical_indicators', symbol, indicators)
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing indicators data callback: {e}")
|
||||
|
||||
def _on_cob_data(self, symbol: str, cob_data: Any):
|
||||
"""Callback for new COB data"""
|
||||
try:
|
||||
self.update_data_queue('cob_data', symbol, cob_data)
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing COB data callback: {e}")
|
||||
|
||||
def _start_data_polling_thread(self):
|
||||
"""Start background thread to poll data and populate queues"""
|
||||
def data_polling_worker():
|
||||
"""Background worker to poll data and update queues"""
|
||||
poll_count = 0
|
||||
while self.running:
|
||||
try:
|
||||
poll_count += 1
|
||||
|
||||
# Log polling activity every 30 seconds
|
||||
if poll_count % 30 == 1:
|
||||
logger.info(f"Data polling cycle #{poll_count} - checking data sources")
|
||||
# Poll OHLCV data for all symbols and timeframes
|
||||
for symbol in [self.symbol] + self.ref_symbols:
|
||||
for timeframe in ['1s', '1m', '1h', '1d']:
|
||||
try:
|
||||
# Get latest data from data provider using correct method
|
||||
if hasattr(self.data_provider, 'get_latest_candles'):
|
||||
df = self.data_provider.get_latest_candles(symbol, timeframe, limit=1)
|
||||
if df is not None and not df.empty:
|
||||
# Convert DataFrame row to OHLCVBar
|
||||
latest_row = df.iloc[-1]
|
||||
from core.data_models import OHLCVBar
|
||||
ohlcv_bar = OHLCVBar(
|
||||
symbol=symbol,
|
||||
timestamp=latest_row.name if hasattr(latest_row.name, 'to_pydatetime') else datetime.now(),
|
||||
open=float(latest_row['open']),
|
||||
high=float(latest_row['high']),
|
||||
low=float(latest_row['low']),
|
||||
close=float(latest_row['close']),
|
||||
volume=float(latest_row['volume']),
|
||||
timeframe=timeframe
|
||||
)
|
||||
self.update_data_queue(f'ohlcv_{timeframe}', symbol, ohlcv_bar)
|
||||
elif hasattr(self.data_provider, 'get_historical_data'):
|
||||
df = self.data_provider.get_historical_data(symbol, timeframe, limit=1)
|
||||
if df is not None and not df.empty:
|
||||
# Convert DataFrame row to OHLCVBar
|
||||
latest_row = df.iloc[-1]
|
||||
from core.data_models import OHLCVBar
|
||||
ohlcv_bar = OHLCVBar(
|
||||
symbol=symbol,
|
||||
timestamp=latest_row.name if hasattr(latest_row.name, 'to_pydatetime') else datetime.now(),
|
||||
open=float(latest_row['open']),
|
||||
high=float(latest_row['high']),
|
||||
low=float(latest_row['low']),
|
||||
close=float(latest_row['close']),
|
||||
volume=float(latest_row['volume']),
|
||||
timeframe=timeframe
|
||||
)
|
||||
self.update_data_queue(f'ohlcv_{timeframe}', symbol, ohlcv_bar)
|
||||
except Exception as e:
|
||||
logger.debug(f"Error polling {symbol} {timeframe}: {e}")
|
||||
|
||||
# Poll technical indicators
|
||||
for symbol in [self.symbol] + self.ref_symbols:
|
||||
try:
|
||||
# Get recent data and calculate basic indicators
|
||||
df = None
|
||||
if hasattr(self.data_provider, 'get_latest_candles'):
|
||||
df = self.data_provider.get_latest_candles(symbol, '1m', limit=50)
|
||||
elif hasattr(self.data_provider, 'get_historical_data'):
|
||||
df = self.data_provider.get_historical_data(symbol, '1m', limit=50)
|
||||
|
||||
if df is not None and not df.empty and len(df) >= 20:
|
||||
# Calculate basic technical indicators
|
||||
indicators = {}
|
||||
try:
|
||||
import ta
|
||||
indicators['rsi'] = ta.momentum.RSIIndicator(df['close']).rsi().iloc[-1]
|
||||
indicators['sma_20'] = df['close'].rolling(20).mean().iloc[-1]
|
||||
indicators['ema_12'] = df['close'].ewm(span=12).mean().iloc[-1]
|
||||
indicators['ema_26'] = df['close'].ewm(span=26).mean().iloc[-1]
|
||||
indicators['macd'] = indicators['ema_12'] - indicators['ema_26']
|
||||
|
||||
# Remove NaN values
|
||||
indicators = {k: float(v) for k, v in indicators.items() if not pd.isna(v)}
|
||||
|
||||
if indicators:
|
||||
self.update_data_queue('technical_indicators', symbol, indicators)
|
||||
except Exception as ta_e:
|
||||
logger.debug(f"Error calculating indicators for {symbol}: {ta_e}")
|
||||
except Exception as e:
|
||||
logger.debug(f"Error polling indicators for {symbol}: {e}")
|
||||
|
||||
# Poll COB data (primary symbol only)
|
||||
try:
|
||||
if hasattr(self.data_provider, 'get_latest_cob_data'):
|
||||
cob_data = self.data_provider.get_latest_cob_data(self.symbol)
|
||||
if cob_data and isinstance(cob_data, dict) and cob_data:
|
||||
self.update_data_queue('cob_data', self.symbol, cob_data)
|
||||
except Exception as e:
|
||||
logger.debug(f"Error polling COB data: {e}")
|
||||
|
||||
# Sleep between polls
|
||||
time.sleep(1) # Poll every second
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in data polling worker: {e}")
|
||||
time.sleep(5) # Wait longer on error
|
||||
|
||||
# Start the polling thread
|
||||
self.data_polling_thread = threading.Thread(target=data_polling_worker, daemon=True)
|
||||
self.data_polling_thread.start()
|
||||
logger.info("Data polling thread started")
|
||||
|
||||
# Populate initial data
|
||||
self._populate_initial_queue_data()
|
||||
|
||||
def _populate_initial_queue_data(self):
|
||||
"""Populate FIFO queues with initial historical data"""
|
||||
try:
|
||||
logger.info("Populating FIFO queues with initial data...")
|
||||
|
||||
# Get initial OHLCV data for all symbols and timeframes
|
||||
for symbol in [self.symbol] + self.ref_symbols:
|
||||
for timeframe in ['1s', '1m', '1h', '1d']:
|
||||
try:
|
||||
# Determine how much data to fetch based on timeframe
|
||||
limits = {'1s': 500, '1m': 300, '1h': 300, '1d': 300}
|
||||
limit = limits.get(timeframe, 300)
|
||||
|
||||
# Get historical data
|
||||
df = None
|
||||
if hasattr(self.data_provider, 'get_historical_data'):
|
||||
df = self.data_provider.get_historical_data(symbol, timeframe, limit=limit)
|
||||
|
||||
if df is not None and not df.empty:
|
||||
logger.info(f"Loading {len(df)} {timeframe} bars for {symbol}")
|
||||
|
||||
# Convert DataFrame to OHLCVBar objects and add to queue
|
||||
from core.data_models import OHLCVBar
|
||||
for idx, row in df.iterrows():
|
||||
try:
|
||||
ohlcv_bar = OHLCVBar(
|
||||
symbol=symbol,
|
||||
timestamp=idx if hasattr(idx, 'to_pydatetime') else datetime.now(),
|
||||
open=float(row['open']),
|
||||
high=float(row['high']),
|
||||
low=float(row['low']),
|
||||
close=float(row['close']),
|
||||
volume=float(row['volume']),
|
||||
timeframe=timeframe
|
||||
)
|
||||
self.update_data_queue(f'ohlcv_{timeframe}', symbol, ohlcv_bar)
|
||||
except Exception as bar_e:
|
||||
logger.debug(f"Error creating OHLCV bar: {bar_e}")
|
||||
else:
|
||||
logger.warning(f"No historical data available for {symbol} {timeframe}")
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Error loading initial data for {symbol} {timeframe}: {e}")
|
||||
|
||||
# Calculate and populate technical indicators
|
||||
logger.info("Calculating technical indicators...")
|
||||
for symbol in [self.symbol] + self.ref_symbols:
|
||||
try:
|
||||
# Use 1m data to calculate indicators
|
||||
if self.ensure_minimum_data('ohlcv_1m', symbol, 50):
|
||||
minute_data = self.get_queue_data('ohlcv_1m', symbol, 100)
|
||||
if minute_data and len(minute_data) >= 20:
|
||||
# Convert to DataFrame for indicator calculation
|
||||
df_data = []
|
||||
for bar in minute_data:
|
||||
df_data.append({
|
||||
'timestamp': bar.timestamp,
|
||||
'open': bar.open,
|
||||
'high': bar.high,
|
||||
'low': bar.low,
|
||||
'close': bar.close,
|
||||
'volume': bar.volume
|
||||
})
|
||||
|
||||
df = pd.DataFrame(df_data)
|
||||
df.set_index('timestamp', inplace=True)
|
||||
|
||||
# Calculate indicators
|
||||
indicators = {}
|
||||
try:
|
||||
import ta
|
||||
if len(df) >= 14:
|
||||
indicators['rsi'] = ta.momentum.RSIIndicator(df['close']).rsi().iloc[-1]
|
||||
if len(df) >= 20:
|
||||
indicators['sma_20'] = df['close'].rolling(20).mean().iloc[-1]
|
||||
if len(df) >= 12:
|
||||
indicators['ema_12'] = df['close'].ewm(span=12).mean().iloc[-1]
|
||||
if len(df) >= 26:
|
||||
indicators['ema_26'] = df['close'].ewm(span=26).mean().iloc[-1]
|
||||
if 'ema_12' in indicators:
|
||||
indicators['macd'] = indicators['ema_12'] - indicators['ema_26']
|
||||
|
||||
# Bollinger Bands
|
||||
if len(df) >= 20:
|
||||
bb_period = 20
|
||||
bb_std = 2
|
||||
sma = df['close'].rolling(bb_period).mean()
|
||||
std = df['close'].rolling(bb_period).std()
|
||||
indicators['bb_upper'] = (sma + (std * bb_std)).iloc[-1]
|
||||
indicators['bb_lower'] = (sma - (std * bb_std)).iloc[-1]
|
||||
indicators['bb_middle'] = sma.iloc[-1]
|
||||
|
||||
# Remove NaN values
|
||||
indicators = {k: float(v) for k, v in indicators.items() if not pd.isna(v)}
|
||||
|
||||
if indicators:
|
||||
self.update_data_queue('technical_indicators', symbol, indicators)
|
||||
logger.info(f"Calculated {len(indicators)} indicators for {symbol}")
|
||||
|
||||
except Exception as ta_e:
|
||||
logger.warning(f"Error calculating indicators for {symbol}: {ta_e}")
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Error processing indicators for {symbol}: {e}")
|
||||
|
||||
# Log final queue status
|
||||
logger.info("Initial data population completed")
|
||||
self.log_queue_status(detailed=True)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error populating initial queue data: {e}")
|
||||
|
||||
def _try_fallback_data_strategy(self, symbol: str, missing_data: List[Tuple[str, int, int]]) -> bool:
|
||||
"""
|
||||
Try to fill missing data using fallback strategies
|
||||
|
||||
Args:
|
||||
symbol: Trading symbol
|
||||
missing_data: List of (data_type, actual_count, min_count) tuples
|
||||
|
||||
Returns:
|
||||
bool: True if fallback successful
|
||||
"""
|
||||
try:
|
||||
from core.data_models import OHLCVBar
|
||||
|
||||
for data_type, actual_count, min_count in missing_data:
|
||||
needed_count = min_count - actual_count
|
||||
|
||||
if data_type == 'ohlcv_1s' and needed_count > 0:
|
||||
# Try to use 1m data to generate 1s data (simple interpolation)
|
||||
if self.ensure_minimum_data('ohlcv_1m', symbol, 10):
|
||||
logger.info(f"Using 1m data to generate {needed_count} 1s bars for {symbol}")
|
||||
|
||||
# Get some 1m data
|
||||
minute_data = self.get_queue_data('ohlcv_1m', symbol, 10)
|
||||
if minute_data:
|
||||
# Generate synthetic 1s bars from 1m data
|
||||
for i, minute_bar in enumerate(minute_data[-5:]): # Use last 5 minutes
|
||||
# Create 60 synthetic 1s bars from each 1m bar
|
||||
for second in range(60):
|
||||
if len(self.data_queues['ohlcv_1s'][symbol]) >= min_count:
|
||||
break
|
||||
|
||||
# Simple interpolation (not perfect but functional)
|
||||
synthetic_bar = OHLCVBar(
|
||||
symbol=symbol,
|
||||
timestamp=minute_bar.timestamp,
|
||||
open=minute_bar.open,
|
||||
high=minute_bar.high,
|
||||
low=minute_bar.low,
|
||||
close=minute_bar.close,
|
||||
volume=minute_bar.volume / 60, # Distribute volume
|
||||
timeframe='1s'
|
||||
)
|
||||
self.update_data_queue('ohlcv_1s', symbol, synthetic_bar)
|
||||
|
||||
elif data_type == 'ohlcv_1h' and needed_count > 0:
|
||||
# Try to use 1m data to generate 1h data
|
||||
if self.ensure_minimum_data('ohlcv_1m', symbol, 60):
|
||||
logger.info(f"Using 1m data to generate {needed_count} 1h bars for {symbol}")
|
||||
|
||||
minute_data = self.get_queue_data('ohlcv_1m', symbol, 300)
|
||||
if minute_data and len(minute_data) >= 60:
|
||||
# Group 1m bars into 1h bars
|
||||
for hour_start in range(0, len(minute_data) - 60, 60):
|
||||
if len(self.data_queues['ohlcv_1h'][symbol]) >= min_count:
|
||||
break
|
||||
|
||||
hour_bars = minute_data[hour_start:hour_start + 60]
|
||||
if len(hour_bars) == 60:
|
||||
# Aggregate 1m bars into 1h bar
|
||||
hour_bar = OHLCVBar(
|
||||
symbol=symbol,
|
||||
timestamp=hour_bars[0].timestamp,
|
||||
open=hour_bars[0].open,
|
||||
high=max(bar.high for bar in hour_bars),
|
||||
low=min(bar.low for bar in hour_bars),
|
||||
close=hour_bars[-1].close,
|
||||
volume=sum(bar.volume for bar in hour_bars),
|
||||
timeframe='1h'
|
||||
)
|
||||
self.update_data_queue('ohlcv_1h', symbol, hour_bar)
|
||||
|
||||
# Check if we now have minimum data
|
||||
all_satisfied = True
|
||||
for data_type, _, min_count in missing_data:
|
||||
if not self.ensure_minimum_data(data_type, symbol, min_count):
|
||||
all_satisfied = False
|
||||
break
|
||||
|
||||
return all_satisfied
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in fallback data strategy: {e}")
|
||||
return False
|
||||
Reference in New Issue
Block a user