popov/gogo2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cleanup and removed dummy data

Browse Source
This commit is contained in:
Dobromir Popov
2025-07-26 23:35:14 +03:00
parent 3eb6335169
commit 87942d3807
14 changed files with 220 additions and 2465 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
core/orchestrator.py
View File

@ -179,12 +179,7 @@ class TradingOrchestrator:
self.fusion_decisions_count: int = 0
self.fusion_training_data: List[Any] = [] # Store training examples for decision model
# Simplified Data Integration - Replace complex FIFO queues with efficient cache
from core.simplified_data_integration import SimplifiedDataIntegration
self.data_integration = SimplifiedDataIntegration(
data_provider=self.data_provider,
symbols=[self.symbol] + self.ref_symbols
)
# Use data provider directly for BaseDataInput building (optimized)
# COB Integration - Real-time market microstructure data
self.cob_integration = None # Will be set to COBIntegration instance if available
@ -250,8 +245,7 @@ class TradingOrchestrator:
self.data_provider.start_centralized_data_collection()
logger.info("Centralized data collection started - all models and dashboard will receive data")
# Initialize simplified data integration
self._initialize_simplified_data_integration()
# Data provider is already initialized and optimized
# Log initial data status
logger.info("Simplified data integration initialized")
@ -897,31 +891,10 @@ class TradingOrchestrator:
try:
self.latest_cob_data[symbol] = cob_data
# Update data cache with COB data for BaseDataInput
if hasattr(self, 'data_integration') and self.data_integration:
# Convert cob_data to COBData format if needed
from .data_models import COBData
# Create COBData object from the raw cob_data
if 'price_buckets' in cob_data and 'current_price' in cob_data:
cob_data_obj = COBData(
symbol=symbol,
timestamp=datetime.now(),
current_price=cob_data['current_price'],
bucket_size=1.0 if 'ETH' in symbol else 10.0,
price_buckets=cob_data.get('price_buckets', {}),
bid_ask_imbalance=cob_data.get('bid_ask_imbalance', {}),
volume_weighted_prices=cob_data.get('volume_weighted_prices', {}),
order_flow_metrics=cob_data.get('order_flow_metrics', {}),
ma_1s_imbalance=cob_data.get('ma_1s_imbalance', {}),
ma_5s_imbalance=cob_data.get('ma_5s_imbalance', {}),
ma_15s_imbalance=cob_data.get('ma_15s_imbalance', {}),
ma_60s_imbalance=cob_data.get('ma_60s_imbalance', {})
)
# Update cache with COB data
self.data_integration.cache.update('cob_data', symbol, cob_data_obj, 'cob_integration')
logger.debug(f"Updated cache with COB data for {symbol}")
# Invalidate data provider cache when new COB data arrives
if hasattr(self.data_provider, 'invalidate_ohlcv_cache'):
self.data_provider.invalidate_ohlcv_cache(symbol)
logger.debug(f"Invalidated data provider cache for {symbol} due to COB update")
# Update dashboard
if self.dashboard and hasattr(self.dashboard, 'update_cob_data'):
@ -3722,54 +3695,34 @@ class TradingOrchestrator:
"""
return self.db_manager.get_best_checkpoint_metadata(model_name)
# === SIMPLIFIED DATA MANAGEMENT ===
def _initialize_simplified_data_integration(self):
"""Initialize the simplified data integration system"""
try:
# Start the data integration system
self.data_integration.start()
logger.info("Simplified data integration started successfully")
except Exception as e:
logger.error(f"Error starting simplified data integration: {e}")
# === DATA MANAGEMENT ===
def _log_data_status(self):
"""Log current data status"""
try:
status = self.data_integration.get_cache_status()
cache_status = status.get('cache_status', {})
logger.info("=== Data Cache Status ===")
for data_type, symbols_data in cache_status.items():
symbol_info = []
for symbol, info in symbols_data.items():
age = info.get('age_seconds', 0)
has_data = info.get('has_data', False)
if has_data and age < 300: # Recent data
symbol_info.append(f"{symbol}:✅")
else:
symbol_info.append(f"{symbol}:❌")
if symbol_info:
logger.info(f"{data_type}: {', '.join(symbol_info)}")
logger.info("=== Data Provider Status ===")
logger.info("Data provider is running and optimized for BaseDataInput building")
except Exception as e:
logger.error(f"Error logging data status: {e}")
def update_data_cache(self, data_type: str, symbol: str, data: Any, source: str = "orchestrator") -> bool:
"""
Update data cache with new data (simplified approach)
Update data cache through data provider
Args:
data_type: Type of data ('ohlcv_1s', 'technical_indicators', etc.)
symbol: Trading symbol
data: New data to store
source: Source of the data
data: Data to store
source: Source of the update
Returns:
bool: True if successful
bool: True if updated successfully
"""
try:
return self.data_integration.cache.update(data_type, symbol, data, source)
# Invalidate cache when new data arrives
if hasattr(self.data_provider, 'invalidate_ohlcv_cache'):
self.data_provider.invalidate_ohlcv_cache(symbol)
return True
except Exception as e:
logger.error(f"Error updating data cache {data_type}/{symbol}: {e}")
return False
@ -3929,7 +3882,7 @@ class TradingOrchestrator:
def build_base_data_input(self, symbol: str) -> Optional[Any]:
"""
Build BaseDataInput using simplified data integration (optimized for speed)
Build BaseDataInput using optimized data provider (should be instantaneous)
Args:
symbol: Trading symbol
@ -3938,8 +3891,8 @@ class TradingOrchestrator:
BaseDataInput with consistent data structure
"""
try:
# Use simplified data integration to build BaseDataInput (should be instantaneous)
return self.data_integration.build_base_data_input(symbol)
# Use data provider's optimized build_base_data_input method
return self.data_provider.build_base_data_input(symbol)
except Exception as e:
logger.error(f"Error building BaseDataInput for {symbol}: {e}")