popov/gogo2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

different confidence for open/close position

Browse Source
This commit is contained in:
Dobromir Popov
2025-05-27 01:03:40 +03:00
parent 678cf951a5
commit 97d348d517
5 changed files with 1097 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files

325
core/enhanced_orchestrator.py
View File

@ -22,7 +22,7 @@ from collections import deque
import torch
from .config import get_config
from .data_provider import DataProvider
from .data_provider import DataProvider, RawTick, OHLCVBar
from .universal_data_adapter import UniversalDataAdapter, UniversalDataStream
from .realtime_tick_processor import RealTimeTickProcessor, ProcessedTickFeatures, integrate_with_orchestrator
from models import get_model_registry, ModelInterface, CNNModelInterface, RLAgentInterface
@ -112,8 +112,9 @@ class EnhancedTradingOrchestrator:
self.symbols = self.config.symbols
self.timeframes = self.config.timeframes
# Configuration
self.confidence_threshold = self.config.orchestrator.get('confidence_threshold', 0.6)
# Configuration with different thresholds for opening vs closing
self.confidence_threshold_open = self.config.orchestrator.get('confidence_threshold', 0.6)
self.confidence_threshold_close = self.config.orchestrator.get('confidence_threshold_close', 0.25) # Much lower for closing
self.decision_frequency = self.config.orchestrator.get('decision_frequency', 30)
# Enhanced weighting system
@ -125,9 +126,11 @@ class EnhancedTradingOrchestrator:
self.recent_actions = {symbol: deque(maxlen=100) for symbol in self.symbols}
self.market_states = {symbol: deque(maxlen=1000) for symbol in self.symbols}
# Perfect move tracking for CNN training
# Perfect move tracking for CNN training with enhanced retrospective learning
self.perfect_moves = deque(maxlen=10000)
self.performance_tracker = {}
self.retrospective_learning_active = False
self.last_retrospective_analysis = datetime.now()
# RL feedback system
self.rl_evaluation_queue = deque(maxlen=1000)
@ -140,12 +143,35 @@ class EnhancedTradingOrchestrator:
# Integrate tick processor with orchestrator
integrate_with_orchestrator(self, self.tick_processor)
# Subscribe to raw tick data and OHLCV bars from data provider
self.raw_tick_subscriber_id = self.data_provider.subscribe_to_raw_ticks(self._handle_raw_tick)
self.ohlcv_bar_subscriber_id = self.data_provider.subscribe_to_ohlcv_bars(self._handle_ohlcv_bar)
# Raw tick and OHLCV data storage for models
self.raw_tick_buffers = {symbol: deque(maxlen=1000) for symbol in self.symbols}
self.ohlcv_bar_buffers = {symbol: deque(maxlen=3600) for symbol in self.symbols} # 1 hour of 1s bars
# Pattern-based decision enhancement
self.pattern_weights = {
'rapid_fire': 1.5,
'volume_spike': 1.3,
'price_acceleration': 1.4,
'high_frequency_bar': 1.2,
'volume_concentration': 1.1
}
# Current open positions tracking for closing logic
self.open_positions = {} # symbol -> {'side': str, 'entry_price': float, 'timestamp': datetime}
logger.info("Enhanced TradingOrchestrator initialized with Universal Data Format")
logger.info(f"Symbols: {self.symbols}")
logger.info(f"Timeframes: {self.timeframes}")
logger.info(f"Universal format: ETH ticks, 1m, 1h, 1d + BTC reference ticks")
logger.info(f"Enhanced confidence threshold: {self.confidence_threshold}")
logger.info(f"Opening confidence threshold: {self.confidence_threshold_open}")
logger.info(f"Closing confidence threshold: {self.confidence_threshold_close}")
logger.info("Real-time tick processor integrated for ultra-low latency processing")
logger.info("Raw tick and OHLCV bar processing enabled for pattern detection")
logger.info("Enhanced retrospective learning enabled for perfect opportunity detection")
def _initialize_timeframe_weights(self) -> Dict[str, float]:
"""Initialize weights for different timeframes"""
@ -520,7 +546,7 @@ class EnhancedTradingOrchestrator:
async def _make_coordinated_decision(self, symbol: str, predictions: List[EnhancedPrediction],
all_predictions: Dict[str, List[EnhancedPrediction]],
market_state: MarketState) -> Optional[TradingAction]:
"""Make decision considering symbol correlations"""
"""Make decision considering symbol correlations and different thresholds for opening/closing"""
if not predictions:
return None
@ -540,10 +566,21 @@ class EnhancedTradingOrchestrator:
final_confidence *= 0.8
logger.info(f"Reduced confidence for {symbol} due to correlation disagreement")
# Apply confidence threshold
if final_confidence < self.confidence_threshold:
# Determine if this is an opening or closing action
has_open_position = symbol in self.open_positions
is_closing_action = self._is_closing_action(symbol, final_action)
# Apply appropriate confidence threshold
if is_closing_action:
threshold = self.confidence_threshold_close
threshold_type = "closing"
else:
threshold = self.confidence_threshold_open
threshold_type = "opening"
if final_confidence < threshold:
final_action = 'HOLD'
logger.info(f"Action for {symbol} changed to HOLD due to low confidence: {final_confidence:.3f}")
logger.info(f"Action for {symbol} changed to HOLD due to low {threshold_type} confidence: {final_confidence:.3f} < {threshold:.3f}")
# Create trading action
if final_action != 'HOLD':
@ -562,11 +599,17 @@ class EnhancedTradingOrchestrator:
'timeframe_breakdown': [(tf.timeframe, tf.action, tf.confidence)
for tf in primary_pred.timeframe_predictions],
'correlated_sentiment': correlated_sentiment,
'market_regime': market_state.market_regime
'market_regime': market_state.market_regime,
'threshold_type': threshold_type,
'threshold_used': threshold,
'is_closing': is_closing_action
},
timeframe_analysis=primary_pred.timeframe_predictions
)
# Update position tracking
self._update_position_tracking(symbol, action)
# Store recent action
self.recent_actions[symbol].append(action)
@ -577,6 +620,131 @@ class EnhancedTradingOrchestrator:
return None
def _is_closing_action(self, symbol: str, action: str) -> bool:
"""Determine if an action would close an existing position"""
if symbol not in self.open_positions:
return False
current_position = self.open_positions[symbol]
# Closing logic: opposite action closes position
if current_position['side'] == 'LONG' and action == 'SELL':
return True
elif current_position['side'] == 'SHORT' and action == 'BUY':
return True
return False
def _update_position_tracking(self, symbol: str, action: TradingAction):
"""Update internal position tracking for threshold logic"""
if action.action == 'BUY':
# Close any short position, open long position
if symbol in self.open_positions and self.open_positions[symbol]['side'] == 'SHORT':
del self.open_positions[symbol]
else:
self.open_positions[symbol] = {
'side': 'LONG',
'entry_price': action.price,
'timestamp': action.timestamp
}
elif action.action == 'SELL':
# Close any long position, open short position
if symbol in self.open_positions and self.open_positions[symbol]['side'] == 'LONG':
del self.open_positions[symbol]
else:
self.open_positions[symbol] = {
'side': 'SHORT',
'entry_price': action.price,
'timestamp': action.timestamp
}
async def trigger_retrospective_learning(self):
"""Trigger retrospective learning analysis on recent perfect opportunities"""
try:
current_time = datetime.now()
# Only run retrospective analysis every 5 minutes to avoid overload
if (current_time - self.last_retrospective_analysis).total_seconds() < 300:
return
self.last_retrospective_analysis = current_time
# Analyze recent market moves for missed opportunities
await self._analyze_missed_opportunities()
# Update model confidence thresholds based on recent performance
self._adjust_confidence_thresholds()
# Mark retrospective learning as active
self.retrospective_learning_active = True
logger.info("Retrospective learning analysis completed")
except Exception as e:
logger.error(f"Error in retrospective learning: {e}")
async def _analyze_missed_opportunities(self):
"""Analyze recent price movements to identify missed perfect opportunities"""
try:
for symbol in self.symbols:
# Get recent price data
recent_data = self.data_provider.get_latest_candles(symbol, '1m', limit=60)
if recent_data is None or len(recent_data) < 10:
continue
# Look for significant price movements (>1% in 5 minutes)
for i in range(5, len(recent_data)):
price_change = (recent_data.iloc[i]['close'] - recent_data.iloc[i-5]['close']) / recent_data.iloc[i-5]['close']
if abs(price_change) > 0.01: # 1% move
# This was a perfect opportunity
optimal_action = 'BUY' if price_change > 0 else 'SELL'
# Create perfect move for retrospective learning
perfect_move = PerfectMove(
symbol=symbol,
timeframe='1m',
timestamp=recent_data.iloc[i-5]['timestamp'],
optimal_action=optimal_action,
actual_outcome=price_change,
market_state_before=None, # Would need to reconstruct
market_state_after=None, # Would need to reconstruct
confidence_should_have_been=min(0.95, abs(price_change) * 20) # Higher confidence for bigger moves
)
self.perfect_moves.append(perfect_move)
logger.info(f"Retrospective perfect opportunity identified: {optimal_action} {symbol} ({price_change*100:+.2f}%)")
except Exception as e:
logger.error(f"Error analyzing missed opportunities: {e}")
def _adjust_confidence_thresholds(self):
"""Dynamically adjust confidence thresholds based on recent performance"""
try:
if len(self.perfect_moves) < 10:
return
# Analyze recent perfect moves
recent_moves = list(self.perfect_moves)[-20:]
avg_confidence_needed = np.mean([move.confidence_should_have_been for move in recent_moves])
avg_outcome = np.mean([abs(move.actual_outcome) for move in recent_moves])
# Adjust opening threshold based on missed opportunities
if avg_confidence_needed > self.confidence_threshold_open:
adjustment = min(0.1, (avg_confidence_needed - self.confidence_threshold_open) * 0.1)
self.confidence_threshold_open = max(0.3, self.confidence_threshold_open - adjustment)
logger.info(f"Adjusted opening confidence threshold to {self.confidence_threshold_open:.3f}")
# Keep closing threshold very low for sensitivity
if avg_outcome > 0.02: # If we're seeing big moves
self.confidence_threshold_close = max(0.15, self.confidence_threshold_close * 0.9)
logger.info(f"Lowered closing confidence threshold to {self.confidence_threshold_close:.3f}")
except Exception as e:
logger.error(f"Error adjusting confidence thresholds: {e}")
def _get_correlated_sentiment(self, symbol: str,
all_predictions: Dict[str, List[EnhancedPrediction]]) -> Dict[str, Any]:
"""Get sentiment from correlated symbols"""
@ -993,7 +1161,7 @@ class EnhancedTradingOrchestrator:
return self.tick_processor.get_processing_stats()
def get_performance_metrics(self) -> Dict[str, Any]:
"""Get performance metrics for dashboard compatibility"""
"""Get enhanced performance metrics for dashboard compatibility"""
total_actions = sum(len(actions) for actions in self.recent_actions.values())
perfect_moves_count = len(self.perfect_moves)
@ -1004,6 +1172,17 @@ class EnhancedTradingOrchestrator:
# Add tick processing stats
tick_stats = self.get_realtime_tick_stats()
# Calculate retrospective learning metrics
recent_perfect_moves = list(self.perfect_moves)[-10:] if self.perfect_moves else []
avg_confidence_needed = np.mean([move.confidence_should_have_been for move in recent_perfect_moves]) if recent_perfect_moves else 0.6
# Pattern detection stats
patterns_detected = 0
for symbol_buffer in self.ohlcv_bar_buffers.values():
for bar in list(symbol_buffer)[-10:]: # Last 10 bars
if hasattr(bar, 'patterns') and bar.patterns:
patterns_detected += len(bar.patterns)
return {
'total_actions': total_actions,
'perfect_moves': perfect_moves_count,
@ -1011,11 +1190,28 @@ class EnhancedTradingOrchestrator:
'total_pnl': total_pnl,
'symbols_active': len(self.symbols),
'rl_queue_size': len(self.rl_evaluation_queue),
'confidence_threshold': self.confidence_threshold,
'confidence_threshold_open': self.confidence_threshold_open,
'confidence_threshold_close': self.confidence_threshold_close,
'decision_frequency': self.decision_frequency,
'leverage': '500x', # Ultra-fast scalping
'primary_timeframe': '1s', # Main scalping timeframe
'tick_processing': tick_stats # Real-time tick processing stats
'tick_processing': tick_stats, # Real-time tick processing stats
'retrospective_learning': {
'active': self.retrospective_learning_active,
'perfect_moves_recent': len(recent_perfect_moves),
'avg_confidence_needed': avg_confidence_needed,
'last_analysis': self.last_retrospective_analysis.isoformat(),
'patterns_detected': patterns_detected
},
'position_tracking': {
'open_positions': len(self.open_positions),
'positions': {symbol: pos['side'] for symbol, pos in self.open_positions.items()}
},
'thresholds': {
'opening': self.confidence_threshold_open,
'closing': self.confidence_threshold_close,
'adaptive': True
}
}
def analyze_market_conditions(self, symbol: str) -> Dict[str, Any]:
@ -1067,4 +1263,105 @@ class EnhancedTradingOrchestrator:
'symbol': symbol,
'error': str(e),
'analysis': f'Error analyzing {symbol}'
}
}
def _handle_raw_tick(self, raw_tick: RawTick):
"""Handle incoming raw tick data for pattern detection and learning"""
try:
symbol = raw_tick.symbol if hasattr(raw_tick, 'symbol') else 'UNKNOWN'
# Store raw tick for analysis
if symbol in self.raw_tick_buffers:
self.raw_tick_buffers[symbol].append(raw_tick)
# Detect violent moves for retrospective learning
if raw_tick.time_since_last < 50 and abs(raw_tick.price_change) > 0: # Fast tick with price change
price_change_pct = abs(raw_tick.price_change) / raw_tick.price if raw_tick.price > 0 else 0
if price_change_pct > 0.001: # 0.1% price change in single tick
logger.info(f"Violent tick detected: {symbol} {raw_tick.price_change:+.2f} ({price_change_pct*100:.3f}%) in {raw_tick.time_since_last:.0f}ms")
# Create perfect move for immediate learning
optimal_action = 'BUY' if raw_tick.price_change > 0 else 'SELL'
perfect_move = PerfectMove(
symbol=symbol,
timeframe='tick',
timestamp=raw_tick.timestamp,
optimal_action=optimal_action,
actual_outcome=price_change_pct,
market_state_before=None,
market_state_after=None,
confidence_should_have_been=min(0.95, price_change_pct * 50)
)
self.perfect_moves.append(perfect_move)
self.retrospective_learning_active = True
except Exception as e:
logger.error(f"Error handling raw tick: {e}")
def _handle_ohlcv_bar(self, ohlcv_bar: OHLCVBar):
"""Handle incoming 1s OHLCV bar for pattern detection"""
try:
symbol = ohlcv_bar.symbol if hasattr(ohlcv_bar, 'symbol') else 'UNKNOWN'
# Store OHLCV bar for analysis
if symbol in self.ohlcv_bar_buffers:
self.ohlcv_bar_buffers[symbol].append(ohlcv_bar)
# Analyze bar patterns for learning opportunities
if ohlcv_bar.patterns:
for pattern in ohlcv_bar.patterns:
pattern_weight = self.pattern_weights.get(pattern.pattern_type, 1.0)
if pattern.confidence > 0.7 and pattern_weight > 1.2:
logger.info(f"High-confidence pattern detected: {pattern.pattern_type} for {symbol} (conf: {pattern.confidence:.3f})")
# Create learning opportunity based on pattern
if pattern.price_change != 0:
optimal_action = 'BUY' if pattern.price_change > 0 else 'SELL'
outcome = abs(pattern.price_change) / ohlcv_bar.close if ohlcv_bar.close > 0 else 0
perfect_move = PerfectMove(
symbol=symbol,
timeframe='1s',
timestamp=pattern.start_time,
optimal_action=optimal_action,
actual_outcome=outcome,
market_state_before=None,
market_state_after=None,
confidence_should_have_been=min(0.9, pattern.confidence * pattern_weight)
)
self.perfect_moves.append(perfect_move)
# Check for significant 1s bar moves
if ohlcv_bar.high > 0 and ohlcv_bar.low > 0:
bar_range = (ohlcv_bar.high - ohlcv_bar.low) / ohlcv_bar.close
if bar_range > 0.002: # 0.2% range in 1 second
logger.info(f"Significant 1s bar range: {symbol} {bar_range*100:.3f}% range")
# Determine optimal action based on close vs open
if ohlcv_bar.close > ohlcv_bar.open:
optimal_action = 'BUY'
outcome = (ohlcv_bar.close - ohlcv_bar.open) / ohlcv_bar.open
else:
optimal_action = 'SELL'
outcome = (ohlcv_bar.open - ohlcv_bar.close) / ohlcv_bar.open
perfect_move = PerfectMove(
symbol=symbol,
timeframe='1s',
timestamp=ohlcv_bar.timestamp,
optimal_action=optimal_action,
actual_outcome=outcome,
market_state_before=None,
market_state_after=None,
confidence_should_have_been=min(0.85, bar_range * 100)
)
self.perfect_moves.append(perfect_move)
except Exception as e:
logger.error(f"Error handling OHLCV bar: {e}")