239 lines
9.9 KiB
Python
239 lines
9.9 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Training Status Audit - Check if models are actively training
|
|
"""
|
|
|
|
import asyncio
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
sys.path.append(str(Path('.').absolute()))
|
|
|
|
from core.enhanced_orchestrator import EnhancedTradingOrchestrator
|
|
from core.data_provider import DataProvider
|
|
|
|
async def check_training_status():
|
|
print("=" * 70)
|
|
print("TRAINING STATUS AUDIT")
|
|
print("=" * 70)
|
|
|
|
try:
|
|
data_provider = DataProvider()
|
|
orchestrator = EnhancedTradingOrchestrator(
|
|
data_provider=data_provider,
|
|
symbols=['ETH/USDT', 'BTC/USDT'],
|
|
enhanced_rl_training=True
|
|
)
|
|
|
|
print(f"✓ Enhanced Orchestrator created")
|
|
|
|
# 1. Check DQN Agent Status
|
|
print("\n--- DQN AGENT STATUS ---")
|
|
if hasattr(orchestrator, 'sensitivity_dqn_agent'):
|
|
dqn_agent = orchestrator.sensitivity_dqn_agent
|
|
print(f"DQN Agent: {dqn_agent}")
|
|
|
|
if dqn_agent is not None:
|
|
print(f"DQN Agent Type: {type(dqn_agent)}")
|
|
|
|
# Check if it has training stats
|
|
if hasattr(dqn_agent, 'get_enhanced_training_stats'):
|
|
try:
|
|
stats = dqn_agent.get_enhanced_training_stats()
|
|
print(f"DQN Training Stats: {stats}")
|
|
except Exception as e:
|
|
print(f"Error getting DQN stats: {e}")
|
|
|
|
# Check memory and training status
|
|
if hasattr(dqn_agent, 'memory'):
|
|
print(f"DQN Memory Size: {len(dqn_agent.memory)}")
|
|
if hasattr(dqn_agent, 'batch_size'):
|
|
print(f"DQN Batch Size: {dqn_agent.batch_size}")
|
|
if hasattr(dqn_agent, 'epsilon'):
|
|
print(f"DQN Epsilon: {dqn_agent.epsilon}")
|
|
|
|
# Check if training is possible
|
|
can_train = hasattr(dqn_agent, 'replay') and hasattr(dqn_agent, 'memory')
|
|
print(f"DQN Can Train: {can_train}")
|
|
|
|
else:
|
|
print("❌ DQN Agent is None - needs initialization")
|
|
try:
|
|
orchestrator._initialize_sensitivity_dqn()
|
|
print("✓ DQN Agent initialized")
|
|
dqn_agent = orchestrator.sensitivity_dqn_agent
|
|
print(f"New DQN Agent: {type(dqn_agent)}")
|
|
except Exception as e:
|
|
print(f"Error initializing DQN: {e}")
|
|
else:
|
|
print("❌ No DQN agent attribute found")
|
|
|
|
# 2. Check CNN Status
|
|
print("\n--- CNN MODEL STATUS ---")
|
|
if hasattr(orchestrator, 'williams_structure'):
|
|
williams = orchestrator.williams_structure
|
|
print(f"Williams CNN: {williams}")
|
|
|
|
if williams is not None:
|
|
print(f"Williams Type: {type(williams)}")
|
|
|
|
# Check if it has training stats
|
|
if hasattr(williams, 'get_training_stats'):
|
|
try:
|
|
stats = williams.get_training_stats()
|
|
print(f"CNN Training Stats: {stats}")
|
|
except Exception as e:
|
|
print(f"Error getting CNN stats: {e}")
|
|
|
|
# Check if it's enabled
|
|
print(f"Williams Enabled: {getattr(orchestrator, 'williams_enabled', False)}")
|
|
else:
|
|
print("❌ Williams CNN is None")
|
|
else:
|
|
print("❌ No Williams CNN attribute found")
|
|
|
|
# 3. Check COB Integration Training
|
|
print("\n--- COB INTEGRATION STATUS ---")
|
|
if hasattr(orchestrator, 'cob_integration'):
|
|
cob = orchestrator.cob_integration
|
|
print(f"COB Integration: {cob}")
|
|
|
|
if cob is not None:
|
|
print(f"COB Type: {type(cob)}")
|
|
|
|
# Check if COB is started
|
|
cob_active = getattr(orchestrator, 'cob_integration_active', False)
|
|
print(f"COB Active: {cob_active}")
|
|
|
|
# Try to start COB if not active
|
|
if not cob_active:
|
|
print("Starting COB integration...")
|
|
try:
|
|
await orchestrator.start_cob_integration()
|
|
print("✓ COB integration started")
|
|
except Exception as e:
|
|
print(f"Error starting COB: {e}")
|
|
|
|
# Get COB stats
|
|
try:
|
|
stats = cob.get_statistics()
|
|
print(f"COB Statistics: {stats}")
|
|
except Exception as e:
|
|
print(f"Error getting COB stats: {e}")
|
|
|
|
# Check COB feature generation
|
|
cob_features = getattr(orchestrator, 'latest_cob_features', {})
|
|
print(f"COB Features Available: {list(cob_features.keys())}")
|
|
else:
|
|
print("❌ COB Integration is None")
|
|
else:
|
|
print("❌ No COB integration attribute found")
|
|
|
|
# 4. Check Training Queues and Learning
|
|
print("\n--- TRAINING ACTIVITY STATUS ---")
|
|
|
|
# Check extrema trainer
|
|
if hasattr(orchestrator, 'extrema_trainer'):
|
|
extrema = orchestrator.extrema_trainer
|
|
print(f"Extrema Trainer: {extrema}")
|
|
if extrema and hasattr(extrema, 'get_training_stats'):
|
|
try:
|
|
stats = extrema.get_training_stats()
|
|
print(f"Extrema Training Stats: {stats}")
|
|
except Exception as e:
|
|
print(f"Error getting extrema stats: {e}")
|
|
|
|
# Check negative case trainer
|
|
if hasattr(orchestrator, 'negative_case_trainer'):
|
|
negative = orchestrator.negative_case_trainer
|
|
print(f"Negative Case Trainer: {negative}")
|
|
|
|
# Check recent decisions and training queues
|
|
if hasattr(orchestrator, 'recent_decisions'):
|
|
recent_decisions = orchestrator.recent_decisions
|
|
print(f"Recent Decisions: {len(recent_decisions) if recent_decisions else 0}")
|
|
|
|
if hasattr(orchestrator, 'sensitivity_learning_queue'):
|
|
queue = orchestrator.sensitivity_learning_queue
|
|
print(f"Sensitivity Learning Queue: {len(queue) if queue else 0}")
|
|
|
|
if hasattr(orchestrator, 'rl_evaluation_queue'):
|
|
queue = orchestrator.rl_evaluation_queue
|
|
print(f"RL Evaluation Queue: {len(queue) if queue else 0}")
|
|
|
|
# 5. Test Signal Generation and Training
|
|
print("\n--- TESTING SIGNAL GENERATION ---")
|
|
|
|
# Generate a test decision to see if training is triggered
|
|
try:
|
|
print("Making coordinated decisions...")
|
|
decisions = await orchestrator.make_coordinated_decisions()
|
|
print(f"Decisions Generated: {len(decisions) if decisions else 0}")
|
|
|
|
for symbol, decision in decisions.items():
|
|
if decision:
|
|
print(f"{symbol}: {decision.action} (confidence: {decision.confidence:.3f})")
|
|
else:
|
|
print(f"{symbol}: No decision")
|
|
|
|
except Exception as e:
|
|
print(f"Error making decisions: {e}")
|
|
|
|
# 6. Wait and check for training activity
|
|
print("\n--- MONITORING TRAINING ACTIVITY (10 seconds) ---")
|
|
|
|
initial_stats = {}
|
|
|
|
# Capture initial state
|
|
if hasattr(orchestrator, 'sensitivity_dqn_agent') and orchestrator.sensitivity_dqn_agent:
|
|
if hasattr(orchestrator.sensitivity_dqn_agent, 'memory'):
|
|
initial_stats['dqn_memory'] = len(orchestrator.sensitivity_dqn_agent.memory)
|
|
|
|
# Wait and monitor
|
|
for i in range(10):
|
|
await asyncio.sleep(1)
|
|
print(f"Monitoring... {i+1}/10")
|
|
|
|
# Check if any training happened
|
|
if hasattr(orchestrator, 'sensitivity_dqn_agent') and orchestrator.sensitivity_dqn_agent:
|
|
if hasattr(orchestrator.sensitivity_dqn_agent, 'memory'):
|
|
current_memory = len(orchestrator.sensitivity_dqn_agent.memory)
|
|
if current_memory != initial_stats.get('dqn_memory', 0):
|
|
print(f"🔥 DQN training detected! Memory: {initial_stats.get('dqn_memory', 0)} -> {current_memory}")
|
|
|
|
# Final status
|
|
print("\n--- FINAL TRAINING STATUS ---")
|
|
|
|
# Check if models are actively learning
|
|
dqn_learning = False
|
|
cnn_learning = False
|
|
cob_learning = False
|
|
|
|
if hasattr(orchestrator, 'sensitivity_dqn_agent') and orchestrator.sensitivity_dqn_agent:
|
|
memory_size = getattr(orchestrator.sensitivity_dqn_agent, 'memory', [])
|
|
batch_size = getattr(orchestrator.sensitivity_dqn_agent, 'batch_size', 32)
|
|
dqn_learning = len(memory_size) >= batch_size if hasattr(memory_size, '__len__') else False
|
|
|
|
print(f"DQN Learning Ready: {dqn_learning}")
|
|
print(f"CNN Learning Ready: {cnn_learning}")
|
|
print(f"COB Learning Ready: {cob_learning}")
|
|
|
|
# GPU Utilization Check
|
|
try:
|
|
import GPUtil
|
|
gpus = GPUtil.getGPUs()
|
|
if gpus:
|
|
for gpu in gpus:
|
|
print(f"GPU {gpu.id}: {gpu.load*100:.1f}% utilization, {gpu.memoryUtil*100:.1f}% memory")
|
|
else:
|
|
print("No GPUs detected")
|
|
except ImportError:
|
|
print("GPUtil not available - cannot check GPU status")
|
|
|
|
except Exception as e:
|
|
print(f"Error in training status check: {e}")
|
|
import traceback
|
|
traceback.print_exc()
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(check_training_status()) |