#!/usr/bin/env python3
"""
Test Pivot-Based Normalization System

This script tests the comprehensive pivot-based normalization system:
1. Monthly 1s data collection with pagination
2. Williams Market Structure pivot analysis  
3. Pivot bounds extraction and caching
4. Pivot-based feature normalization
5. Integration with model training pipeline
"""

import sys
import os
import logging
from datetime import datetime, timedelta

# Add project root to path
sys.path.append(os.path.dirname(os.path.abspath(__file__)))

from core.data_provider import DataProvider
from core.config import get_config

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)

def test_pivot_normalization_system():
    """Test the complete pivot-based normalization system"""
    
    print("="*80)
    print("TESTING PIVOT-BASED NORMALIZATION SYSTEM")
    print("="*80)
    
    # Initialize data provider
    symbols = ['ETH/USDT']  # Test with ETH only
    timeframes = ['1s', '1m', '1h']
    
    logger.info("Initializing DataProvider with pivot-based normalization...")
    data_provider = DataProvider(symbols=symbols, timeframes=timeframes)
    
    # Test 1: Monthly Data Collection
    print("\n" + "="*60)
    print("TEST 1: MONTHLY 1S DATA COLLECTION")
    print("="*60)
    
    symbol = 'ETH/USDT'
    
    try:
        # This will trigger monthly data collection and pivot analysis
        logger.info(f"Testing monthly data collection for {symbol}...")
        monthly_data = data_provider._collect_monthly_1s_data(symbol)
        
        if monthly_data is not None:
            print(f"✅ Monthly data collection SUCCESS")
            print(f"   📊 Collected {len(monthly_data):,} 1s candles")
            print(f"   📅 Period: {monthly_data['timestamp'].min()} to {monthly_data['timestamp'].max()}")
            print(f"   💰 Price range: ${monthly_data['low'].min():.2f} - ${monthly_data['high'].max():.2f}")
            print(f"   📈 Volume range: {monthly_data['volume'].min():.2f} - {monthly_data['volume'].max():.2f}")
        else:
            print("❌ Monthly data collection FAILED")
            return False
            
    except Exception as e:
        print(f"❌ Monthly data collection ERROR: {e}")
        return False
    
    # Test 2: Pivot Bounds Extraction
    print("\n" + "="*60)
    print("TEST 2: PIVOT BOUNDS EXTRACTION")
    print("="*60)
    
    try:
        logger.info("Testing pivot bounds extraction...")
        bounds = data_provider._extract_pivot_bounds_from_monthly_data(symbol, monthly_data)
        
        if bounds is not None:
            print(f"✅ Pivot bounds extraction SUCCESS")
            print(f"   💰 Price bounds: ${bounds.price_min:.2f} - ${bounds.price_max:.2f}")
            print(f"   📊 Volume bounds: {bounds.volume_min:.2f} - {bounds.volume_max:.2f}")
            print(f"   🔸 Support levels: {len(bounds.pivot_support_levels)}")
            print(f"   🔹 Resistance levels: {len(bounds.pivot_resistance_levels)}")
            print(f"   📈 Candles analyzed: {bounds.total_candles_analyzed:,}")
            print(f"   ⏰ Created: {bounds.created_timestamp}")
            
            # Store bounds for next tests
            data_provider.pivot_bounds[symbol] = bounds
        else:
            print("❌ Pivot bounds extraction FAILED")
            return False
            
    except Exception as e:
        print(f"❌ Pivot bounds extraction ERROR: {e}")
        return False
    
    # Test 3: Pivot Context Features
    print("\n" + "="*60)
    print("TEST 3: PIVOT CONTEXT FEATURES")
    print("="*60)
    
    try:
        logger.info("Testing pivot context features...")
        
        # Get recent data for testing
        recent_data = data_provider.get_historical_data(symbol, '1m', limit=100)
        
        if recent_data is not None and not recent_data.empty:
            # Add pivot context features
            with_pivot_features = data_provider._add_pivot_context_features(recent_data, symbol)
            
            # Check if pivot features were added
            pivot_features = [col for col in with_pivot_features.columns if 'pivot' in col]
            
            if pivot_features:
                print(f"✅ Pivot context features SUCCESS")
                print(f"   🎯 Added features: {pivot_features}")
                
                # Show sample values
                latest_row = with_pivot_features.iloc[-1]
                print(f"   📊 Latest values:")
                for feature in pivot_features:
                    print(f"      {feature}: {latest_row[feature]:.4f}")
            else:
                print("❌ No pivot context features added")
                return False
        else:
            print("❌ Could not get recent data for testing")
            return False
            
    except Exception as e:
        print(f"❌ Pivot context features ERROR: {e}")
        return False
    
    # Test 4: Pivot-Based Normalization
    print("\n" + "="*60)
    print("TEST 4: PIVOT-BASED NORMALIZATION")
    print("="*60)
    
    try:
        logger.info("Testing pivot-based normalization...")
        
        # Get data with technical indicators
        data_with_indicators = data_provider.get_historical_data(symbol, '1m', limit=50)
        
        if data_with_indicators is not None and not data_with_indicators.empty:
            # Test traditional vs pivot normalization
            traditional_norm = data_provider._normalize_features(data_with_indicators.tail(10))
            pivot_norm = data_provider._normalize_features(data_with_indicators.tail(10), symbol=symbol)
            
            print(f"✅ Pivot-based normalization SUCCESS")
            print(f"   📊 Traditional normalization shape: {traditional_norm.shape}")
            print(f"   🎯 Pivot normalization shape: {pivot_norm.shape}")
            
            # Compare price normalization
            if 'close' in pivot_norm.columns:
                trad_close_range = traditional_norm['close'].max() - traditional_norm['close'].min()
                pivot_close_range = pivot_norm['close'].max() - pivot_norm['close'].min()
                
                print(f"   💰 Traditional close range: {trad_close_range:.6f}")
                print(f"   🎯 Pivot close range: {pivot_close_range:.6f}")
                
                # Pivot normalization should be better bounded
                if 0 <= pivot_norm['close'].min() and pivot_norm['close'].max() <= 1:
                    print(f"   ✅ Pivot normalization properly bounded [0,1]")
                else:
                    print(f"   ⚠️  Pivot normalization outside [0,1] bounds")
        else:
            print("❌ Could not get data for normalization testing")
            return False
            
    except Exception as e:
        print(f"❌ Pivot-based normalization ERROR: {e}")
        return False
    
    # Test 5: Feature Matrix with Pivot Normalization
    print("\n" + "="*60)
    print("TEST 5: FEATURE MATRIX WITH PIVOT NORMALIZATION")
    print("="*60)
    
    try:
        logger.info("Testing feature matrix with pivot normalization...")
        
        # Create feature matrix using pivot normalization
        feature_matrix = data_provider.get_feature_matrix(symbol, timeframes=['1m'], window_size=20)
        
        if feature_matrix is not None:
            print(f"✅ Feature matrix with pivot normalization SUCCESS")
            print(f"   📊 Matrix shape: {feature_matrix.shape}")
            print(f"   🎯 Data range: [{feature_matrix.min():.4f}, {feature_matrix.max():.4f}]")
            print(f"   📈 Mean: {feature_matrix.mean():.4f}")
            print(f"   📊 Std: {feature_matrix.std():.4f}")
            
            # Check for proper normalization
            if feature_matrix.min() >= -5 and feature_matrix.max() <= 5:  # Reasonable bounds
                print(f"   ✅ Feature matrix reasonably bounded")
            else:
                print(f"   ⚠️  Feature matrix may have extreme values")
        else:
            print("❌ Feature matrix creation FAILED")
            return False
            
    except Exception as e:
        print(f"❌ Feature matrix ERROR: {e}")
        return False
    
    # Test 6: Caching System
    print("\n" + "="*60)
    print("TEST 6: CACHING SYSTEM")
    print("="*60)
    
    try:
        logger.info("Testing caching system...")
        
        # Test pivot bounds caching
        original_bounds = data_provider.pivot_bounds[symbol]
        data_provider._save_pivot_bounds_to_cache(symbol, original_bounds)
        
        # Clear from memory and reload
        del data_provider.pivot_bounds[symbol]
        loaded_bounds = data_provider._load_pivot_bounds_from_cache(symbol)
        
        if loaded_bounds is not None:
            print(f"✅ Pivot bounds caching SUCCESS")
            print(f"   💾 Original price range: ${original_bounds.price_min:.2f} - ${original_bounds.price_max:.2f}")
            print(f"   💾 Loaded price range: ${loaded_bounds.price_min:.2f} - ${loaded_bounds.price_max:.2f}")
            
            # Restore bounds
            data_provider.pivot_bounds[symbol] = loaded_bounds
        else:
            print("❌ Pivot bounds caching FAILED")
            return False
            
    except Exception as e:
        print(f"❌ Caching system ERROR: {e}")
        return False
    
    # Test 7: Public API Methods
    print("\n" + "="*60)
    print("TEST 7: PUBLIC API METHODS")
    print("="*60)
    
    try:
        logger.info("Testing public API methods...")
        
        # Test get_pivot_bounds
        api_bounds = data_provider.get_pivot_bounds(symbol)
        if api_bounds is not None:
            print(f"✅ get_pivot_bounds() SUCCESS")
            print(f"   📊 Returned bounds for {api_bounds.symbol}")
        
        # Test get_pivot_normalized_features
        test_data = data_provider.get_historical_data(symbol, '1m', limit=10)
        if test_data is not None:
            normalized_data = data_provider.get_pivot_normalized_features(symbol, test_data)
            if normalized_data is not None:
                print(f"✅ get_pivot_normalized_features() SUCCESS")
                print(f"   📊 Normalized data shape: {normalized_data.shape}")
            else:
                print("❌ get_pivot_normalized_features() FAILED")
                return False
        
    except Exception as e:
        print(f"❌ Public API methods ERROR: {e}")
        return False
    
    # Final Summary
    print("\n" + "="*80)
    print("🎉 PIVOT-BASED NORMALIZATION SYSTEM TEST COMPLETE")
    print("="*80)
    print("✅ All tests PASSED successfully!")
    print("\n📋 System Features Verified:")
    print("   ✅ Monthly 1s data collection with pagination")
    print("   ✅ Williams Market Structure pivot analysis")
    print("   ✅ Pivot bounds extraction and validation")
    print("   ✅ Pivot context features generation")
    print("   ✅ Pivot-based feature normalization")
    print("   ✅ Feature matrix creation with pivot bounds")
    print("   ✅ Comprehensive caching system")
    print("   ✅ Public API methods")
    
    print(f"\n🎯 Ready for model training with pivot-normalized features!")
    return True

if __name__ == "__main__":
    try:
        success = test_pivot_normalization_system()
        
        if success:
            print("\n🚀 Pivot-based normalization system ready for production!")
            sys.exit(0)
        else:
            print("\n❌ Pivot-based normalization system has issues!")
            sys.exit(1)
            
    except KeyboardInterrupt:
        print("\n⏹️  Test interrupted by user")
        sys.exit(1)
    except Exception as e:
        print(f"\n💥 Unexpected error: {e}")
        import traceback
        traceback.print_exc()
        sys.exit(1)