"""
Report Data Crawler Sub-Module

This module provides functionality to crawl and compile comprehensive trading reports
in the specified format for different trading pairs. It extends the DataProvider
with specialized methods for report generation.

Enhanced with:
- Polymarket bets/odds data for sentiment analysis
- Multiple open interest data sources for improved reliability
- Real-time market sentiment from prediction markets

CRITICAL POLICY: NO SYNTHETIC DATA ALLOWED
This module MUST ONLY use real market data from exchanges.
NEVER use np.random.*, mock/fake/synthetic data, or placeholder values.
If data is unavailable: return None/0/empty, log errors, raise exceptions.
"""

import logging
import pandas as pd
import numpy as np
import requests
import time
import json
from datetime import datetime, timedelta
from typing import Dict, List, Optional, Tuple, Any
from dataclasses import dataclass
import ta
import warnings

# Suppress ta library deprecation warnings
warnings.filterwarnings("ignore", category=FutureWarning, module="ta")

logger = logging.getLogger(__name__)

@dataclass
class PolymarketData:
    """Polymarket prediction market data"""
    market_id: str
    question: str
    outcome: str
    probability: float
    volume_24h: float
    liquidity: float
    last_trade_time: datetime
    market_type: str  # 'crypto', 'election', 'sports', etc.

@dataclass
class OpenInterestData:
    """Open interest data from multiple sources"""
    source: str
    symbol: str
    open_interest: float
    timestamp: datetime
    change_24h: float
    change_percent: float

@dataclass
class ReportData:
    """Data structure for comprehensive trading report"""
    symbol: str
    timestamp: datetime
    
    # Current values
    current_price: float
    current_ema20: float
    current_macd: float
    current_rsi_7: float
    
    # Enhanced Open Interest and Funding Rate (multiple sources)
    open_interest_data: List[OpenInterestData]
    funding_rate: float
    
    # Polymarket sentiment data
    polymarket_data: List[PolymarketData]
    market_sentiment_score: float
    
    # Intraday series (3-minute intervals, oldest → newest)
    mid_prices: List[float]
    ema_20_series: List[float]
    macd_series: List[float]
    rsi_7_series: List[float]
    rsi_14_series: List[float]
    
    # Longer-term context (4-hour timeframe)
    ema_20_4h: float
    ema_50_4h: float
    atr_3_4h: float
    atr_14_4h: float
    current_volume: float
    average_volume: float
    macd_4h_series: List[float]
    rsi_14_4h_series: List[float]

class ReportDataCrawler:
    """Specialized crawler for compiling comprehensive trading reports"""
    
    def __init__(self, data_provider=None):
        """Initialize the report data crawler
        
        Args:
            data_provider: Instance of DataProvider to use for data fetching
        """
        self.data_provider = data_provider
        self.session = requests.Session()
        self.session.headers.update({
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
        })
        
        # Cache for API responses to avoid rate limiting
        self._cache = {}
        self._cache_timeout = 30  # 30 seconds cache
        
        # Polymarket API configuration
        self.polymarket_base_url = "https://gamma-api.polymarket.com"
        self.polymarket_graphql_url = f"{self.polymarket_base_url}/graphql"
        
        # Open interest data sources
        self.open_interest_sources = {
            'binance': 'https://fapi.binance.com/fapi/v1/openInterest',
            'bybit': 'https://api.bybit.com/v5/market/open-interest',
            'okx': 'https://www.okx.com/api/v5/public/open-interest',
            'coinglass': 'https://open-api.coinglass.com/public/v2/open_interest'
        }
        
    def _get_cached_or_fetch(self, cache_key: str, fetch_func, *args, **kwargs):
        """Get data from cache or fetch if expired"""
        now = time.time()
        
        if cache_key in self._cache:
            data, timestamp = self._cache[cache_key]
            if now - timestamp < self._cache_timeout:
                return data
        
        # Fetch new data
        data = fetch_func(*args, **kwargs)
        if data is not None:
            self._cache[cache_key] = (data, now)
        
        return data
    
    def _fetch_polymarket_data(self, symbol: str) -> List[PolymarketData]:
        """Fetch Polymarket prediction market data for cryptocurrency-related markets
        
        Args:
            symbol: Trading pair symbol (e.g., 'BTC/USDT')
            
        Returns:
            List of PolymarketData objects
        """
        try:
            cache_key = f"polymarket_{symbol}"
            cached_data = self._get_cached_or_fetch(cache_key, self._fetch_polymarket_raw, symbol)
            
            if not cached_data:
                return []
            
            polymarket_data = []
            base_symbol = symbol.split('/')[0].lower()  # Extract BTC from BTC/USDT
            
            for market in cached_data:
                try:
                    # Filter for crypto-related markets
                    if any(keyword in market.get('question', '').lower() for keyword in [base_symbol, 'bitcoin', 'ethereum', 'crypto', 'cryptocurrency']):
                        polymarket_data.append(PolymarketData(
                            market_id=market.get('id', ''),
                            question=market.get('question', ''),
                            outcome=market.get('outcome', ''),
                            probability=float(market.get('probability', 0)),
                            volume_24h=float(market.get('volume24h', 0)),
                            liquidity=float(market.get('liquidity', 0)),
                            last_trade_time=datetime.fromisoformat(market.get('lastTradeTime', datetime.now().isoformat())),
                            market_type='crypto'
                        ))
                except Exception as e:
                    logger.warning(f"Error parsing Polymarket market data: {e}")
                    continue
            
            logger.info(f"Fetched {len(polymarket_data)} Polymarket markets for {symbol}")
            return polymarket_data
            
        except Exception as e:
            logger.error(f"Error fetching Polymarket data for {symbol}: {e}")
            return []
    
    def _fetch_polymarket_raw(self, symbol: str) -> List[Dict]:
        """Fetch raw Polymarket data using GraphQL API"""
        try:
            # GraphQL query for active markets
            query = """
            query GetMarkets($limit: Int!, $offset: Int!) {
                markets(
                    limit: $limit
                    offset: $offset
                    orderBy: "volume"
                    orderDirection: "desc"
                    where: {
                        active: true
                        closed: false
                    }
                ) {
                    id
                    question
                    outcome
                    probability
                    volume24h
                    liquidity
                    lastTradeTime
                    endDate
                }
            }
            """
            
            variables = {
                "limit": 50,
                "offset": 0
            }
            
            payload = {
                "query": query,
                "variables": variables
            }
            
            response = self.session.post(
                self.polymarket_graphql_url,
                json=payload,
                timeout=10,
                headers={'Content-Type': 'application/json'}
            )
            
            if response.status_code == 200:
                data = response.json()
                if 'data' in data and 'markets' in data['data']:
                    return data['data']['markets']
            
            logger.warning(f"Polymarket API returned status {response.status_code}")
            return []
            
        except Exception as e:
            logger.error(f"Error fetching raw Polymarket data: {e}")
            return []
    
    def _calculate_market_sentiment_score(self, polymarket_data: List[PolymarketData], symbol: str) -> float:
        """Calculate market sentiment score from Polymarket data
        
        Args:
            polymarket_data: List of PolymarketData objects
            symbol: Trading pair symbol
            
        Returns:
            Sentiment score between -1 (bearish) and 1 (bullish)
        """
        try:
            if not polymarket_data:
                return 0.0
            
            base_symbol = symbol.split('/')[0].lower()
            sentiment_scores = []
            
            for data in polymarket_data:
                # Weight by volume and liquidity
                weight = (data.volume_24h + data.liquidity) / 1000000  # Normalize
                
                # Extract sentiment from probability
                if 'above' in data.question.lower() or 'higher' in data.question.lower():
                    # Bullish sentiment
                    sentiment = (data.probability - 0.5) * 2  # Convert 0-1 to -1 to 1
                elif 'below' in data.question.lower() or 'lower' in data.question.lower():
                    # Bearish sentiment
                    sentiment = (0.5 - data.probability) * 2  # Convert 0-1 to -1 to 1
                else:
                    # Neutral or unclear sentiment
                    sentiment = 0.0
                
                sentiment_scores.append(sentiment * weight)
            
            if sentiment_scores:
                # Weighted average sentiment
                total_weight = sum((data.volume_24h + data.liquidity) / 1000000 for data in polymarket_data)
                if total_weight > 0:
                    return sum(sentiment_scores) / total_weight
            
            return 0.0
            
        except Exception as e:
            logger.error(f"Error calculating market sentiment score: {e}")
            return 0.0
    
    def _fetch_open_interest_from_multiple_sources(self, symbol: str) -> List[OpenInterestData]:
        """Fetch open interest data from multiple sources for improved reliability
        
        Args:
            symbol: Trading pair symbol (e.g., 'BTC/USDT')
            
        Returns:
            List of OpenInterestData objects from different sources
        """
        try:
            open_interest_data = []
            binance_symbol = symbol.replace('/', '')
            
            # Source 1: Binance
            try:
                cache_key = f"binance_oi_{symbol}"
                binance_data = self._get_cached_or_fetch(cache_key, self._fetch_binance_open_interest, binance_symbol)
                if binance_data:
                    open_interest_data.append(OpenInterestData(
                        source='binance',
                        symbol=symbol,
                        open_interest=binance_data['openInterest'],
                        timestamp=datetime.now(),
                        change_24h=binance_data.get('change24h', 0),
                        change_percent=binance_data.get('changePercent', 0)
                    ))
            except Exception as e:
                logger.warning(f"Failed to fetch Binance open interest: {e}")
            
            # Source 2: Bybit
            try:
                cache_key = f"bybit_oi_{symbol}"
                bybit_data = self._get_cached_or_fetch(cache_key, self._fetch_bybit_open_interest, symbol)
                if bybit_data:
                    open_interest_data.append(OpenInterestData(
                        source='bybit',
                        symbol=symbol,
                        open_interest=bybit_data['openInterest'],
                        timestamp=datetime.now(),
                        change_24h=bybit_data.get('change24h', 0),
                        change_percent=bybit_data.get('changePercent', 0)
                    ))
            except Exception as e:
                logger.warning(f"Failed to fetch Bybit open interest: {e}")
            
            # Source 3: OKX
            try:
                cache_key = f"okx_oi_{symbol}"
                okx_data = self._get_cached_or_fetch(cache_key, self._fetch_okx_open_interest, symbol)
                if okx_data:
                    open_interest_data.append(OpenInterestData(
                        source='okx',
                        symbol=symbol,
                        open_interest=okx_data['openInterest'],
                        timestamp=datetime.now(),
                        change_24h=okx_data.get('change24h', 0),
                        change_percent=okx_data.get('changePercent', 0)
                    ))
            except Exception as e:
                logger.warning(f"Failed to fetch OKX open interest: {e}")
            
            logger.info(f"Fetched open interest data from {len(open_interest_data)} sources for {symbol}")
            return open_interest_data
            
        except Exception as e:
            logger.error(f"Error fetching open interest from multiple sources for {symbol}: {e}")
            return []
    
    def _fetch_binance_open_interest(self, symbol: str) -> Dict:
        """Fetch open interest data from Binance"""
        try:
            url = "https://fapi.binance.com/fapi/v1/openInterest"
            params = {'symbol': symbol}
            
            response = self.session.get(url, params=params, timeout=10)
            if response.status_code == 200:
                return response.json()
            return {}
        except Exception as e:
            logger.error(f"Error fetching Binance open interest: {e}")
            return {}
    
    def _fetch_bybit_open_interest(self, symbol: str) -> Dict:
        """Fetch open interest data from Bybit"""
        try:
            # Convert BTC/USDT to BTCUSDT for Bybit
            bybit_symbol = symbol.replace('/', '')
            url = "https://api.bybit.com/v5/market/open-interest"
            params = {
                'category': 'linear',
                'symbol': bybit_symbol,
                'intervalTime': '5min'
            }
            
            response = self.session.get(url, params=params, timeout=10)
            if response.status_code == 200:
                data = response.json()
                if 'result' in data and 'list' in data['result'] and data['result']['list']:
                    latest = data['result']['list'][0]
                    return {
                        'openInterest': float(latest['openInterest']),
                        'change24h': float(latest.get('change24h', 0)),
                        'changePercent': float(latest.get('changePercent', 0))
                    }
            return {}
        except Exception as e:
            logger.error(f"Error fetching Bybit open interest: {e}")
            return {}
    
    def _fetch_okx_open_interest(self, symbol: str) -> Dict:
        """Fetch open interest data from OKX"""
        try:
            # Convert BTC/USDT to BTC-USDT-SWAP for OKX
            okx_symbol = symbol.replace('/', '-') + '-SWAP'
            url = "https://www.okx.com/api/v5/public/open-interest"
            params = {'instId': okx_symbol}
            
            response = self.session.get(url, params=params, timeout=10)
            if response.status_code == 200:
                data = response.json()
                if 'data' in data and data['data']:
                    latest = data['data'][0]
                    return {
                        'openInterest': float(latest['oi']),
                        'change24h': float(latest.get('oiCcy', 0)),
                        'changePercent': 0  # OKX doesn't provide percentage change
                    }
            return {}
        except Exception as e:
            logger.error(f"Error fetching OKX open interest: {e}")
            return {}
    
    def _fetch_funding_rate(self, symbol: str) -> float:
        """Fetch funding rate from Binance"""
        try:
            binance_symbol = symbol.replace('/', '')
            url = "https://fapi.binance.com/fapi/v1/premiumIndex"
            params = {'symbol': binance_symbol}
            
            response = self.session.get(url, params=params, timeout=10)
            if response.status_code == 200:
                data = response.json()
                return float(data.get('lastFundingRate', 0))
            return 0.0
        except Exception as e:
            logger.error(f"Error fetching funding rate: {e}")
            return 0.0
    
    def _calculate_technical_indicators(self, df: pd.DataFrame) -> Dict[str, Any]:
        """Calculate technical indicators for the given dataframe"""
        try:
            if df.empty or len(df) < 5:
                logger.warning("Insufficient data for technical indicators")
                return {}
            
            indicators = {}
            
            # EMA 20 (need at least 20 periods)
            if len(df) >= 20:
                indicators['ema_20'] = ta.trend.ema_indicator(df['close'], window=20)
            else:
                indicators['ema_20'] = pd.Series(index=df.index, dtype=float)
            
            # MACD (need at least 26 periods)
            if len(df) >= 26:
                macd = ta.trend.MACD(df['close'])
                indicators['macd'] = macd.macd()
                indicators['macd_signal'] = macd.macd_signal()
                indicators['macd_histogram'] = macd.macd_diff()
            else:
                indicators['macd'] = pd.Series(index=df.index, dtype=float)
                indicators['macd_signal'] = pd.Series(index=df.index, dtype=float)
                indicators['macd_histogram'] = pd.Series(index=df.index, dtype=float)
            
            # RSI (7 and 14 period) - need at least 14 periods
            if len(df) >= 14:
                indicators['rsi_7'] = self._calculate_rsi(df['close'], period=7)
                indicators['rsi_14'] = self._calculate_rsi(df['close'], period=14)
            else:
                indicators['rsi_7'] = pd.Series(index=df.index, dtype=float)
                indicators['rsi_14'] = pd.Series(index=df.index, dtype=float)
            
            # ATR (need at least 14 periods)
            if len(df) >= 14:
                indicators['atr_3'] = ta.volatility.average_true_range(df['high'], df['low'], df['close'], window=3)
                indicators['atr_14'] = ta.volatility.average_true_range(df['high'], df['low'], df['close'], window=14)
            else:
                indicators['atr_3'] = pd.Series(index=df.index, dtype=float)
                indicators['atr_14'] = pd.Series(index=df.index, dtype=float)
            
            # EMA 50 (need at least 50 periods)
            if len(df) >= 50:
                indicators['ema_50'] = ta.trend.ema_indicator(df['close'], window=50)
            else:
                indicators['ema_50'] = pd.Series(index=df.index, dtype=float)
            
            return indicators
            
        except Exception as e:
            logger.error(f"Error calculating technical indicators: {e}")
            return {}
    
    def _calculate_rsi(self, prices: pd.Series, period: int = 14) -> pd.Series:
        """Calculate RSI using our own implementation"""
        try:
            delta = prices.diff()
            gain = (delta.where(delta > 0, 0)).rolling(window=period).mean()
            loss = (-delta.where(delta < 0, 0)).rolling(window=period).mean()
            rs = gain / loss
            rsi = 100 - (100 / (1 + rs))
            return rsi
        except Exception as e:
            logger.error(f"Error calculating RSI: {e}")
            return pd.Series(index=prices.index, dtype=float)
    
    def _get_intraday_series(self, symbol: str, limit: int = 10) -> Dict[str, List[float]]:
        """Get intraday series data (3-minute intervals)"""
        try:
            if not self.data_provider:
                logger.error("No data provider available")
                return {}
            
            # Get 1-minute data and resample to 3-minute intervals
            df_1m = self.data_provider.get_historical_data(symbol, '1m', limit=limit*3, refresh=True)
            if df_1m is None or df_1m.empty:
                logger.warning(f"No 1-minute data available for {symbol}")
                return {}
            
            # Resample to 3-minute intervals
            df_3m = df_1m.resample('3min').agg({
                'open': 'first',
                'high': 'max',
                'low': 'min',
                'close': 'last',
                'volume': 'sum'
            }).dropna()
            
            if len(df_3m) < limit:
                logger.warning(f"Insufficient 3-minute data: {len(df_3m)} < {limit}")
                return {}
            
            # Take the last 'limit' periods
            df_3m = df_3m.tail(limit)
            
            # Calculate technical indicators
            indicators = self._calculate_technical_indicators(df_3m)
            
            # Extract series data
            series_data = {
                'mid_prices': df_3m['close'].tolist(),
                'ema_20_series': indicators.get('ema_20', pd.Series()).tolist(),
                'macd_series': indicators.get('macd', pd.Series()).tolist(),
                'rsi_7_series': indicators.get('rsi_7', pd.Series()).tolist(),
                'rsi_14_series': indicators.get('rsi_14', pd.Series()).tolist()
            }
            
            return series_data
            
        except Exception as e:
            logger.error(f"Error getting intraday series for {symbol}: {e}")
            return {}
    
    def _get_longer_term_context(self, symbol: str, limit: int = 10) -> Dict[str, Any]:
        """Get longer-term context data (4-hour timeframe)"""
        try:
            if not self.data_provider:
                logger.error("No data provider available")
                return {}
            
            # Get 4-hour data
            df_4h = self.data_provider.get_historical_data(symbol, '4h', limit=limit, refresh=True)
            if df_4h is None or df_4h.empty:
                logger.warning(f"No 4-hour data available for {symbol}")
                return {}
            
            # Calculate technical indicators
            indicators = self._calculate_technical_indicators(df_4h)
            
            # Calculate volume metrics
            current_volume = df_4h['volume'].iloc[-1] if not df_4h.empty else 0
            average_volume = df_4h['volume'].mean() if not df_4h.empty else 0
            
            context_data = {
                'ema_20_4h': indicators.get('ema_20', pd.Series()).iloc[-1] if not indicators.get('ema_20', pd.Series()).empty else 0,
                'ema_50_4h': indicators.get('ema_50', pd.Series()).iloc[-1] if not indicators.get('ema_50', pd.Series()).empty else 0,
                'atr_3_4h': indicators.get('atr_3', pd.Series()).iloc[-1] if not indicators.get('atr_3', pd.Series()).empty else 0,
                'atr_14_4h': indicators.get('atr_14', pd.Series()).iloc[-1] if not indicators.get('atr_14', pd.Series()).empty else 0,
                'current_volume': current_volume,
                'average_volume': average_volume,
                'macd_4h_series': indicators.get('macd', pd.Series()).tolist(),
                'rsi_14_4h_series': indicators.get('rsi_14', pd.Series()).tolist()
            }
            
            return context_data
            
        except Exception as e:
            logger.error(f"Error getting longer-term context for {symbol}: {e}")
            return {}
    
    def crawl_report_data(self, symbol: str) -> Optional[ReportData]:
        """Crawl comprehensive report data for a trading pair
        
        Args:
            symbol: Trading pair symbol (e.g., 'BTC/USDT')
            
        Returns:
            ReportData object with all required information, or None if failed
        """
        try:
            logger.info(f"Crawling report data for {symbol}")
            
            if not self.data_provider:
                logger.error("No data provider available for crawling")
                return None
            
            # Force refresh data to ensure we have current information
            logger.info(f"Refreshing data for {symbol}...")
            for timeframe in ['1m', '1h', '4h']:
                try:
                    self.data_provider.get_historical_data(symbol, timeframe, limit=100, refresh=True)
                except Exception as e:
                    logger.warning(f"Could not refresh {timeframe} data for {symbol}: {e}")
            
            # Get current price and basic data
            current_price = self.data_provider.get_current_price(symbol)
            if current_price is None:
                logger.error(f"Could not get current price for {symbol}")
                return None
            
            # Get 1-minute data for current indicators
            df_1m = self.data_provider.get_historical_data(symbol, '1m', limit=50, refresh=True)
            if df_1m is None or df_1m.empty:
                logger.error(f"Could not get 1-minute data for {symbol}")
                return None
            
            # Calculate current technical indicators
            indicators = self._calculate_technical_indicators(df_1m)
            
            # Get current indicator values
            current_ema20 = indicators.get('ema_20', pd.Series()).iloc[-1] if not indicators.get('ema_20', pd.Series()).empty else 0
            current_macd = indicators.get('macd', pd.Series()).iloc[-1] if not indicators.get('macd', pd.Series()).empty else 0
            current_rsi_7 = indicators.get('rsi_7', pd.Series()).iloc[-1] if not indicators.get('rsi_7', pd.Series()).empty else 0
            
            # Fetch enhanced open interest data from multiple sources
            open_interest_data = self._fetch_open_interest_from_multiple_sources(symbol)
            
            # Fetch funding rate (still using Binance as primary source)
            funding_rate = self._fetch_funding_rate(symbol)
            
            # Fetch Polymarket sentiment data
            polymarket_data = self._fetch_polymarket_data(symbol)
            market_sentiment_score = self._calculate_market_sentiment_score(polymarket_data, symbol)
            
            # Get intraday series data
            intraday_data = self._get_intraday_series(symbol, limit=10)
            
            # Get longer-term context
            longer_term_data = self._get_longer_term_context(symbol, limit=10)
            
            # Create ReportData object
            report_data = ReportData(
                symbol=symbol,
                timestamp=datetime.now(),
                current_price=current_price,
                current_ema20=current_ema20,
                current_macd=current_macd,
                current_rsi_7=current_rsi_7,
                open_interest_data=open_interest_data,
                funding_rate=funding_rate,
                polymarket_data=polymarket_data,
                market_sentiment_score=market_sentiment_score,
                mid_prices=intraday_data.get('mid_prices', []),
                ema_20_series=intraday_data.get('ema_20_series', []),
                macd_series=intraday_data.get('macd_series', []),
                rsi_7_series=intraday_data.get('rsi_7_series', []),
                rsi_14_series=intraday_data.get('rsi_14_series', []),
                ema_20_4h=longer_term_data.get('ema_20_4h', 0),
                ema_50_4h=longer_term_data.get('ema_50_4h', 0),
                atr_3_4h=longer_term_data.get('atr_3_4h', 0),
                atr_14_4h=longer_term_data.get('atr_14_4h', 0),
                current_volume=longer_term_data.get('current_volume', 0),
                average_volume=longer_term_data.get('average_volume', 0),
                macd_4h_series=longer_term_data.get('macd_4h_series', []),
                rsi_14_4h_series=longer_term_data.get('rsi_14_4h_series', [])
            )
            
            logger.info(f"Successfully crawled report data for {symbol}")
            return report_data
            
        except Exception as e:
            logger.error(f"Error crawling report data for {symbol}: {e}")
            return None
    
    def generate_report_text(self, report_data: ReportData) -> str:
        """Generate formatted report text in the specified format
        
        Args:
            report_data: ReportData object with all required information
            
        Returns:
            Formatted report string
        """
        try:
            if not report_data:
                return "Error: No report data available"
            
            # Format current values
            current_values = (
                f"current_price = {report_data.current_price:.1f}, "
                f"current_ema20 = {report_data.current_ema20:.3f}, "
                f"current_macd = {report_data.current_macd:.3f}, "
                f"current_rsi (7 period) = {report_data.current_rsi_7:.3f}"
            )
            
            # Format enhanced open interest data from multiple sources
            oi_section = "Open Interest (Multiple Sources):\n"
            if report_data.open_interest_data:
                for oi_data in report_data.open_interest_data:
                    try:
                        open_interest_val = float(oi_data.open_interest) if oi_data.open_interest else 0
                        change_percent_val = float(oi_data.change_percent) if oi_data.change_percent else 0
                        oi_section += f"  {oi_data.source.upper()}: {open_interest_val:,.2f} "
                        if change_percent_val != 0:
                            oi_section += f"(24h: {change_percent_val:+.2f}%)\n"
                        else:
                            oi_section += "\n"
                    except (ValueError, TypeError) as e:
                        logger.warning(f"Error formatting open interest data: {e}")
                        oi_section += f"  {oi_data.source.upper()}: Error formatting data\n"
            else:
                oi_section += "  No open interest data available\n"
            
            # Format funding rate
            funding_section = f"Funding Rate: {report_data.funding_rate:.2e}\n"
            
            # Format Polymarket sentiment data
            sentiment_section = ""
            if report_data.polymarket_data:
                sentiment_section = f"\nPolymarket Sentiment Analysis:\n"
                sentiment_section += f"Market Sentiment Score: {report_data.market_sentiment_score:.3f} "
                if report_data.market_sentiment_score > 0.1:
                    sentiment_section += "(Bullish)\n"
                elif report_data.market_sentiment_score < -0.1:
                    sentiment_section += "(Bearish)\n"
                else:
                    sentiment_section += "(Neutral)\n"
                
                sentiment_section += f"Active Markets: {len(report_data.polymarket_data)}\n"
                for market in report_data.polymarket_data[:3]:  # Show top 3 markets
                    sentiment_section += f"  - {market.question[:50]}... (Prob: {market.probability:.2f})\n"
            else:
                sentiment_section = "\nPolymarket Sentiment: No data available\n"
            
            # Format intraday series
            intraday_section = f"""Intraday series (by minute, oldest → latest):

Mid prices: {report_data.mid_prices}
EMA indicators (20‑period): {report_data.ema_20_series}
MACD indicators: {report_data.macd_series}
RSI indicators (7‑Period): {report_data.rsi_7_series}
RSI indicators (14‑Period): {report_data.rsi_14_series}"""
            
            # Format longer-term context
            longer_term_section = f"""Longer‑term context (4‑hour timeframe):

20‑Period EMA: {report_data.ema_20_4h:.3f} vs. 50‑Period EMA: {report_data.ema_50_4h:.3f}
3‑Period ATR: {report_data.atr_3_4h:.3f} vs. 14‑Period ATR: {report_data.atr_14_4h:.3f}
Current Volume: {report_data.current_volume:.3f} vs. Average Volume: {report_data.average_volume:.3f}
MACD indicators: {report_data.macd_4h_series}
RSI indicators (14‑Period): {report_data.rsi_14_4h_series}"""
            
            # Combine all sections
            report_text = f"""ALL OF THE PRICE OR SIGNAL DATA BELOW IS ORDERED: OLDEST → NEWEST

Timeframes note: Unless stated otherwise in a section title, intraday series are provided at 3‑minute intervals. If a coin uses a different interval, it is explicitly stated in that coin's section.

{current_values}

In addition, here is the latest {report_data.symbol} open interest and funding rate for perps (the instrument you are trading):

{oi_section}
{funding_section}{sentiment_section}

{intraday_section}

{longer_term_section}"""
            
            return report_text
            
        except Exception as e:
            logger.error(f"Error generating report text: {e}")
            return f"Error generating report: {e}"
    
    def crawl_and_generate_report(self, symbol: str) -> Optional[str]:
        """Crawl data and generate formatted report in one call
        
        Args:
            symbol: Trading pair symbol (e.g., 'BTC/USDT')
            
        Returns:
            Formatted report string, or None if failed
        """
        try:
            report_data = self.crawl_report_data(symbol)
            if report_data:
                return self.generate_report_text(report_data)
            else:
                return None
        except Exception as e:
            logger.error(f"Error in crawl_and_generate_report for {symbol}: {e}")
            return None