diff --git a/realtime.py b/realtime.py
index 5b20662..c60fc3a 100644
--- a/realtime.py
+++ b/realtime.py
@@ -625,14 +625,14 @@ class ExchangeWebSocket:
return self.ws.running if self.ws else False
class CandleCache:
- def __init__(self, max_candles: int = 2000):
+ def __init__(self, max_candles: int = 5000):
self.candles = {
'1s': deque(maxlen=max_candles),
'1m': deque(maxlen=max_candles),
'1h': deque(maxlen=max_candles),
'1d': deque(maxlen=max_candles)
}
- logger.info("Initialized CandleCache with max candles: {}".format(max_candles))
+ logger.info(f"Initialized CandleCache with max candles: {max_candles}")
def add_candles(self, interval: str, new_candles: pd.DataFrame):
if interval in self.candles and not new_candles.empty:
@@ -645,8 +645,27 @@ class CandleCache:
def get_recent_candles(self, interval: str, count: int = 500) -> pd.DataFrame:
if interval in self.candles and self.candles[interval]:
# Convert deque to list of dicts first
- recent_candles = list(self.candles[interval])[-count:]
- return pd.DataFrame(recent_candles)
+ all_candles = list(self.candles[interval])
+ # Check if we're requesting more candles than we have
+ if count > len(all_candles):
+ logger.debug(f"Requested {count} candles, but only have {len(all_candles)} for {interval}")
+ count = len(all_candles)
+
+ recent_candles = all_candles[-count:]
+ logger.debug(f"Returning {len(recent_candles)} recent candles for {interval} (requested {count})")
+
+ # Create DataFrame and ensure timestamp is datetime type
+ df = pd.DataFrame(recent_candles)
+ if not df.empty and 'timestamp' in df.columns:
+ try:
+ if not pd.api.types.is_datetime64_any_dtype(df['timestamp']):
+ df['timestamp'] = pd.to_datetime(df['timestamp'])
+ except Exception as e:
+ logger.warning(f"Error converting timestamps in get_recent_candles: {str(e)}")
+
+ return df
+
+ logger.debug(f"No candles available for {interval}")
return pd.DataFrame()
def update_cache(self, interval: str, new_candles: pd.DataFrame):
@@ -707,9 +726,10 @@ class RealTimeChart:
'1h': None,
'1d': None
}
- self.candle_cache = CandleCache() # Initialize local candle cache
+ self.candle_cache = CandleCache(max_candles=5000) # Increase max candles to 5000 for better history
self.historical_data = BinanceHistoricalData() # For fetching historical data
self.last_cache_save_time = time.time() # Track last time we saved cache to disk
+ self.first_render = True # Flag to track first render
logger.info(f"Initializing RealTimeChart for {symbol}")
# Load historical data for longer timeframes at startup
@@ -717,6 +737,9 @@ class RealTimeChart:
# Setup the multi-page layout
self._setup_app_layout()
+
+ # Save the loaded cache immediately to ensure we have the data saved
+ self._save_candles_to_disk(force=True)
def _setup_app_layout(self):
# Button style
@@ -1034,8 +1057,29 @@ class RealTimeChart:
interval = interval_data.get('interval', 1)
logger.debug(f"Updating chart for {self.symbol} with interval {interval}s")
- # Get candlesticks from tick storage
+ # First render flag - used to force cache loading
+ is_first_render = self.first_render
+ if is_first_render:
+ logger.info("First render - ensuring cached data is loaded")
+ self.first_render = False
+
+ # Check if we have cached data for the requested interval
+ cached_candles = None
+ if interval == 1 and self.ohlcv_cache['1s'] is not None and not self.ohlcv_cache['1s'].empty:
+ cached_candles = self.ohlcv_cache['1s']
+ logger.info(f"We have {len(cached_candles)} cached 1s candles available")
+ elif interval == 60 and self.ohlcv_cache['1m'] is not None and not self.ohlcv_cache['1m'].empty:
+ cached_candles = self.ohlcv_cache['1m']
+ logger.info(f"We have {len(cached_candles)} cached 1m candles available")
+
+ # Get candlesticks from tick storage for real-time data
df = self.tick_storage.get_candles(interval_seconds=interval)
+ logger.debug(f"Got {len(df) if not df.empty else 0} candles from tick storage")
+
+ # If we don't have real-time data yet, use cached data
+ if df.empty and cached_candles is not None and not cached_candles.empty:
+ df = cached_candles
+ logger.info(f"Using {len(df)} cached candles for main chart")
# Get current price and stats using our enhanced methods
current_price = self.tick_storage.get_latest_price()
@@ -1043,29 +1087,37 @@ class RealTimeChart:
time_stats = self.tick_storage.get_time_based_stats()
# Periodically save candles to disk
- if n % 60 == 0: # Every 60 chart updates (~ every 30 seconds at 500ms interval)
+ if n % 60 == 0 or is_first_render: # Every 60 chart updates or on first render
self._save_candles_to_disk()
logger.debug(f"Current price: {current_price}, Stats: {price_stats}")
+ # Create subplot layout - don't include 1s since that's the main chart
fig = make_subplots(
- rows=6, cols=1, # Adjusted to accommodate new subcharts
- vertical_spacing=0.05, # Reduced for better use of vertical space
- subplot_titles=(f'{self.symbol} Price ({interval}s)', 'Volume', '1s OHLCV', '1m OHLCV', '1h OHLCV', '1d OHLCV'),
- row_heights=[0.3, 0.15, 0.15, 0.15, 0.15, 0.15] # Give more space to main chart
+ rows=5, cols=1,
+ vertical_spacing=0.05,
+ subplot_titles=(f'{self.symbol} Price ({interval}s)', 'Volume', '1m OHLCV', '1h OHLCV', '1d OHLCV'),
+ row_heights=[0.2, 0.2, 0.2, 0.2, 0.2]
)
+ # Process and add main chart data
if not df.empty and len(df) > 0:
- logger.debug(f"Candles dataframe shape: {df.shape}, columns: {df.columns.tolist()}")
+ # Limit how many candles we display for better performance
+ display_df = df
+ if len(df) > 500:
+ logger.debug(f"Limiting main chart display from {len(df)} to 500 candles")
+ display_df = df.tail(500)
+
+ logger.debug(f"Displaying {len(display_df)} candles in main chart")
# Add candlestick chart
fig.add_trace(
go.Candlestick(
- x=df['timestamp'],
- open=df['open'],
- high=df['high'],
- low=df['low'],
- close=df['close'],
+ x=display_df['timestamp'],
+ open=display_df['open'],
+ high=display_df['high'],
+ low=display_df['low'],
+ close=display_df['close'],
name='Price',
increasing_line_color='#33CC33', # Green
decreasing_line_color='#FF4136' # Red
@@ -1075,12 +1127,12 @@ class RealTimeChart:
# Add volume bars
colors = ['#33CC33' if close >= open else '#FF4136'
- for close, open in zip(df['close'], df['open'])]
+ for close, open in zip(display_df['close'], display_df['open'])]
fig.add_trace(
go.Bar(
- x=df['timestamp'],
- y=df['volume'],
+ x=display_df['timestamp'],
+ y=display_df['volume'],
name='Volume',
marker_color=colors
),
@@ -1088,12 +1140,12 @@ class RealTimeChart:
)
# Add latest price line from the candlestick data
- latest_price = df['close'].iloc[-1]
+ latest_price = display_df['close'].iloc[-1]
fig.add_shape(
type="line",
- x0=df['timestamp'].min(),
+ x0=display_df['timestamp'].min(),
y0=latest_price,
- x1=df['timestamp'].max(),
+ x1=display_df['timestamp'].max(),
y1=latest_price,
line=dict(color="yellow", width=1, dash="dash"),
row=1, col=1
@@ -1101,7 +1153,7 @@ class RealTimeChart:
# Annotation for last candle close price
fig.add_annotation(
- x=df['timestamp'].max(),
+ x=display_df['timestamp'].max(),
y=latest_price,
text=f"{latest_price:.2f}",
showarrow=False,
@@ -1115,9 +1167,9 @@ class RealTimeChart:
# Add current price line
fig.add_shape(
type="line",
- x0=df['timestamp'].min(),
+ x0=display_df['timestamp'].min(),
y0=current_price,
- x1=df['timestamp'].max(),
+ x1=display_df['timestamp'].max(),
y1=current_price,
line=dict(color="cyan", width=1, dash="dot"),
row=1, col=1
@@ -1125,7 +1177,7 @@ class RealTimeChart:
# Add current price annotation
fig.add_annotation(
- x=df['timestamp'].max(),
+ x=display_df['timestamp'].max(),
y=current_price,
text=f"Current: {current_price:.2f}",
showarrow=False,
@@ -1135,22 +1187,40 @@ class RealTimeChart:
row=1, col=1
)
- # Fetch and cache OHLCV data for different intervals
- for interval_key in self.ohlcv_cache.keys():
+ # Update candle cache for all timeframes if we have new data
+ if not df.empty:
try:
- new_candles = self.tick_storage.get_candles(interval_seconds=self._interval_to_seconds(interval_key))
- if not new_candles.empty:
- # Update the cache with new candles
- self.candle_cache.update_cache(interval_key, new_candles)
- # Get the updated candles from cache for display
- self.ohlcv_cache[interval_key] = self.candle_cache.get_recent_candles(interval_key)
- logger.debug(f"Updated cache for {interval_key}, now has {len(self.ohlcv_cache[interval_key])} candles")
+ # Update the 1s cache with current df (if it's 1s data)
+ if interval == 1:
+ self.candle_cache.update_cache('1s', df)
+ self.ohlcv_cache['1s'] = self.candle_cache.get_recent_candles('1s', count=2000)
+ logger.debug(f"Updated 1s cache, now has {len(self.ohlcv_cache['1s'])} candles")
+
+ # For other intervals, get fresh data from tick storage
+ for interval_key in ['1m', '1h', '1d']:
+ int_seconds = self._interval_to_seconds(interval_key)
+ new_candles = self.tick_storage.get_candles(interval_seconds=int_seconds)
+ if not new_candles.empty:
+ self.candle_cache.update_cache(interval_key, new_candles)
+ self.ohlcv_cache[interval_key] = self.candle_cache.get_recent_candles(interval_key, count=2000)
+ logger.debug(f"Updated cache for {interval_key}, now has {len(self.ohlcv_cache[interval_key])} candles")
except Exception as e:
- logger.error(f"Error updating cache for {interval_key}: {str(e)}")
+ logger.error(f"Error updating candle caches: {str(e)}")
- # Add OHLCV subcharts
- for i, (interval_key, ohlcv_df) in enumerate(self.ohlcv_cache.items(), start=3):
+ # Add OHLCV subcharts for 1m, 1h, 1d (not 1s since it's the main chart)
+ timeframe_map = {
+ '1m': (3, '1 Minute'),
+ '1h': (4, '1 Hour'),
+ '1d': (5, '1 Day')
+ }
+
+ for interval_key, (row_idx, label) in timeframe_map.items():
+ ohlcv_df = self.ohlcv_cache.get(interval_key)
if ohlcv_df is not None and not ohlcv_df.empty:
+ # Limit to last 100 candles to avoid overcrowding
+ if len(ohlcv_df) > 100:
+ ohlcv_df = ohlcv_df.tail(100)
+
fig.add_trace(
go.Candlestick(
x=ohlcv_df['timestamp'],
@@ -1158,24 +1228,91 @@ class RealTimeChart:
high=ohlcv_df['high'],
low=ohlcv_df['low'],
close=ohlcv_df['close'],
- name=f'{interval_key} OHLCV',
+ name=f'{label}',
increasing_line_color='#33CC33',
- decreasing_line_color='#FF4136'
+ decreasing_line_color='#FF4136',
+ showlegend=False
),
- row=i, col=1
+ row=row_idx, col=1
)
+
+ # Add latest price line
+ latest_timeframe_price = ohlcv_df['close'].iloc[-1] if len(ohlcv_df) > 0 else None
+ if latest_timeframe_price:
+ fig.add_shape(
+ type="line",
+ x0=ohlcv_df['timestamp'].min(),
+ y0=latest_timeframe_price,
+ x1=ohlcv_df['timestamp'].max(),
+ y1=latest_timeframe_price,
+ line=dict(color="yellow", width=1, dash="dash"),
+ row=row_idx, col=1
+ )
+
+ # Add price annotation
+ fig.add_annotation(
+ x=ohlcv_df['timestamp'].max(),
+ y=latest_timeframe_price,
+ text=f"{latest_timeframe_price:.2f}",
+ showarrow=False,
+ font=dict(size=12, color="yellow"),
+ xshift=50,
+ row=row_idx, col=1
+ )
else:
# If no data, add a text annotation to the chart
logger.warning(f"No data to display for {self.symbol} - tick count: {len(self.tick_storage.ticks)}")
- # Add a message to the empty chart
+ # Try to display cached data for each timeframe even if main chart is empty
+ timeframe_map = {
+ '1m': (3, '1 Minute'),
+ '1h': (4, '1 Hour'),
+ '1d': (5, '1 Day')
+ }
+
+ has_any_data = False
+ for interval_key, (row_idx, label) in timeframe_map.items():
+ ohlcv_df = self.ohlcv_cache.get(interval_key)
+ if ohlcv_df is not None and not ohlcv_df.empty:
+ has_any_data = True
+ # Limit to last 100 candles
+ if len(ohlcv_df) > 100:
+ ohlcv_df = ohlcv_df.tail(100)
+
+ fig.add_trace(
+ go.Candlestick(
+ x=ohlcv_df['timestamp'],
+ open=ohlcv_df['open'],
+ high=ohlcv_df['high'],
+ low=ohlcv_df['low'],
+ close=ohlcv_df['close'],
+ name=f'{label}',
+ increasing_line_color='#33CC33',
+ decreasing_line_color='#FF4136',
+ showlegend=False
+ ),
+ row=row_idx, col=1
+ )
+
+ # Add a message to the empty main chart
fig.add_annotation(
x=0.5, y=0.5,
- text=f"Waiting for {self.symbol} data...",
+ text=f"Waiting for {self.symbol} real-time data...",
showarrow=False,
font=dict(size=20, color="white"),
- xref="paper", yref="paper"
+ xref="paper", yref="paper",
+ row=1, col=1
)
+
+ if not has_any_data:
+ # If no data at all, show message
+ fig.add_annotation(
+ x=0.5, y=0.5,
+ text=f"No data available for {self.symbol}",
+ showarrow=False,
+ font=dict(size=20, color="white"),
+ xref="paper", yref="paper"
+ )
# Build info box text with all the statistics
info_lines = [f"{self.symbol}"]
@@ -1216,6 +1353,12 @@ class RealTimeChart:
change_pct = (price_change / stats['min_price']) * 100 if stats['min_price'] > 0 else 0
info_lines.append(f" Range: {stats['min_price']:.2f}-{stats['max_price']:.2f} ({change_pct:.2f}%)")
+ # Add cache information
+ info_lines.append("Cached Candles:")
+ for interval_key, cache_df in self.ohlcv_cache.items():
+ count = len(cache_df) if cache_df is not None else 0
+ info_lines.append(f"{interval_key}: {count}")
+
# Add info box to the chart
fig.add_annotation(
x=0.01,
@@ -1879,31 +2022,45 @@ class RealTimeChart:
'1d': 86400
}
+ # Track load status
+ load_status = {interval: False for interval in intervals.keys()}
+
# First try to load from local cache files
+ logger.info("Step 1: Loading from local cache files...")
for interval_key, interval_seconds in intervals.items():
try:
cache_file = os.path.join(self.historical_data.cache_dir,
f"{self.symbol.replace('/', '_')}_{interval_key}_candles.csv")
+ logger.info(f"Checking for cached {interval_key} data at {cache_file}")
if os.path.exists(cache_file):
# Check if cache is fresh (less than 1 day old for anything but 1d, 3 days for 1d)
file_age = time.time() - os.path.getmtime(cache_file)
max_age = 259200 if interval_key == '1d' else 86400 # 3 days for 1d, 1 day for others
+ logger.info(f"Cache file age: {file_age:.1f}s, max allowed: {max_age}s")
if file_age <= max_age:
+ logger.info(f"Loading {interval_key} candles from cache")
cached_df = pd.read_csv(cache_file)
if not cached_df.empty:
+ # Diagnostic info about the loaded data
+ logger.info(f"Loaded {len(cached_df)} candles from {cache_file}")
+ logger.info(f"Columns: {cached_df.columns.tolist()}")
+ logger.info(f"First few rows: {cached_df.head(2).to_dict('records')}")
+
# Convert timestamp string back to datetime
if 'timestamp' in cached_df.columns:
try:
- cached_df['timestamp'] = pd.to_datetime(cached_df['timestamp'])
- except:
- # If conversion fails, it might already be in the right format
- pass
+ if not pd.api.types.is_datetime64_any_dtype(cached_df['timestamp']):
+ cached_df['timestamp'] = pd.to_datetime(cached_df['timestamp'])
+ logger.info("Successfully converted timestamps to datetime")
+ except Exception as e:
+ logger.warning(f"Could not convert timestamp column for {interval_key}: {str(e)}")
- # Only keep the last 500 candles
- if len(cached_df) > 500:
- cached_df = cached_df.tail(500)
+ # Only keep the last 2000 candles for memory efficiency
+ if len(cached_df) > 2000:
+ cached_df = cached_df.tail(2000)
+ logger.info(f"Truncated to last 2000 candles")
# Add to cache
for _, row in cached_df.iterrows():
@@ -1911,19 +2068,29 @@ class RealTimeChart:
self.candle_cache.candles[interval_key].append(candle_dict)
# Update ohlcv_cache
- self.ohlcv_cache[interval_key] = self.candle_cache.get_recent_candles(interval_key)
- logger.info(f"Loaded {len(cached_df)} cached {interval_key} candles from disk")
+ self.ohlcv_cache[interval_key] = self.candle_cache.get_recent_candles(interval_key, count=2000)
+ logger.info(f"Successfully loaded {len(self.ohlcv_cache[interval_key])} cached {interval_key} candles")
- # Skip fetching from API if we loaded from cache (except for 1d timeframe which we always refresh)
- if interval_key != '1d' and interval_key != '1h':
- continue
+ if len(self.ohlcv_cache[interval_key]) >= 500:
+ load_status[interval_key] = True
+ # Skip fetching from API if we loaded from cache (except for 1d timeframe which we always refresh)
+ if interval_key != '1d':
+ continue
+ else:
+ logger.info(f"Cache file for {interval_key} is too old ({file_age:.1f}s)")
+ else:
+ logger.info(f"No cache file found for {interval_key}")
except Exception as e:
logger.error(f"Error loading cached {interval_key} candles: {str(e)}")
+ import traceback
+ logger.error(traceback.format_exc())
# For timeframes other than 1s, fetch from API as backup or for fresh data
+ logger.info("Step 2: Fetching data from API for missing timeframes...")
for interval_key, interval_seconds in intervals.items():
# Skip 1s for API requests
- if interval_key == '1s':
+ if interval_key == '1s' or load_status[interval_key]:
+ logger.info(f"Skipping API fetch for {interval_key}: already loaded or 1s timeframe")
continue
# Fetch historical data from API
@@ -1958,24 +2125,38 @@ class RealTimeChart:
self.candle_cache.candles[interval_key].append(candle_dict)
# Update ohlcv_cache with combined data
- self.ohlcv_cache[interval_key] = self.candle_cache.get_recent_candles(interval_key)
+ self.ohlcv_cache[interval_key] = self.candle_cache.get_recent_candles(interval_key, count=2000)
logger.info(f"Total {interval_key} candles in cache: {len(self.ohlcv_cache[interval_key])}")
+
+ if len(self.ohlcv_cache[interval_key]) >= 500:
+ load_status[interval_key] = True
else:
logger.warning(f"No historical data available from API for {self.symbol} {interval_key}")
except Exception as e:
logger.error(f"Error fetching {interval_key} data from API: {str(e)}")
+ import traceback
+ logger.error(traceback.format_exc())
+
+ # Log summary of loaded data
+ logger.info("Historical data load summary:")
+ for interval_key in intervals.keys():
+ count = len(self.ohlcv_cache[interval_key]) if self.ohlcv_cache[interval_key] is not None else 0
+ status = "Success" if load_status[interval_key] else "Failed"
+ if count > 0 and count < 500:
+ status = "Partial"
+ logger.info(f"{interval_key}: {count} candles - {status}")
except Exception as e:
logger.error(f"Error in _load_historical_data: {str(e)}")
import traceback
logger.error(traceback.format_exc())
- def _save_candles_to_disk(self):
+ def _save_candles_to_disk(self, force=False):
"""Save current candle cache to disk for persistence between runs"""
try:
# Only save if we have data and sufficient time has passed (every 5 minutes)
current_time = time.time()
- if current_time - self.last_cache_save_time < 300: # 5 minutes
+ if not force and current_time - self.last_cache_save_time < 300: # 5 minutes
return
# Save each timeframe's candles to disk
@@ -1984,6 +2165,14 @@ class RealTimeChart:
# Convert to DataFrame
df = pd.DataFrame(list(candles))
if not df.empty:
+ # Ensure timestamp is properly formatted
+ if 'timestamp' in df.columns:
+ try:
+ if not pd.api.types.is_datetime64_any_dtype(df['timestamp']):
+ df['timestamp'] = pd.to_datetime(df['timestamp'])
+ except:
+ logger.warning(f"Could not convert timestamp column for {interval_key}")
+
# Save to disk in the cache directory
cache_file = os.path.join(self.historical_data.cache_dir,
f"{self.symbol.replace('/', '_')}_{interval_key}_candles.csv")
@@ -1994,6 +2183,8 @@ class RealTimeChart:
logger.info(f"Saved all candle caches to disk at {datetime.now()}")
except Exception as e:
logger.error(f"Error saving candles to disk: {str(e)}")
+ import traceback
+ logger.error(traceback.format_exc())
async def main():