popov/gogo2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

charts working with cahe historical data

Browse Source
This commit is contained in:
Dobromir Popov 2025-04-02 11:44:21 +03:00
parent 938eef8bc9
commit aec536d007
1 changed files with 165 additions and 89 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

254
realtime.py
View File

@ -134,8 +134,18 @@ class BinanceHistoricalData:
try: try:
cache_file = self._get_cache_filename(symbol, interval) cache_file = self._get_cache_filename(symbol, interval)
if os.path.exists(cache_file): if os.path.exists(cache_file):
# For 1s interval, check if the cache is recent (less than 10 minutes old)
if interval == "1s" or interval == 1:
file_mod_time = datetime.fromtimestamp(os.path.getmtime(cache_file))
time_diff = (datetime.now() - file_mod_time).total_seconds() / 60
if time_diff > 10:
logger.info("1s cache is older than 10 minutes, skipping load")
return None
logger.info(f"Using recent 1s cache (age: {time_diff:.1f} minutes)")
df = pd.read_csv(cache_file) df = pd.read_csv(cache_file)
df["timestamp"] = pd.to_datetime(df["timestamp"]) df["timestamp"] = pd.to_datetime(df["timestamp"])
logger.info(f"Loaded {len(df)} candles from cache: {cache_file}")
return df return df
except Exception as e: except Exception as e:
logger.error(f"Error loading cached data: {str(e)}") logger.error(f"Error loading cached data: {str(e)}")
@ -411,6 +421,12 @@ class TickStorage:
# Update all timeframe candles # Update all timeframe candles
self._update_all_candles(tick) self._update_all_candles(tick)
# Verify 1s candles are being updated - periodically log for monitoring
if len(self.ticks) % 100 == 0:
logger.debug(f"Tick count: {len(self.ticks)}, 1s candles count: {len(self.candles['1s'])}")
return tick
def get_latest_price(self): def get_latest_price(self):
"""Get the latest price""" """Get the latest price"""
return self.latest_price return self.latest_price
@ -455,6 +471,10 @@ class TickStorage:
current_candle['close'] = tick['price'] current_candle['close'] = tick['price']
current_candle['volume'] += tick['volume'] current_candle['volume'] += tick['volume']
# For 1s timeframe specifically, log a debug message to confirm updates are occurring
if interval_key == '1s':
logger.debug(f"Updated 1s candle at {current_candle['timestamp']} with price {tick['price']}")
# Limit the number of candles to keep for each timeframe # Limit the number of candles to keep for each timeframe
# Keep more candles for shorter timeframes, fewer for longer ones # Keep more candles for shorter timeframes, fewer for longer ones
max_candles = { max_candles = {
@ -495,6 +515,10 @@ class TickStorage:
'volume': 0 'volume': 0
} }
# Log when we're creating a new 1s candle for debugging
if interval_key == '1s':
logger.debug(f"Creating new 1s candle at {candle_start}")
self.candles[interval_key].append(candle) self.candles[interval_key].append(candle)
return candle return candle
@ -602,12 +626,17 @@ class TickStorage:
def load_historical_data(self, historical_data, symbol): def load_historical_data(self, historical_data, symbol):
"""Load historical data for all timeframes""" """Load historical data for all timeframes"""
try: try:
# Load data for different timeframes # Clear any existing 1s candles to prevent using old cached data
self.candles['1s'] = []
# Clear tick data to ensure we start with an empty collection
self.ticks = []
# Load data for different timeframes (without 1s - we'll handle it separately)
timeframes = [ timeframes = [
(1, '1s'), # 1 second - limit to 20 minutes (1200 seconds) (60, '1m'), # 1 minute
(5, '5s'), # 5 seconds (5, '5s'), # 5 seconds
(15, '15s'), # 15 seconds (15, '15s'), # 15 seconds
(60, '1m'), # 1 minute
(300, '5m'), # 5 minutes (300, '5m'), # 5 minutes
(900, '15m'), # 15 minutes (900, '15m'), # 15 minutes
(3600, '1h'), # 1 hour (3600, '1h'), # 1 hour
@ -615,12 +644,35 @@ class TickStorage:
(86400, '1d') # 1 day (86400, '1d') # 1 day
] ]
# For 1s, we only load from cache if available (handled in _load_from_cache method)
# The _load_from_cache method will check if cache is no more than 10 minutes old
df_1s = historical_data.get_historical_candles(symbol, 1, 300) # Try to get 1s data from cache
if df_1s is not None and not df_1s.empty:
logger.info(f"Loaded {len(df_1s)} recent 1s candles from cache")
# Convert to our candle format and store
candles_1s = []
for _, row in df_1s.iterrows():
candle = {
'timestamp': row['timestamp'],
'open': row['open'],
'high': row['high'],
'low': row['low'],
'close': row['close'],
'volume': row['volume']
}
candles_1s.append(candle)
# Add the 1s candles to our candles storage
self.candles['1s'] = candles_1s
else:
logger.info("No recent 1s cache available, starting with empty 1s data")
# Load the remaining timeframes normally
for interval_seconds, interval_key in timeframes: for interval_seconds, interval_key in timeframes:
# Set appropriate limits based on timeframe # Set appropriate limits based on timeframe
limit = 1000 # Default limit = 1000 # Default
if interval_seconds == 1: if interval_seconds < 60:
limit = 1200 # 1s data - limit to 20 minutes as requested
elif interval_seconds < 60:
limit = 500 # For seconds-level data limit = 500 # For seconds-level data
elif interval_seconds < 300: elif interval_seconds < 300:
limit = 1000 # 1m limit = 1000 # 1m
@ -632,81 +684,32 @@ class TickStorage:
limit = 200 # hourly/daily data limit = 200 # hourly/daily data
try: try:
# For 1s data, we might need to generate it from 1m data # Load normal historical data
if interval_seconds == 1: df = historical_data.get_historical_candles(symbol, interval_seconds, limit)
# Get 1m data first if df is not None and not df.empty:
df_1m = historical_data.get_historical_candles(symbol, 60, 60) # Get 60 minutes of 1m data logger.info(f"Loaded {len(df)} historical candles for {symbol} ({interval_key})")
if df_1m is not None and not df_1m.empty:
# Create simulated 1s data from 1m data # Convert to our candle format and store
simulated_1s = [] candles = []
for _, row in df_1m.iterrows(): for _, row in df.iterrows():
# For each 1m candle, create 60 1s candles candle = {
start_time = row['timestamp'] 'timestamp': row['timestamp'],
for i in range(60): 'open': row['open'],
# Calculate second-level timestamp 'high': row['high'],
second_time = start_time + timedelta(seconds=i) 'low': row['low'],
'close': row['close'],
# Create candle with random price movement around close price 'volume': row['volume']
close_price = row['close'] }
price_range = (row['high'] - row['low']) / 60 # Reduced range candles.append(candle)
# Interpolate price - gradual movement from open to close # Set the candles for this timeframe
progress = i / 60 self.candles[interval_key] = candles
interp_price = row['open'] + (row['close'] - row['open']) * progress
# No longer load 1m data into ticks collection, as this persists the problem
# Add some small random movement # Just store the latest price from the most recent candle for reference
random_factor = np.random.normal(0, price_range * 0.5) if interval_key == '1m' and candles:
s_price = max(0, interp_price + random_factor) self.latest_price = candles[-1]['close']
logger.info(f"Set latest price to ${self.latest_price:.2f} from historical data without adding to ticks")
# Create 1s candle
s_candle = {
'timestamp': second_time,
'open': s_price,
'high': s_price * 1.0001, # Tiny movement
'low': s_price * 0.9999, # Tiny movement
'close': s_price,
'volume': row['volume'] / 60 # Distribute volume
}
simulated_1s.append(s_candle)
# Add the simulated 1s candles to our candles storage
self.candles['1s'] = simulated_1s
logger.info(f"Generated {len(simulated_1s)} simulated 1s candles for {symbol}")
else:
# Load normal historical data
df = historical_data.get_historical_candles(symbol, interval_seconds, limit)
if df is not None and not df.empty:
logger.info(f"Loaded {len(df)} historical candles for {symbol} ({interval_key})")
# Convert to our candle format and store
candles = []
for _, row in df.iterrows():
candle = {
'timestamp': row['timestamp'],
'open': row['open'],
'high': row['high'],
'low': row['low'],
'close': row['close'],
'volume': row['volume']
}
candles.append(candle)
# Set the candles for this timeframe
self.candles[interval_key] = candles
# For 1m data, also use it to generate tick data
if interval_key == '1m':
for candle in candles[-20:]: # Use only the last 20 candles for tick data
self.add_tick(
price=candle['close'],
volume=candle['volume'] / 10, # Distribute volume
timestamp=candle['timestamp']
)
# Set latest price from most recent candle
if candles:
self.latest_price = candles[-1]['close']
logger.info(f"Set latest price to ${self.latest_price:.2f} from historical data")
except Exception as e: except Exception as e:
logger.error(f"Error loading {interval_key} data: {e}") logger.error(f"Error loading {interval_key} data: {e}")
@ -767,8 +770,11 @@ class RealTimeChart:
self.latest_price = None self.latest_price = None
self.latest_volume = None self.latest_volume = None
self.latest_timestamp = None self.latest_timestamp = None
self.positions = [] # List to store positions
self.accumulative_pnl = 0.0 # Track total PnL # Initialize with empty positions list to prevent old trade actions from affecting chart resizing
# We MUST start with a clean state for each new chart instance
self.positions = [] # Empty positions list - CRITICAL for proper chart resizing
self.accumulative_pnl = 0.0 # Reset PnL
self.current_balance = 100.0 # Start with $100 balance self.current_balance = 100.0 # Start with $100 balance
# Store historical data for different timeframes # Store historical data for different timeframes
@ -992,8 +998,8 @@ class RealTimeChart:
interval_key = self._get_interval_key(interval) interval_key = self._get_interval_key(interval)
# Make sure we have data for this interval # Make sure we have data for this interval
if interval_key not in self.tick_storage.candles or not self.tick_storage.candles[interval_key]: if interval_key not in self.tick_storage.candles:
logger.warning(f"No candle data available for {interval_key}") logger.warning(f"No candle data structure available for {interval_key}")
# Return empty figure with a message # Return empty figure with a message
fig = go.Figure() fig = go.Figure()
fig.add_annotation( fig.add_annotation(
@ -1003,6 +1009,26 @@ class RealTimeChart:
) )
fig.update_layout(title=f"{self.symbol} - {interval_key}") fig.update_layout(title=f"{self.symbol} - {interval_key}")
return fig return fig
# For 1s specifically, log more debug info
if interval_key == '1s':
logger.info(f"1s candles count: {len(self.tick_storage.candles[interval_key])}")
logger.info(f"Ticks count: {len(self.tick_storage.ticks)}")
if not self.tick_storage.candles[interval_key]:
logger.warning("No 1s candles available - this may indicate the WebSocket isn't sending data, or candles aren't being created")
# Check if we have any candles for this interval
if not self.tick_storage.candles[interval_key]:
logger.warning(f"No candle data available for {interval_key}")
# Return empty figure with a message
fig = go.Figure()
fig.add_annotation(
text=f"No data available for {interval_key}. Waiting for real-time data...",
xref="paper", yref="paper",
x=0.5, y=0.5, showarrow=False
)
fig.update_layout(title=f"{self.symbol} - {interval_key} (waiting for data)")
return fig
# For rendering, limit to the last 500 candles for performance # For rendering, limit to the last 500 candles for performance
candles = self.tick_storage.candles[interval_key][-500:] candles = self.tick_storage.candles[interval_key][-500:]
@ -1047,6 +1073,7 @@ class RealTimeChart:
marker_color='rgba(100, 100, 255, 0.5)' marker_color='rgba(100, 100, 255, 0.5)'
), row=2, col=1) ), row=2, col=1)
# Add trading markers if available # Add trading markers if available
if hasattr(self, 'positions') and self.positions: if hasattr(self, 'positions') and self.positions:
# Get last 100 positions for display (to avoid too many markers) # Get last 100 positions for display (to avoid too many markers)
@ -1095,6 +1122,8 @@ class RealTimeChart:
) )
), row=1, col=1) ), row=1, col=1)
# Update layout # Update layout
fig.update_layout( fig.update_layout(
title=f"{self.symbol} - {interval_key}", title=f"{self.symbol} - {interval_key}",
@ -1348,18 +1377,51 @@ class RealTimeChart:
else: else:
return f"{interval_seconds // 86400}d" return f"{interval_seconds // 86400}d"
def _update_chart_and_positions(self):
"""Update the chart with current data and positions"""
try:
# Force an update of the charts
self._update_main_chart(1) # Update 1s chart by default
self._update_secondary_charts()
logger.debug("Updated charts and positions")
return True
except Exception as e:
logger.error(f"Error updating chart and positions: {str(e)}")
import traceback
logger.error(traceback.format_exc())
return False
async def start_websocket(self): async def start_websocket(self):
"""Start the websocket connection for real-time data""" """Start the websocket connection for real-time data"""
try: try:
# Step 1: Clear everything related to positions FIRST, before any other operations
logger.info(f"Initializing fresh chart for {self.symbol} - clearing all previous positions")
self.positions = [] # Clear positions list
self.accumulative_pnl = 0.0 # Reset accumulated PnL
self.current_balance = 100.0 # Reset balance
# Step 2: Clear any previous tick data to avoid using stale data from previous training sessions
self.tick_storage.ticks = []
# Step 3: Clear any previous 1s candles before loading historical data
self.tick_storage.candles['1s'] = []
logger.info("Initialized empty 1s candles, tick collection, and positions for fresh data")
# Load historical data first to ensure we have candles for all timeframes # Load historical data first to ensure we have candles for all timeframes
logger.info(f"Loading historical data for {self.symbol}") logger.info(f"Loading historical data for {self.symbol}")
# Initialize a BinanceHistoricalData instance # Initialize a BinanceHistoricalData instance
historical_data = BinanceHistoricalData() historical_data = BinanceHistoricalData()
# Load historical data for display # Load historical data for all timeframes (1s will load from cache if recent, otherwise empty)
self.tick_storage.load_historical_data(historical_data, self.symbol) self.tick_storage.load_historical_data(historical_data, self.symbol)
# Double check that we have the 1s timeframe initialized
if '1s' not in self.tick_storage.candles:
self.tick_storage.candles['1s'] = []
logger.info(f"After loading historical data, 1s candles count: {len(self.tick_storage.candles['1s'])}")
# Make sure we update the charts once with historical data before websocket starts # Make sure we update the charts once with historical data before websocket starts
# Update all the charts with the initial historical data # Update all the charts with the initial historical data
self._update_chart_and_positions() self._update_chart_and_positions()
@ -1370,6 +1432,10 @@ class RealTimeChart:
logger.info(f"WebSocket connected for {self.symbol}") logger.info(f"WebSocket connected for {self.symbol}")
# Counter for received ticks
tick_count = 0
last_update_time = time.time()
# Start receiving data # Start receiving data
while self.websocket.running: while self.websocket.running:
try: try:
@ -1377,6 +1443,8 @@ class RealTimeChart:
if data: if data:
# Process the received data # Process the received data
if 'price' in data: if 'price' in data:
tick_count += 1
# Update tick storage # Update tick storage
self.tick_storage.add_tick( self.tick_storage.add_tick(
price=data['price'], price=data['price'],
@ -1389,9 +1457,17 @@ class RealTimeChart:
self.latest_volume = data.get('volume', 0) self.latest_volume = data.get('volume', 0)
self.latest_timestamp = datetime.fromtimestamp(data['timestamp'] / 1000) self.latest_timestamp = datetime.fromtimestamp(data['timestamp'] / 1000)
# Log occasional price updates (every 500 messages) # Force chart update every 5 seconds
if hasattr(self.websocket.ws, 'message_count') and self.websocket.ws.message_count % 500 == 0: current_time = time.time()
logger.info(f"Current {self.symbol} price: ${self.latest_price:.2f}") if current_time - last_update_time >= 5.0:
self._update_chart_and_positions()
last_update_time = current_time
logger.debug("Forced chart update after new ticks")
# Log tick processing for debugging (every 100 ticks)
if tick_count % 100 == 0:
logger.info(f"Processed {tick_count} ticks, current price: ${self.latest_price:.2f}")
logger.info(f"Current 1s candles count: {len(self.tick_storage.candles['1s'])}")
except Exception as e: except Exception as e:
logger.error(f"Error processing websocket data: {str(e)}") logger.error(f"Error processing websocket data: {str(e)}")