popov/gogo2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

UI and stability

Browse Source
This commit is contained in:
Dobromir Popov
2025-07-28 14:05:37 +03:00
parent 25b2d3840a
commit 44821b2a89
7 changed files with 934 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
core/data_provider.py
View File

@ -36,6 +36,12 @@ import math
# Suppress ta library deprecation warnings
warnings.filterwarnings("ignore", category=FutureWarning, module="ta")
# Import timezone utilities
from utils.timezone_utils import (
normalize_timestamp, normalize_dataframe_timestamps, normalize_dataframe_index,
now_system, now_utc, to_sofia, UTC, SOFIA_TZ, log_timezone_info
)
from .config import get_config
from .tick_aggregator import RealTimeTickAggregator, RawTick, OHLCVBar
from .cnn_monitor import log_cnn_prediction
@ -472,7 +478,7 @@ class DataProvider:
# Create raw tick entry
raw_tick = {
'symbol': symbol,
'timestamp': datetime.utcnow(),
'timestamp': now_system(), # Use system timezone consistently
'bids': actual_data.get('bids', [])[:50], # Top 50 levels
'asks': actual_data.get('asks', [])[:50], # Top 50 levels
'stats': actual_data.get('stats', {}),
@ -1097,8 +1103,7 @@ class DataProvider:
# Process columns with proper timezone handling (MEXC returns UTC timestamps)
df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms', utc=True)
# Convert from UTC to Europe/Sofia timezone
df['timestamp'] = df['timestamp'].dt.tz_convert('Europe/Sofia')
# Keep in UTC to match COB WebSocket data (no timezone conversion)
for col in ['open', 'high', 'low', 'close', 'volume']:
df[col] = df[col].astype(float)
@ -1144,18 +1149,16 @@ class DataProvider:
if isinstance(timestamp, (int, float)):
import pytz
utc = pytz.UTC
sofia_tz = pytz.timezone('Europe/Sofia')
tick_time = datetime.fromtimestamp(timestamp, tz=utc)
tick_time = tick_time.astimezone(sofia_tz)
# Keep in UTC to match COB WebSocket data
elif isinstance(timestamp, datetime):
import pytz
sofia_tz = pytz.timezone('Europe/Sofia')
utc = pytz.UTC
tick_time = timestamp
# If no timezone info, assume UTC and convert to Europe/Sofia
# If no timezone info, assume UTC and keep in UTC
if tick_time.tzinfo is None:
utc = pytz.UTC
tick_time = utc.localize(tick_time)
tick_time = tick_time.astimezone(sofia_tz)
# Keep in UTC (no timezone conversion)
else:
continue
@ -1195,15 +1198,15 @@ class DataProvider:
# Convert to DataFrame
df = pd.DataFrame(candles)
# Ensure timestamps are timezone-aware (Europe/Sofia)
# Ensure timestamps are timezone-aware (UTC to match COB WebSocket data)
if not df.empty and 'timestamp' in df.columns:
import pytz
sofia_tz = pytz.timezone('Europe/Sofia')
# If timestamps are not timezone-aware, make them Europe/Sofia
utc = pytz.UTC
# If timestamps are not timezone-aware, make them UTC
if df['timestamp'].dt.tz is None:
df['timestamp'] = df['timestamp'].dt.tz_localize(sofia_tz)
df['timestamp'] = df['timestamp'].dt.tz_localize(utc)
else:
df['timestamp'] = df['timestamp'].dt.tz_convert(sofia_tz)
df['timestamp'] = df['timestamp'].dt.tz_convert(utc)
df = df.sort_values('timestamp').reset_index(drop=True)
@ -1283,8 +1286,8 @@ class DataProvider:
# Process columns with proper timezone handling (Binance returns UTC timestamps)
df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms', utc=True)
# Convert from UTC to Europe/Sofia timezone
df['timestamp'] = df['timestamp'].dt.tz_convert('Europe/Sofia')
# Keep in UTC to match COB WebSocket data (no timezone conversion)
# This prevents the 3-hour gap when appending live COB data
for col in ['open', 'high', 'low', 'close', 'volume']:
df[col] = df[col].astype(float)
@ -1491,9 +1494,8 @@ class DataProvider:
import pytz
utc = pytz.UTC
sofia_tz = pytz.timezone('Europe/Sofia')
end_time = datetime.utcnow().replace(tzinfo=utc).astimezone(sofia_tz)
end_time = datetime.utcnow().replace(tzinfo=utc)
start_time = end_time - timedelta(days=30)
if cached_data is not None and not cached_data.empty:
@ -1596,8 +1598,7 @@ class DataProvider:
# Process columns with proper timezone handling
df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms', utc=True)
# Convert from UTC to Europe/Sofia timezone to match cached data
df['timestamp'] = df['timestamp'].dt.tz_convert('Europe/Sofia')
# Keep in UTC to match COB WebSocket data (no timezone conversion)
for col in ['open', 'high', 'low', 'close', 'volume']:
df[col] = df[col].astype(float)
@ -1669,8 +1670,7 @@ class DataProvider:
# Process columns with proper timezone handling
batch_df['timestamp'] = pd.to_datetime(batch_df['timestamp'], unit='ms', utc=True)
# Convert from UTC to Europe/Sofia timezone to match cached data
batch_df['timestamp'] = batch_df['timestamp'].dt.tz_convert('Europe/Sofia')
# Keep in UTC to match COB WebSocket data (no timezone conversion)
for col in ['open', 'high', 'low', 'close', 'volume']:
batch_df[col] = batch_df[col].astype(float)
@ -2033,15 +2033,14 @@ class DataProvider:
if cache_file.exists():
try:
df = pd.read_parquet(cache_file)
# Ensure cached monthly data has proper timezone (Europe/Sofia)
# Ensure cached monthly data has proper timezone (UTC to match COB WebSocket data)
if not df.empty and 'timestamp' in df.columns:
if df['timestamp'].dt.tz is None:
# If no timezone info, assume UTC and convert to Europe/Sofia
# If no timezone info, assume UTC and keep in UTC
df['timestamp'] = pd.to_datetime(df['timestamp'], utc=True)
df['timestamp'] = df['timestamp'].dt.tz_convert('Europe/Sofia')
elif str(df['timestamp'].dt.tz) != 'Europe/Sofia':
# Convert to Europe/Sofia if different timezone
df['timestamp'] = df['timestamp'].dt.tz_convert('Europe/Sofia')
elif str(df['timestamp'].dt.tz) != 'UTC':
# Convert to UTC if different timezone
df['timestamp'] = df['timestamp'].dt.tz_convert('UTC')
logger.info(f"Loaded {len(df)} 1m candles from cache for {symbol}")
return df
except Exception as parquet_e:
@ -2317,15 +2316,14 @@ class DataProvider:
if cache_age < max_age:
try:
df = pd.read_parquet(cache_file)
# Ensure cached data has proper timezone (Europe/Sofia)
# Ensure cached data has proper timezone (UTC to match COB WebSocket data)
if not df.empty and 'timestamp' in df.columns:
if df['timestamp'].dt.tz is None:
# If no timezone info, assume UTC and convert to Europe/Sofia
# If no timezone info, assume UTC and keep in UTC
df['timestamp'] = pd.to_datetime(df['timestamp'], utc=True)
df['timestamp'] = df['timestamp'].dt.tz_convert('Europe/Sofia')
elif str(df['timestamp'].dt.tz) != 'Europe/Sofia':
# Convert to Europe/Sofia if different timezone
df['timestamp'] = df['timestamp'].dt.tz_convert('Europe/Sofia')
elif str(df['timestamp'].dt.tz) != 'UTC':
# Convert to UTC if different timezone
df['timestamp'] = df['timestamp'].dt.tz_convert('UTC')
logger.debug(f"Loaded {len(df)} rows from cache for {symbol} {timeframe} (age: {cache_age/60:.1f}min)")
return df
except Exception as parquet_e: