popov/gogo2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sqlite for checkpoints, cleanup

Browse Source
This commit is contained in:
Dobromir Popov
2025-07-25 22:34:13 +03:00
parent 130a52fb9b
commit dd9f4b63ba
42 changed files with 2017 additions and 1485 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
core/data_provider.py
View File

@ -201,6 +201,9 @@ class DataProvider:
self.last_pivot_calculation: Dict[str, datetime] = {}
self.pivot_calculation_interval = timedelta(minutes=5) # Recalculate every 5 minutes
# Auto-fix corrupted cache files on startup
self._auto_fix_corrupted_cache()
# Load existing pivot bounds from cache
self._load_all_pivot_bounds()
@ -1231,6 +1234,36 @@ class DataProvider:
return symbol # Return first symbol for now - can be improved
return None
# === CACHE MANAGEMENT ===
def _auto_fix_corrupted_cache(self):
"""Automatically fix corrupted cache files on startup"""
try:
from utils.cache_manager import get_cache_manager
cache_manager = get_cache_manager()
# Quick health check
health_summary = cache_manager.get_cache_summary()
if health_summary['corrupted_files'] > 0:
logger.warning(f"Found {health_summary['corrupted_files']} corrupted cache files, cleaning up...")
# Auto-cleanup corrupted files (no confirmation needed)
deleted_files = cache_manager.cleanup_corrupted_files(dry_run=False)
deleted_count = 0
for cache_dir, files in deleted_files.items():
for file_info in files:
if "DELETED:" in file_info:
deleted_count += 1
logger.info(f"Auto-cleaned {deleted_count} corrupted cache files")
else:
logger.debug("Cache health check passed - no corrupted files found")
except Exception as e:
logger.warning(f"Cache auto-fix failed: {e}")
# === PIVOT BOUNDS CACHING ===
def _load_all_pivot_bounds(self):
@ -1285,13 +1318,25 @@ class DataProvider:
logger.info(f"Loaded {len(df)} 1m candles from cache for {symbol}")
return df
except Exception as parquet_e:
# Handle corrupted Parquet file
if "Parquet magic bytes not found" in str(parquet_e) or "corrupted" in str(parquet_e).lower():
# Handle corrupted Parquet file - expanded error detection
error_str = str(parquet_e).lower()
corrupted_indicators = [
"parquet magic bytes not found",
"corrupted",
"couldn't deserialize thrift",
"don't know what type",
"invalid parquet file",
"unexpected end of file",
"invalid metadata"
]
if any(indicator in error_str for indicator in corrupted_indicators):
logger.warning(f"Corrupted Parquet cache file for {symbol}, removing and returning None: {parquet_e}")
try:
cache_file.unlink() # Delete corrupted file
except Exception:
pass
logger.info(f"Deleted corrupted monthly cache file: {cache_file}")
except Exception as delete_e:
logger.error(f"Failed to delete corrupted monthly cache file: {delete_e}")
return None
else:
raise parquet_e
@ -1393,13 +1438,25 @@ class DataProvider:
logger.debug(f"Loaded {len(df)} rows from cache for {symbol} {timeframe} (age: {cache_age/60:.1f}min)")
return df
except Exception as parquet_e:
# Handle corrupted Parquet file
if "Parquet magic bytes not found" in str(parquet_e) or "corrupted" in str(parquet_e).lower():
# Handle corrupted Parquet file - expanded error detection
error_str = str(parquet_e).lower()
corrupted_indicators = [
"parquet magic bytes not found",
"corrupted",
"couldn't deserialize thrift",
"don't know what type",
"invalid parquet file",
"unexpected end of file",
"invalid metadata"
]
if any(indicator in error_str for indicator in corrupted_indicators):
logger.warning(f"Corrupted Parquet cache file for {symbol} {timeframe}, removing and returning None: {parquet_e}")
try:
cache_file.unlink() # Delete corrupted file
except Exception:
pass
logger.info(f"Deleted corrupted cache file: {cache_file}")
except Exception as delete_e:
logger.error(f"Failed to delete corrupted cache file: {delete_e}")
return None
else:
raise parquet_e