sqlite for checkpoints, cleanup
This commit is contained in:
Dobromir Popov
133
fix_cache.py
Normal file
133
fix_cache.py
Normal file
@ -0,0 +1,133 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Cache Fix Script
|
||||
|
||||
Quick script to diagnose and fix cache issues, including the Parquet deserialization error
|
||||
"""
|
||||
|
||||
import sys
|
||||
import logging
|
||||
from utils.cache_manager import get_cache_manager, cleanup_corrupted_cache, get_cache_health
|
||||
|
||||
# Set up logging
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def main():
|
||||
"""Main cache fix routine"""
|
||||
print("=== Trading System Cache Fix ===")
|
||||
print()
|
||||
|
||||
# Get cache manager
|
||||
cache_manager = get_cache_manager()
|
||||
|
||||
# 1. Scan cache health
|
||||
print("1. Scanning cache health...")
|
||||
health_summary = get_cache_health()
|
||||
|
||||
print(f"Total files: {health_summary['total_files']}")
|
||||
print(f"Valid files: {health_summary['valid_files']}")
|
||||
print(f"Corrupted files: {health_summary['corrupted_files']}")
|
||||
print(f"Health percentage: {health_summary['health_percentage']:.1f}%")
|
||||
print(f"Total cache size: {health_summary['total_size_mb']:.1f} MB")
|
||||
print()
|
||||
|
||||
# Show detailed report
|
||||
for cache_dir, report in health_summary['directories'].items():
|
||||
if report['total_files'] > 0:
|
||||
print(f"Directory: {cache_dir}")
|
||||
print(f" Files: {report['valid_files']}/{report['total_files']} valid")
|
||||
print(f" Size: {report['total_size_mb']:.1f} MB")
|
||||
|
||||
if report['corrupted_files'] > 0:
|
||||
print(f" CORRUPTED FILES ({report['corrupted_files']}):")
|
||||
for corrupted in report['corrupted_files_list']:
|
||||
print(f" - {corrupted['file']}: {corrupted['error']}")
|
||||
|
||||
if report['old_files']:
|
||||
print(f" OLD FILES ({len(report['old_files'])}):")
|
||||
for old_file in report['old_files'][:3]: # Show first 3
|
||||
print(f" - {old_file['file']}: {old_file['age_days']} days old")
|
||||
if len(report['old_files']) > 3:
|
||||
print(f" ... and {len(report['old_files']) - 3} more")
|
||||
print()
|
||||
|
||||
# 2. Fix corrupted files
|
||||
if health_summary['corrupted_files'] > 0:
|
||||
print("2. Fixing corrupted files...")
|
||||
|
||||
# First show what would be deleted
|
||||
print("Files that will be deleted:")
|
||||
dry_run_result = cleanup_corrupted_cache(dry_run=True)
|
||||
for cache_dir, files in dry_run_result.items():
|
||||
if files:
|
||||
print(f" {cache_dir}:")
|
||||
for file_info in files:
|
||||
print(f" {file_info}")
|
||||
|
||||
# Ask for confirmation
|
||||
response = input("\nProceed with deletion? (y/N): ").strip().lower()
|
||||
if response == 'y':
|
||||
print("Deleting corrupted files...")
|
||||
actual_result = cleanup_corrupted_cache(dry_run=False)
|
||||
|
||||
deleted_count = 0
|
||||
for cache_dir, files in actual_result.items():
|
||||
for file_info in files:
|
||||
if "DELETED:" in file_info:
|
||||
deleted_count += 1
|
||||
|
||||
print(f"Deleted {deleted_count} corrupted files")
|
||||
else:
|
||||
print("Skipped deletion")
|
||||
else:
|
||||
print("2. No corrupted files found - cache is healthy!")
|
||||
|
||||
print()
|
||||
|
||||
# 3. Optional: Clean old files
|
||||
print("3. Checking for old files...")
|
||||
old_files_result = cache_manager.cleanup_old_files(days_to_keep=7, dry_run=True)
|
||||
|
||||
old_file_count = sum(len(files) for files in old_files_result.values())
|
||||
if old_file_count > 0:
|
||||
print(f"Found {old_file_count} old files (>7 days)")
|
||||
response = input("Clean up old files? (y/N): ").strip().lower()
|
||||
if response == 'y':
|
||||
actual_old_result = cache_manager.cleanup_old_files(days_to_keep=7, dry_run=False)
|
||||
deleted_old_count = sum(len([f for f in files if "DELETED:" in f]) for files in actual_old_result.values())
|
||||
print(f"Deleted {deleted_old_count} old files")
|
||||
else:
|
||||
print("Skipped old file cleanup")
|
||||
else:
|
||||
print("No old files found")
|
||||
|
||||
print()
|
||||
print("=== Cache Fix Complete ===")
|
||||
print("The system should now work without Parquet deserialization errors.")
|
||||
print("If you continue to see issues, consider running with --emergency-reset")
|
||||
|
||||
def emergency_reset():
|
||||
"""Emergency cache reset"""
|
||||
print("=== EMERGENCY CACHE RESET ===")
|
||||
print("WARNING: This will delete ALL cache files!")
|
||||
print("You will need to re-download all historical data.")
|
||||
print()
|
||||
|
||||
response = input("Are you sure you want to proceed? Type 'DELETE ALL CACHE' to confirm: ")
|
||||
if response == "DELETE ALL CACHE":
|
||||
cache_manager = get_cache_manager()
|
||||
success = cache_manager.emergency_cache_reset(confirm=True)
|
||||
if success:
|
||||
print("Emergency cache reset completed.")
|
||||
print("All cache files have been deleted.")
|
||||
else:
|
||||
print("Emergency reset failed.")
|
||||
else:
|
||||
print("Emergency reset cancelled.")
|
||||
|
||||
if __name__ == "__main__":
|
||||
if len(sys.argv) > 1 and sys.argv[1] == "--emergency-reset":
|
||||
emergency_reset()
|
||||
else:
|
||||
main()
|
||||
Reference in New Issue
Block a user