BIG CLEANUP
This commit is contained in:
Dobromir Popov
@ -1,197 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Kill Stale Processes Script
|
||||
|
||||
Safely terminates stale Python processes related to the trading dashboard
|
||||
with proper error handling and graceful termination.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import signal
|
||||
from pathlib import Path
|
||||
import threading
|
||||
|
||||
# Global timeout flag
|
||||
timeout_reached = False
|
||||
|
||||
def timeout_handler():
|
||||
"""Handler for overall script timeout"""
|
||||
global timeout_reached
|
||||
timeout_reached = True
|
||||
print("\n⚠️ WARNING: Script timeout reached (10s) - forcing exit")
|
||||
os._exit(0) # Force exit
|
||||
|
||||
def kill_stale_processes():
|
||||
"""Kill stale trading dashboard processes safely"""
|
||||
global timeout_reached
|
||||
|
||||
# Set up overall timeout (10 seconds)
|
||||
timer = threading.Timer(10.0, timeout_handler)
|
||||
timer.daemon = True
|
||||
timer.start()
|
||||
|
||||
try:
|
||||
import psutil
|
||||
except ImportError:
|
||||
print("psutil not available - using fallback method")
|
||||
return kill_stale_fallback()
|
||||
|
||||
current_pid = os.getpid()
|
||||
killed_processes = []
|
||||
failed_processes = []
|
||||
|
||||
# Keywords to identify trading dashboard processes
|
||||
target_keywords = [
|
||||
'dashboard', 'scalping', 'trading', 'tensorboard',
|
||||
'run_clean', 'run_main', 'gogo2', 'mexc'
|
||||
]
|
||||
|
||||
try:
|
||||
print("Scanning for stale processes...")
|
||||
|
||||
# Get all Python processes with timeout
|
||||
python_processes = []
|
||||
scan_start = time.time()
|
||||
|
||||
for proc in psutil.process_iter(['pid', 'name', 'cmdline']):
|
||||
if timeout_reached or (time.time() - scan_start) > 3.0: # 3s max for scanning
|
||||
print("Process scanning timeout - proceeding with found processes")
|
||||
break
|
||||
|
||||
try:
|
||||
if proc.info['pid'] == current_pid:
|
||||
continue
|
||||
|
||||
name = proc.info['name'].lower()
|
||||
if 'python' in name or 'tensorboard' in name:
|
||||
cmdline_str = ' '.join(proc.info['cmdline']) if proc.info['cmdline'] else ''
|
||||
|
||||
# Check if this is a target process
|
||||
if any(keyword in cmdline_str.lower() for keyword in target_keywords):
|
||||
python_processes.append({
|
||||
'proc': proc,
|
||||
'pid': proc.info['pid'],
|
||||
'name': proc.info['name'],
|
||||
'cmdline': cmdline_str
|
||||
})
|
||||
|
||||
except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
|
||||
continue
|
||||
|
||||
if not python_processes:
|
||||
print("No stale processes found")
|
||||
timer.cancel() # Cancel the timeout
|
||||
return True
|
||||
|
||||
print(f"Found {len(python_processes)} target processes to terminate:")
|
||||
for p in python_processes[:5]: # Show max 5 to save time
|
||||
print(f" - PID {p['pid']}: {p['name']} - {p['cmdline'][:80]}...")
|
||||
if len(python_processes) > 5:
|
||||
print(f" ... and {len(python_processes) - 5} more")
|
||||
|
||||
# Graceful termination first (with reduced wait time)
|
||||
print("\nAttempting graceful termination...")
|
||||
termination_start = time.time()
|
||||
|
||||
for p in python_processes:
|
||||
if timeout_reached or (time.time() - termination_start) > 2.0:
|
||||
print("Termination timeout - moving to force kill")
|
||||
break
|
||||
|
||||
try:
|
||||
proc = p['proc']
|
||||
if proc.is_running():
|
||||
proc.terminate()
|
||||
print(f" Sent SIGTERM to PID {p['pid']}")
|
||||
except Exception as e:
|
||||
failed_processes.append(f"Failed to terminate PID {p['pid']}: {e}")
|
||||
|
||||
# Wait for graceful shutdown (reduced from 2.0 to 1.0)
|
||||
time.sleep(1.0)
|
||||
|
||||
# Force kill remaining processes
|
||||
print("\nChecking for remaining processes...")
|
||||
kill_start = time.time()
|
||||
|
||||
for p in python_processes:
|
||||
if timeout_reached or (time.time() - kill_start) > 2.0:
|
||||
print("Force kill timeout - exiting")
|
||||
break
|
||||
|
||||
try:
|
||||
proc = p['proc']
|
||||
if proc.is_running():
|
||||
print(f" Force killing PID {p['pid']} ({p['name']})")
|
||||
proc.kill()
|
||||
killed_processes.append(f"Force killed PID {p['pid']} ({p['name']})")
|
||||
else:
|
||||
killed_processes.append(f"Gracefully terminated PID {p['pid']} ({p['name']})")
|
||||
except (psutil.NoSuchProcess, psutil.AccessDenied):
|
||||
killed_processes.append(f"Process PID {p['pid']} already terminated")
|
||||
except Exception as e:
|
||||
failed_processes.append(f"Failed to kill PID {p['pid']}: {e}")
|
||||
|
||||
# Results (quick summary)
|
||||
print(f"\n=== Quick Results ===")
|
||||
print(f"✓ Cleaned up {len(killed_processes)} processes")
|
||||
if failed_processes:
|
||||
print(f"✗ Failed: {len(failed_processes)} processes")
|
||||
|
||||
timer.cancel() # Cancel the timeout if we finished early
|
||||
return len(failed_processes) == 0
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error during process cleanup: {e}")
|
||||
timer.cancel()
|
||||
return False
|
||||
|
||||
def kill_stale_fallback():
|
||||
"""Fallback method using basic OS commands"""
|
||||
print("Using fallback process killing method...")
|
||||
|
||||
try:
|
||||
if os.name == 'nt': # Windows
|
||||
import subprocess
|
||||
# Kill Python processes with dashboard keywords (with timeout)
|
||||
result = subprocess.run([
|
||||
'taskkill', '/f', '/im', 'python.exe'
|
||||
], capture_output=True, text=True, timeout=5.0)
|
||||
|
||||
if result.returncode == 0:
|
||||
print("Windows: Killed all Python processes")
|
||||
else:
|
||||
print("Windows: No Python processes to kill or access denied")
|
||||
|
||||
else: # Unix/Linux
|
||||
import subprocess
|
||||
# More targeted approach for Unix (with timeouts)
|
||||
subprocess.run(['pkill', '-f', 'dashboard'], capture_output=True, timeout=2.0)
|
||||
subprocess.run(['pkill', '-f', 'scalping'], capture_output=True, timeout=2.0)
|
||||
subprocess.run(['pkill', '-f', 'tensorboard'], capture_output=True, timeout=2.0)
|
||||
print("Unix: Killed dashboard-related processes")
|
||||
|
||||
return True
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
print("Fallback method timed out")
|
||||
return False
|
||||
except Exception as e:
|
||||
print(f"Fallback method failed: {e}")
|
||||
return False
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("=" * 50)
|
||||
print("STALE PROCESS CLEANUP (10s timeout)")
|
||||
print("=" * 50)
|
||||
|
||||
start_time = time.time()
|
||||
success = kill_stale_processes()
|
||||
elapsed = time.time() - start_time
|
||||
|
||||
exit_code = 0 if success else 1
|
||||
|
||||
print(f"Completed in {elapsed:.1f}s")
|
||||
print("=" * 50)
|
||||
sys.exit(exit_code)
|
||||
@ -1,40 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Restart Dashboard with Forced Learning Enabled
|
||||
Simple script to start dashboard with all learning features enabled
|
||||
"""
|
||||
|
||||
import sys
|
||||
import logging
|
||||
from datetime import datetime
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def main():
|
||||
"""Start dashboard with forced learning"""
|
||||
logger.info("🚀 Starting Dashboard with FORCED LEARNING ENABLED")
|
||||
logger.info("=" * 60)
|
||||
logger.info(f"Timestamp: {datetime.now()}")
|
||||
logger.info("Fixes Applied:")
|
||||
logger.info("✅ Enhanced RL: FORCED ENABLED")
|
||||
logger.info("✅ CNN Training: FORCED ENABLED")
|
||||
logger.info("✅ Williams Pivots: CNN INTEGRATED")
|
||||
logger.info("✅ Learning Pipeline: ACTIVE")
|
||||
logger.info("=" * 60)
|
||||
|
||||
try:
|
||||
# Import and run main
|
||||
from main_clean import run_web_dashboard
|
||||
logger.info("Starting web dashboard...")
|
||||
run_web_dashboard()
|
||||
|
||||
except KeyboardInterrupt:
|
||||
logger.info("Dashboard stopped by user")
|
||||
except Exception as e:
|
||||
logger.error(f"Dashboard failed: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,188 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Overnight Training Restart Script
|
||||
Keeps main.py running continuously, restarting it if it crashes.
|
||||
Designed for overnight training sessions with unstable code.
|
||||
|
||||
Usage:
|
||||
python restart_main_overnight.py
|
||||
|
||||
Press Ctrl+C to stop the restart loop.
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import os
|
||||
|
||||
# Setup logging for the restart script
|
||||
def setup_restart_logging():
|
||||
"""Setup logging for restart events"""
|
||||
log_dir = Path("logs")
|
||||
log_dir.mkdir(exist_ok=True)
|
||||
|
||||
# Create restart log file with timestamp
|
||||
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
log_file = log_dir / f"restart_main_{timestamp}.log"
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(levelname)s - %(message)s',
|
||||
handlers=[
|
||||
logging.FileHandler(log_file, encoding='utf-8'),
|
||||
logging.StreamHandler(sys.stdout)
|
||||
]
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
logger.info(f"Restart script logging to: {log_file}")
|
||||
return logger
|
||||
|
||||
def kill_existing_processes(logger):
|
||||
"""Kill any existing main.py processes to avoid conflicts"""
|
||||
try:
|
||||
if os.name == 'nt': # Windows
|
||||
# Kill any existing Python processes running main.py
|
||||
subprocess.run(['taskkill', '/f', '/im', 'python.exe'],
|
||||
capture_output=True, check=False)
|
||||
subprocess.run(['taskkill', '/f', '/im', 'pythonw.exe'],
|
||||
capture_output=True, check=False)
|
||||
time.sleep(2)
|
||||
except Exception as e:
|
||||
logger.warning(f"Could not kill existing processes: {e}")
|
||||
|
||||
def run_main_with_restart(logger):
|
||||
"""Main restart loop"""
|
||||
restart_count = 0
|
||||
consecutive_fast_exits = 0
|
||||
start_time = datetime.now()
|
||||
|
||||
logger.info("=" * 60)
|
||||
logger.info("OVERNIGHT TRAINING RESTART SCRIPT STARTED")
|
||||
logger.info("=" * 60)
|
||||
logger.info("Press Ctrl+C to stop the restart loop")
|
||||
logger.info("Main script: main.py")
|
||||
logger.info("Restart delay on crash: 10 seconds")
|
||||
logger.info("Fast exit protection: Enabled")
|
||||
logger.info("=" * 60)
|
||||
|
||||
# Kill any existing processes
|
||||
kill_existing_processes(logger)
|
||||
|
||||
while True:
|
||||
try:
|
||||
restart_count += 1
|
||||
run_start_time = datetime.now()
|
||||
|
||||
logger.info(f"[RESTART #{restart_count}] Starting main.py at {run_start_time.strftime('%H:%M:%S')}")
|
||||
|
||||
# Start main.py as subprocess
|
||||
process = subprocess.Popen([
|
||||
sys.executable, "main.py"
|
||||
], stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||
universal_newlines=True, bufsize=1)
|
||||
|
||||
logger.info(f"[PROCESS] main.py started with PID: {process.pid}")
|
||||
|
||||
# Stream output from main.py
|
||||
try:
|
||||
if process.stdout:
|
||||
while True:
|
||||
output = process.stdout.readline()
|
||||
if output == '' and process.poll() is not None:
|
||||
break
|
||||
if output:
|
||||
# Forward output from main.py (remove extra newlines)
|
||||
print(f"[MAIN] {output.rstrip()}")
|
||||
else:
|
||||
# If no stdout, just wait for process to complete
|
||||
process.wait()
|
||||
except KeyboardInterrupt:
|
||||
logger.info("[INTERRUPT] Ctrl+C received, stopping main.py...")
|
||||
process.terminate()
|
||||
try:
|
||||
process.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
logger.warning("[FORCE KILL] Process didn't terminate, force killing...")
|
||||
process.kill()
|
||||
raise
|
||||
|
||||
# Process has exited
|
||||
exit_code = process.poll()
|
||||
run_end_time = datetime.now()
|
||||
run_duration = (run_end_time - run_start_time).total_seconds()
|
||||
|
||||
logger.info(f"[EXIT] main.py exited with code {exit_code}")
|
||||
logger.info(f"[DURATION] Process ran for {run_duration:.1f} seconds")
|
||||
|
||||
# Check for fast exits (potential configuration issues)
|
||||
if run_duration < 30: # Less than 30 seconds
|
||||
consecutive_fast_exits += 1
|
||||
logger.warning(f"[FAST EXIT] Process exited quickly ({consecutive_fast_exits} consecutive)")
|
||||
|
||||
if consecutive_fast_exits >= 5:
|
||||
logger.error("[ABORT] Too many consecutive fast exits (5+)")
|
||||
logger.error("This indicates a configuration or startup problem")
|
||||
logger.error("Please check the main.py script manually")
|
||||
break
|
||||
|
||||
# Longer delay for fast exits
|
||||
delay = min(60, 10 * consecutive_fast_exits)
|
||||
logger.info(f"[DELAY] Waiting {delay} seconds before restart due to fast exit...")
|
||||
time.sleep(delay)
|
||||
else:
|
||||
consecutive_fast_exits = 0 # Reset counter
|
||||
logger.info("[DELAY] Waiting 10 seconds before restart...")
|
||||
time.sleep(10)
|
||||
|
||||
# Log session statistics every 10 restarts
|
||||
if restart_count % 10 == 0:
|
||||
total_duration = (datetime.now() - start_time).total_seconds()
|
||||
logger.info(f"[STATS] Session: {restart_count} restarts in {total_duration/3600:.1f} hours")
|
||||
|
||||
except KeyboardInterrupt:
|
||||
logger.info("[SHUTDOWN] Restart loop interrupted by user")
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"[ERROR] Unexpected error in restart loop: {e}")
|
||||
logger.error("Continuing restart loop after 30 second delay...")
|
||||
time.sleep(30)
|
||||
|
||||
total_duration = (datetime.now() - start_time).total_seconds()
|
||||
logger.info("=" * 60)
|
||||
logger.info("OVERNIGHT TRAINING SESSION COMPLETE")
|
||||
logger.info(f"Total restarts: {restart_count}")
|
||||
logger.info(f"Total session time: {total_duration/3600:.1f} hours")
|
||||
logger.info("=" * 60)
|
||||
|
||||
def main():
|
||||
"""Main entry point"""
|
||||
# Setup signal handlers for clean shutdown
|
||||
def signal_handler(signum, frame):
|
||||
logger.info(f"[SIGNAL] Received signal {signum}, shutting down...")
|
||||
sys.exit(0)
|
||||
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
if hasattr(signal, 'SIGTERM'):
|
||||
signal.signal(signal.SIGTERM, signal_handler)
|
||||
|
||||
# Setup logging
|
||||
global logger
|
||||
logger = setup_restart_logging()
|
||||
|
||||
try:
|
||||
run_main_with_restart(logger)
|
||||
except Exception as e:
|
||||
logger.error(f"[FATAL] Fatal error in restart script: {e}")
|
||||
import traceback
|
||||
logger.error(traceback.format_exc())
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user