# Overnight Training Restart Script (PowerShell) # Keeps main.py running continuously, restarting it if it crashes. # Usage: .\restart_main_overnight.ps1 Write-Host "=" * 60 Write-Host "OVERNIGHT TRAINING RESTART SCRIPT (PowerShell)" Write-Host "=" * 60 Write-Host "Press Ctrl+C to stop the restart loop" Write-Host "Main script: main.py" Write-Host "Restart delay on crash: 10 seconds" Write-Host "=" * 60 $restartCount = 0 $startTime = Get-Date # Create logs directory if it doesn't exist if (!(Test-Path "logs")) { New-Item -ItemType Directory -Path "logs" } # Setup log file $timestamp = Get-Date -Format "yyyyMMdd_HHmmss" $logFile = "logs\restart_main_ps_$timestamp.log" function Write-Log { param($Message) $timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss" $logMessage = "$timestamp - $Message" Write-Host $logMessage Add-Content -Path $logFile -Value $logMessage } Write-Log "Restart script started, logging to: $logFile" # Kill any existing Python processes try { Get-Process python* -ErrorAction SilentlyContinue | Stop-Process -Force -ErrorAction SilentlyContinue Start-Sleep -Seconds 2 Write-Log "Killed existing Python processes" } catch { Write-Log "Could not kill existing processes: $_" } try { while ($true) { $restartCount++ $runStartTime = Get-Date Write-Log "[RESTART #$restartCount] Starting main.py at $(Get-Date -Format 'HH:mm:ss')" # Start main.py try { $process = Start-Process -FilePath "python" -ArgumentList "main.py" -PassThru -Wait $exitCode = $process.ExitCode $runEndTime = Get-Date $runDuration = ($runEndTime - $runStartTime).TotalSeconds Write-Log "[EXIT] main.py exited with code $exitCode" Write-Log "[DURATION] Process ran for $([math]::Round($runDuration, 1)) seconds" # Check for fast exits if ($runDuration -lt 30) { Write-Log "[FAST EXIT] Process exited quickly, waiting 30 seconds..." Start-Sleep -Seconds 30 } else { Write-Log "[DELAY] Waiting 10 seconds before restart..." Start-Sleep -Seconds 10 } # Log stats every 10 restarts if ($restartCount % 10 -eq 0) { $totalDuration = (Get-Date) - $startTime Write-Log "[STATS] Session: $restartCount restarts in $([math]::Round($totalDuration.TotalHours, 1)) hours" } } catch { Write-Log "[ERROR] Error starting main.py: $_" Start-Sleep -Seconds 10 } } } catch { Write-Log "[INTERRUPT] Restart loop interrupted: $_" } finally { $totalDuration = (Get-Date) - $startTime Write-Log "=" * 60 Write-Log "OVERNIGHT TRAINING SESSION COMPLETE" Write-Log "Total restarts: $restartCount" Write-Log "Total session time: $([math]::Round($totalDuration.TotalHours, 1)) hours" Write-Log "=" * 60 }