cleanup and reorgnization
This commit is contained in:
90
scripts/restart_main_overnight.ps1
Normal file
90
scripts/restart_main_overnight.ps1
Normal file
@ -0,0 +1,90 @@
|
||||
# Overnight Training Restart Script (PowerShell)
|
||||
# Keeps main.py running continuously, restarting it if it crashes.
|
||||
# Usage: .\restart_main_overnight.ps1
|
||||
|
||||
Write-Host "=" * 60
|
||||
Write-Host "OVERNIGHT TRAINING RESTART SCRIPT (PowerShell)"
|
||||
Write-Host "=" * 60
|
||||
Write-Host "Press Ctrl+C to stop the restart loop"
|
||||
Write-Host "Main script: main.py"
|
||||
Write-Host "Restart delay on crash: 10 seconds"
|
||||
Write-Host "=" * 60
|
||||
|
||||
$restartCount = 0
|
||||
$startTime = Get-Date
|
||||
|
||||
# Create logs directory if it doesn't exist
|
||||
if (!(Test-Path "logs")) {
|
||||
New-Item -ItemType Directory -Path "logs"
|
||||
}
|
||||
|
||||
# Setup log file
|
||||
$timestamp = Get-Date -Format "yyyyMMdd_HHmmss"
|
||||
$logFile = "logs\restart_main_ps_$timestamp.log"
|
||||
|
||||
function Write-Log {
|
||||
param($Message)
|
||||
$timestamp = Get-Date -Format "yyyy-MM-dd HH:mm:ss"
|
||||
$logMessage = "$timestamp - $Message"
|
||||
Write-Host $logMessage
|
||||
Add-Content -Path $logFile -Value $logMessage
|
||||
}
|
||||
|
||||
Write-Log "Restart script started, logging to: $logFile"
|
||||
|
||||
# Kill any existing Python processes
|
||||
try {
|
||||
Get-Process python* -ErrorAction SilentlyContinue | Stop-Process -Force -ErrorAction SilentlyContinue
|
||||
Start-Sleep -Seconds 2
|
||||
Write-Log "Killed existing Python processes"
|
||||
} catch {
|
||||
Write-Log "Could not kill existing processes: $_"
|
||||
}
|
||||
|
||||
try {
|
||||
while ($true) {
|
||||
$restartCount++
|
||||
$runStartTime = Get-Date
|
||||
|
||||
Write-Log "[RESTART #$restartCount] Starting main.py at $(Get-Date -Format 'HH:mm:ss')"
|
||||
|
||||
# Start main.py
|
||||
try {
|
||||
$process = Start-Process -FilePath "python" -ArgumentList "main.py" -PassThru -Wait
|
||||
$exitCode = $process.ExitCode
|
||||
$runEndTime = Get-Date
|
||||
$runDuration = ($runEndTime - $runStartTime).TotalSeconds
|
||||
|
||||
Write-Log "[EXIT] main.py exited with code $exitCode"
|
||||
Write-Log "[DURATION] Process ran for $([math]::Round($runDuration, 1)) seconds"
|
||||
|
||||
# Check for fast exits
|
||||
if ($runDuration -lt 30) {
|
||||
Write-Log "[FAST EXIT] Process exited quickly, waiting 30 seconds..."
|
||||
Start-Sleep -Seconds 30
|
||||
} else {
|
||||
Write-Log "[DELAY] Waiting 10 seconds before restart..."
|
||||
Start-Sleep -Seconds 10
|
||||
}
|
||||
|
||||
# Log stats every 10 restarts
|
||||
if ($restartCount % 10 -eq 0) {
|
||||
$totalDuration = (Get-Date) - $startTime
|
||||
Write-Log "[STATS] Session: $restartCount restarts in $([math]::Round($totalDuration.TotalHours, 1)) hours"
|
||||
}
|
||||
|
||||
} catch {
|
||||
Write-Log "[ERROR] Error starting main.py: $_"
|
||||
Start-Sleep -Seconds 10
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
Write-Log "[INTERRUPT] Restart loop interrupted: $_"
|
||||
} finally {
|
||||
$totalDuration = (Get-Date) - $startTime
|
||||
Write-Log "=" * 60
|
||||
Write-Log "OVERNIGHT TRAINING SESSION COMPLETE"
|
||||
Write-Log "Total restarts: $restartCount"
|
||||
Write-Log "Total session time: $([math]::Round($totalDuration.TotalHours, 1)) hours"
|
||||
Write-Log "=" * 60
|
||||
}
|
Reference in New Issue
Block a user