124 lines
3.9 KiB
Python
124 lines
3.9 KiB
Python
import os
|
|
import sys
|
|
import subprocess
|
|
import time
|
|
import logging
|
|
from datetime import datetime
|
|
import webbrowser
|
|
from threading import Thread
|
|
|
|
# Configure logging
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format='%(asctime)s - %(levelname)s - %(message)s',
|
|
handlers=[
|
|
logging.FileHandler('training_launch.log'),
|
|
logging.StreamHandler()
|
|
]
|
|
)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
def start_tensorboard(port=6007):
|
|
"""Start TensorBoard on a specified port"""
|
|
try:
|
|
cmd = f"tensorboard --logdir=runs --port={port}"
|
|
process = subprocess.Popen(cmd, shell=True)
|
|
logger.info(f"Started TensorBoard on port {port}")
|
|
return process
|
|
except Exception as e:
|
|
logger.error(f"Failed to start TensorBoard: {str(e)}")
|
|
return None
|
|
|
|
def start_web_chart():
|
|
"""Start the web chart server"""
|
|
try:
|
|
cmd = "python main.py --symbols BTC/USDT ETH/USDT SOL/USDT --timeframes 1m 5m 15m --mode realtime"
|
|
process = subprocess.Popen(cmd, shell=True)
|
|
logger.info("Started web chart server")
|
|
return process
|
|
except Exception as e:
|
|
logger.error(f"Failed to start web chart server: {str(e)}")
|
|
return None
|
|
|
|
def start_training():
|
|
"""Start the RL training process"""
|
|
try:
|
|
cmd = "python NN/train_rl.py"
|
|
process = subprocess.Popen(cmd, shell=True)
|
|
logger.info("Started RL training process")
|
|
return process
|
|
except Exception as e:
|
|
logger.error(f"Failed to start training process: {str(e)}")
|
|
return None
|
|
|
|
def open_web_interfaces():
|
|
"""Open web browsers for TensorBoard and chart after a delay"""
|
|
time.sleep(5) # Wait for servers to start
|
|
try:
|
|
webbrowser.open('http://localhost:6007') # TensorBoard
|
|
webbrowser.open('http://localhost:8050') # Web chart
|
|
except Exception as e:
|
|
logger.error(f"Failed to open web interfaces: {str(e)}")
|
|
|
|
def monitor_processes(processes):
|
|
"""Monitor running processes and log any unexpected terminations"""
|
|
while True:
|
|
for name, process in processes.items():
|
|
if process and process.poll() is not None:
|
|
logger.error(f"{name} process terminated unexpectedly")
|
|
return False
|
|
time.sleep(1)
|
|
|
|
def main():
|
|
"""Main function to orchestrate the training environment"""
|
|
logger.info("Starting training environment setup...")
|
|
|
|
# Start TensorBoard
|
|
tensorboard_process = start_tensorboard(port=6007)
|
|
if not tensorboard_process:
|
|
logger.error("Failed to start TensorBoard")
|
|
return
|
|
|
|
# Start web chart
|
|
web_chart_process = start_web_chart()
|
|
if not web_chart_process:
|
|
tensorboard_process.terminate()
|
|
logger.error("Failed to start web chart")
|
|
return
|
|
|
|
# Start training
|
|
training_process = start_training()
|
|
if not training_process:
|
|
tensorboard_process.terminate()
|
|
web_chart_process.terminate()
|
|
logger.error("Failed to start training")
|
|
return
|
|
|
|
# Open web interfaces in a separate thread
|
|
Thread(target=open_web_interfaces).start()
|
|
|
|
# Monitor processes
|
|
processes = {
|
|
'tensorboard': tensorboard_process,
|
|
'web_chart': web_chart_process,
|
|
'training': training_process
|
|
}
|
|
|
|
try:
|
|
if not monitor_processes(processes):
|
|
raise Exception("One or more processes terminated unexpectedly")
|
|
except KeyboardInterrupt:
|
|
logger.info("Received shutdown signal")
|
|
except Exception as e:
|
|
logger.error(f"Error in monitoring: {str(e)}")
|
|
finally:
|
|
# Cleanup
|
|
logger.info("Shutting down training environment...")
|
|
for name, process in processes.items():
|
|
if process:
|
|
process.terminate()
|
|
logger.info(f"Terminated {name} process")
|
|
logger.info("Training environment shutdown complete")
|
|
|
|
if __name__ == "__main__":
|
|
main() |