realtime client, timeseries storage

This commit is contained in:
Dobromir Popov 2025-03-18 22:52:24 +02:00
parent 3871afd4b8
commit c1ad6cddd6
23 changed files with 59777 additions and 206 deletions

1
.env
View File

@ -1,6 +1,7 @@
# MEXC Exchange API Keys
MEXC_API_KEY=mx0vglGymMT4iLpHXD
MEXC_SECRET_KEY=557300a85ae84cf6b927b86278905fd7
# # BASE ENDPOINTS: https://api.mexc.com wss://wbs-api.mexc.com/ws !!! DO NOT CHANGE THIS
# Trading Parameters
MAX_LEVERAGE=50

5
.gitignore vendored Normal file
View File

@ -0,0 +1,5 @@
models/trading_agent_best_net_pnl.pt
models/trading_agent_checkpoint_*
runs/*
trading_bot.log
backtest_stats_*.csv

64
.vscode/launch.json vendored
View File

@ -1,12 +1,25 @@
{
"version": "0.2.0",
"configurations": [
{
"name": "Python Debugger: Current File",
"type": "debugpy",
"request": "launch",
// "program": "realtime.py",
"program": "${file}",
"console": "integratedTerminal"
},
{
"name": "Train Bot",
"type": "python",
"request": "launch",
"program": "main.py",
"args": ["--mode", "train", "--episodes", "100"],
"args": [
"--mode",
"train",
"--episodes",
"100"
],
"console": "integratedTerminal",
"justMyCode": true
},
@ -15,7 +28,12 @@
"type": "python",
"request": "launch",
"program": "main.py",
"args": ["--mode", "eval", "--episodes", "10"],
"args": [
"--mode",
"eval",
"--episodes",
"10"
],
"console": "integratedTerminal",
"justMyCode": true
},
@ -25,10 +43,14 @@
"request": "launch",
"program": "main.py",
"args": [
"--mode", "live",
"--demo", "true",
"--symbol", "ETH/USDT",
"--timeframe", "1m"
"--mode",
"live",
"--demo",
"true",
"--symbol",
"ETH/USDT",
"--timeframe",
"1m"
],
"console": "integratedTerminal",
"justMyCode": true,
@ -42,11 +64,16 @@
"request": "launch",
"program": "main.py",
"args": [
"--mode", "live",
"--demo", "false",
"--symbol", "ETH/USDT",
"--timeframe", "1m",
"--leverage", "50"
"--mode",
"live",
"--demo",
"false",
"--symbol",
"ETH/USDT",
"--timeframe",
"1m",
"--leverage",
"50"
],
"console": "integratedTerminal",
"justMyCode": true,
@ -60,11 +87,16 @@
"request": "launch",
"program": "main.py",
"args": [
"--mode", "live",
"--demo", "false",
"--symbol", "BTC/USDT",
"--timeframe", "5m",
"--leverage", "20"
"--mode",
"live",
"--demo",
"false",
"--symbol",
"BTC/USDT",
"--timeframe",
"5m",
"--leverage",
"20"
],
"console": "integratedTerminal",
"justMyCode": true,

View File

@ -3,7 +3,7 @@ https://github.com/mexcdevelop/mexc-api-sdk/blob/main/README.md#test-new-order
python mexc_tick_visualizer.py --symbol BTC/USDT --interval 1.0 --candle 60
& 'C:\Users\popov\miniforge3\python.exe' 'c:\Users\popov\.cursor\extensions\ms-python.debugpy-2024.6.0-win32-x64\bundled\libs\debugpy\adapter/../..\debugpy\launcher' '51766' '--' 'main.py' '--mode' 'live' '--demo' 'false' '--symbol' 'ETH/USDT' '--timeframe' '1m' '--leverage' '50'

View File

@ -1,22 +0,0 @@
Period,Episode,Reward,Balance,PnL,Fees,Net_PnL
Day-1,1,0,100,0,0.0,0.0
Day-1,2,0,100,0,0.0,0.0
Day-1,3,0,100,0,0.0,0.0
Day-2,1,0,100,0,0.0,0.0
Day-2,2,0,100,0,0.0,0.0
Day-2,3,0,100,0,0.0,0.0
Day-3,1,0,100,0,0.0,0.0
Day-3,2,0,100,0,0.0,0.0
Day-3,3,0,100,0,0.0,0.0
Day-4,1,0,100,0,0.0,0.0
Day-4,2,0,100,0,0.0,0.0
Day-4,3,0,100,0,0.0,0.0
Day-5,1,0,100,0,0.0,0.0
Day-5,2,0,100,0,0.0,0.0
Day-5,3,0,100,0,0.0,0.0
Day-6,1,0,100,0,0.0,0.0
Day-6,2,0,100,0,0.0,0.0
Day-6,3,0,100,0,0.0,0.0
Day-7,1,0,100,0,0.0,0.0
Day-7,2,0,100,0,0.0,0.0
Day-7,3,0,100,0,0.0,0.0
1 Period Episode Reward Balance PnL Fees Net_PnL
2 Day-1 1 0 100 0 0.0 0.0
3 Day-1 2 0 100 0 0.0 0.0
4 Day-1 3 0 100 0 0.0 0.0
5 Day-2 1 0 100 0 0.0 0.0
6 Day-2 2 0 100 0 0.0 0.0
7 Day-2 3 0 100 0 0.0 0.0
8 Day-3 1 0 100 0 0.0 0.0
9 Day-3 2 0 100 0 0.0 0.0
10 Day-3 3 0 100 0 0.0 0.0
11 Day-4 1 0 100 0 0.0 0.0
12 Day-4 2 0 100 0 0.0 0.0
13 Day-4 3 0 100 0 0.0 0.0
14 Day-5 1 0 100 0 0.0 0.0
15 Day-5 2 0 100 0 0.0 0.0
16 Day-5 3 0 100 0 0.0 0.0
17 Day-6 1 0 100 0 0.0 0.0
18 Day-6 2 0 100 0 0.0 0.0
19 Day-6 3 0 100 0 0.0 0.0
20 Day-7 1 0 100 0 0.0 0.0
21 Day-7 2 0 100 0 0.0 0.0
22 Day-7 3 0 100 0 0.0 0.0

Binary file not shown.

Before

Width:  |  Height:  |  Size: 19 KiB

View File

@ -1,4 +0,0 @@
Episode,Reward,Balance,PnL,Fees,Net PnL,Win Rate,Trades,Loss
1,0,100,0,0.0,0.0,0,0,0
2,0,100,0,0.0,0.0,0,0,0
3,0,100,0,0.0,0.0,0,0,0
1 Episode Reward Balance PnL Fees Net PnL Win Rate Trades Loss
2 1 0 100 0 0.0 0.0 0 0 0
3 2 0 100 0 0.0 0.0 0 0 0
4 3 0 100 0 0.0 0.0 0 0 0

View File

@ -1,4 +0,0 @@
Episode,Reward,Balance,PnL,Fees,Net PnL,Win Rate,Trades,Loss
1,0,100,0,0.0,0.0,0,0,0
2,0,100,0,0.0,0.0,0,0,0
3,0,100,0,0.0,0.0,0,0,0
1 Episode Reward Balance PnL Fees Net PnL Win Rate Trades Loss
2 1 0 100 0 0.0 0.0 0 0 0
3 2 0 100 0 0.0 0.0 0 0 0
4 3 0 100 0 0.0 0.0 0 0 0

View File

@ -1,4 +0,0 @@
Episode,Reward,Balance,PnL,Fees,Net PnL,Win Rate,Trades,Loss
1,0,100,0,0.0,0.0,0,0,0
2,0,100,0,0.0,0.0,0,0,0
3,0,100,0,0.0,0.0,0,0,0
1 Episode Reward Balance PnL Fees Net PnL Win Rate Trades Loss
2 1 0 100 0 0.0 0.0 0 0 0
3 2 0 100 0 0.0 0.0 0 0 0
4 3 0 100 0 0.0 0.0 0 0 0

View File

@ -1,4 +0,0 @@
Episode,Reward,Balance,PnL,Fees,Net PnL,Win Rate,Trades,Loss
1,0,100,0,0.0,0.0,0,0,0
2,0,100,0,0.0,0.0,0,0,0
3,0,100,0,0.0,0.0,0,0,0
1 Episode Reward Balance PnL Fees Net PnL Win Rate Trades Loss
2 1 0 100 0 0.0 0.0 0 0 0
3 2 0 100 0 0.0 0.0 0 0 0
4 3 0 100 0 0.0 0.0 0 0 0

View File

@ -1,4 +0,0 @@
Episode,Reward,Balance,PnL,Fees,Net PnL,Win Rate,Trades,Loss
1,0,100,0,0.0,0.0,0,0,0
2,0,100,0,0.0,0.0,0,0,0
3,0,100,0,0.0,0.0,0,0,0
1 Episode Reward Balance PnL Fees Net PnL Win Rate Trades Loss
2 1 0 100 0 0.0 0.0 0 0 0
3 2 0 100 0 0.0 0.0 0 0 0
4 3 0 100 0 0.0 0.0 0 0 0

View File

@ -1,4 +0,0 @@
Episode,Reward,Balance,PnL,Fees,Net PnL,Win Rate,Trades,Loss
1,0,100,0,0.0,0.0,0,0,0
2,0,100,0,0.0,0.0,0,0,0
3,0,100,0,0.0,0.0,0,0,0
1 Episode Reward Balance PnL Fees Net PnL Win Rate Trades Loss
2 1 0 100 0 0.0 0.0 0 0 0
3 2 0 100 0 0.0 0.0 0 0 0
4 3 0 100 0 0.0 0.0 0 0 0

View File

@ -1,4 +0,0 @@
Episode,Reward,Balance,PnL,Fees,Net PnL,Win Rate,Trades,Loss
1,0,100,0,0.0,0.0,0,0,0
2,0,100,0,0.0,0.0,0,0,0
3,0,100,0,0.0,0.0,0,0,0
1 Episode Reward Balance PnL Fees Net PnL Win Rate Trades Loss
2 1 0 100 0 0.0 0.0 0 0 0
3 2 0 100 0 0.0 0.0 0 0 0
4 3 0 100 0 0.0 0.0 0 0 0

View File

@ -1,3 +0,0 @@
Episode,Reward,Balance,PnL,Fees,Net PnL,Win Rate,Trades,Loss
1,0,100,0,0.0,0.0,0,0,0
2,0,100,0,0.0,0.0,0,0,0
1 Episode Reward Balance PnL Fees Net PnL Win Rate Trades Loss
2 1 0 100 0 0.0 0.0 0 0 0
3 2 0 100 0 0.0 0.0 0 0 0

71
fix_live_trading.py Normal file
View File

@ -0,0 +1,71 @@
def fix_live_trading():
try:
# Read the file content as a single string
with open('main.py', 'r') as f:
content = f.read()
print(f"Read {len(content)} characters from main.py")
# Fix the live_trading function signature
live_trading_pos = content.find('async def live_trading(')
if live_trading_pos != -1:
print(f"Found live_trading function at position {live_trading_pos}")
content = content.replace('async def live_trading(', 'async def live_trading(agent=None, env=None, exchange=None, ')
print("Updated live_trading function signature")
else:
print("WARNING: Could not find live_trading function!")
# Fix the TradingEnvironment initialization
env_init_pos = content.find('env = TradingEnvironment(')
if env_init_pos != -1:
print(f"Found env initialization at position {env_init_pos}")
# Find the closing parenthesis
paren_depth = 0
close_pos = env_init_pos
for i in range(env_init_pos, len(content)):
if content[i] == '(':
paren_depth += 1
elif content[i] == ')':
paren_depth -= 1
if paren_depth == 0:
close_pos = i + 1
break
# Calculate indentation
line_start = content.rfind('\n', 0, env_init_pos) + 1
indent = ' ' * (env_init_pos - line_start)
# Create the new environment initialization code
new_env_init = f'''if env is None:
{indent} env = TradingEnvironment(
{indent} initial_balance=initial_balance,
{indent} leverage=leverage,
{indent} window_size=window_size,
{indent} commission=commission,
{indent} symbol=symbol,
{indent} timeframe=timeframe
{indent} )'''
# Replace the old code with the new code
content = content[:env_init_pos] + new_env_init + content[close_pos:]
print("Updated TradingEnvironment initialization")
else:
print("WARNING: Could not find TradingEnvironment initialization!")
# Write the updated content back to the file
with open('main.py', 'w') as f:
f.write(content)
print(f"Wrote {len(content)} characters back to main.py")
print('Fixed live_trading function and TradingEnvironment initialization')
return True
except Exception as e:
print(f'Error fixing file: {e}')
import traceback
print(traceback.format_exc())
return False
if __name__ == "__main__":
fix_live_trading()

View File

@ -0,0 +1 @@
timestamp,action,price,position_size,balance,pnl
1 timestamp action price position_size balance pnl

700
realtime.py Normal file
View File

@ -0,0 +1,700 @@
import asyncio
import json
import logging
import datetime
from typing import Dict, List, Optional
import websockets
import plotly.graph_objects as go
from plotly.subplots import make_subplots
import dash
from dash import html, dcc
from dash.dependencies import Input, Output
import pandas as pd
import numpy as np
from collections import deque
import time
from threading import Thread
# Configure logging with more detailed format
logging.basicConfig(
level=logging.DEBUG, # Changed to DEBUG for more detailed logs
format='%(asctime)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s',
handlers=[
logging.StreamHandler(),
logging.FileHandler('realtime_chart.log')
]
)
logger = logging.getLogger(__name__)
class TradeTickStorage:
"""Store and manage raw trade ticks for display and candle formation"""
def __init__(self, max_age_seconds: int = 300): # 5 minutes by default
self.ticks = []
self.max_age_seconds = max_age_seconds
logger.info(f"Initialized TradeTickStorage with max age: {max_age_seconds} seconds")
def add_tick(self, tick: Dict):
"""Add a new trade tick to storage"""
self.ticks.append(tick)
logger.debug(f"Added tick: {tick}, total ticks: {len(self.ticks)}")
# Clean up old ticks
self._cleanup()
def _cleanup(self):
"""Remove ticks older than max_age_seconds"""
now = int(time.time() * 1000) # Current time in ms
cutoff = now - (self.max_age_seconds * 1000)
old_count = len(self.ticks)
self.ticks = [tick for tick in self.ticks if tick['timestamp'] >= cutoff]
if old_count > len(self.ticks):
logger.debug(f"Cleaned up {old_count - len(self.ticks)} old ticks")
def get_ticks_as_df(self) -> pd.DataFrame:
"""Return ticks as a DataFrame"""
if not self.ticks:
return pd.DataFrame()
df = pd.DataFrame(self.ticks)
if not df.empty:
df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms')
return df
def get_candles(self, interval_seconds: int = 1) -> pd.DataFrame:
"""Convert ticks to OHLCV candles at specified interval"""
if not self.ticks:
return pd.DataFrame()
# Ensure ticks are up to date
self._cleanup()
# Convert to DataFrame
df = self.get_ticks_as_df()
if df.empty:
return pd.DataFrame()
# Use timestamp column for resampling
df = df.set_index('timestamp')
# Create interval string for resampling
interval_str = f'{interval_seconds}S'
# Resample to create OHLCV candles
candles = df.resample(interval_str).agg({
'price': ['first', 'max', 'min', 'last'],
'volume': 'sum'
})
# Flatten MultiIndex columns
candles.columns = ['open', 'high', 'low', 'close', 'volume']
# Reset index to get timestamp as column
candles = candles.reset_index()
logger.debug(f"Generated {len(candles)} candles from {len(self.ticks)} ticks")
return candles
class CandlestickData:
def __init__(self, max_length: int = 300):
self.timestamps = deque(maxlen=max_length)
self.opens = deque(maxlen=max_length)
self.highs = deque(maxlen=max_length)
self.lows = deque(maxlen=max_length)
self.closes = deque(maxlen=max_length)
self.volumes = deque(maxlen=max_length)
self.current_candle = {
'timestamp': None,
'open': None,
'high': None,
'low': None,
'close': None,
'volume': 0
}
self.candle_interval = 1 # 1 second by default
def update_from_trade(self, trade: Dict):
timestamp = trade['timestamp']
price = trade['price']
volume = trade.get('volume', 0)
# Round timestamp to nearest candle interval
candle_timestamp = int(timestamp / (self.candle_interval * 1000)) * (self.candle_interval * 1000)
if self.current_candle['timestamp'] != candle_timestamp:
# Save current candle if it exists
if self.current_candle['timestamp'] is not None:
self.timestamps.append(self.current_candle['timestamp'])
self.opens.append(self.current_candle['open'])
self.highs.append(self.current_candle['high'])
self.lows.append(self.current_candle['low'])
self.closes.append(self.current_candle['close'])
self.volumes.append(self.current_candle['volume'])
logger.debug(f"New candle saved: {self.current_candle}")
# Start new candle
self.current_candle = {
'timestamp': candle_timestamp,
'open': price,
'high': price,
'low': price,
'close': price,
'volume': volume
}
logger.debug(f"New candle started: {self.current_candle}")
else:
# Update current candle
if self.current_candle['high'] is None or price > self.current_candle['high']:
self.current_candle['high'] = price
if self.current_candle['low'] is None or price < self.current_candle['low']:
self.current_candle['low'] = price
self.current_candle['close'] = price
self.current_candle['volume'] += volume
logger.debug(f"Updated current candle: {self.current_candle}")
def get_dataframe(self) -> pd.DataFrame:
# Include current candle in the dataframe if it exists
timestamps = list(self.timestamps)
opens = list(self.opens)
highs = list(self.highs)
lows = list(self.lows)
closes = list(self.closes)
volumes = list(self.volumes)
if self.current_candle['timestamp'] is not None:
timestamps.append(self.current_candle['timestamp'])
opens.append(self.current_candle['open'])
highs.append(self.current_candle['high'])
lows.append(self.current_candle['low'])
closes.append(self.current_candle['close'])
volumes.append(self.current_candle['volume'])
df = pd.DataFrame({
'timestamp': timestamps,
'open': opens,
'high': highs,
'low': lows,
'close': closes,
'volume': volumes
})
if not df.empty:
df['timestamp'] = pd.to_datetime(df['timestamp'], unit='ms')
return df
class MEXCWebSocket:
"""MEXC-specific WebSocket implementation"""
def __init__(self, symbol: str):
self.symbol = symbol.replace('/', '').upper()
self.ws = None
self.running = False
self.reconnect_delay = 1
self.max_reconnect_delay = 60
self.ping_interval = 20
self.last_ping_time = 0
self.message_count = 0
# MEXC WebSocket configuration
self.ws_url = "wss://wbs-api.mexc.com/ws"
self.ws_sub_params = [
f"spot@public.kline.v3.api@{self.symbol}@Min1"
]
self.subscribe_msgs = [
{
"method": "SUBSCRIPTION",
"params": self.ws_sub_params
}
]
logger.info(f"Initialized MEXC WebSocket for symbol: {self.symbol}")
logger.debug(f"Subscribe messages: {json.dumps(self.subscribe_msgs)}")
async def connect(self):
while True:
try:
logger.info(f"Attempting to connect to {self.ws_url}")
self.ws = await websockets.connect(self.ws_url)
logger.info("WebSocket connection established")
# Subscribe to the streams
for msg in self.subscribe_msgs:
logger.info(f"Sending subscription message: {json.dumps(msg)}")
await self.ws.send(json.dumps(msg))
# Wait for subscription confirmation
response = await self.ws.recv()
logger.info(f"Subscription response: {response}")
if "Not Subscribed" in response:
logger.error(f"Subscription error: {response}")
await self.unsubscribe()
await self.close()
return False
self.running = True
self.reconnect_delay = 1
logger.info(f"Successfully connected to MEXC WebSocket for {self.symbol}")
# Start ping task
asyncio.create_task(self.ping_loop())
return True
except Exception as e:
logger.error(f"WebSocket connection error: {str(e)}")
await self.unsubscribe()
await asyncio.sleep(self.reconnect_delay)
self.reconnect_delay = min(self.reconnect_delay * 2, self.max_reconnect_delay)
continue
async def ping_loop(self):
"""Send ping messages to keep connection alive"""
while self.running:
try:
current_time = time.time()
if current_time - self.last_ping_time >= self.ping_interval:
ping_msg = {"method": "PING"}
logger.debug("Sending ping")
await self.ws.send(json.dumps(ping_msg))
self.last_ping_time = current_time
await asyncio.sleep(1)
except Exception as e:
logger.error(f"Error in ping loop: {str(e)}")
break
async def receive(self) -> Optional[Dict]:
if not self.ws:
return None
try:
message = await self.ws.recv()
self.message_count += 1
if self.message_count % 10 == 0:
logger.info(f"Received message #{self.message_count}")
logger.debug(f"Raw message: {message[:200]}...")
if isinstance(message, bytes):
return None
data = json.loads(message)
# Handle PONG response
if isinstance(data, dict) and data.get('msg') == 'PONG':
logger.debug("Received pong")
return None
# Handle kline data
if isinstance(data, dict) and 'data' in data and isinstance(data['data'], list):
kline = data['data'][0]
if len(kline) >= 6:
kline_data = {
'timestamp': int(kline[0]), # Timestamp
'open': float(kline[1]), # Open
'high': float(kline[2]), # High
'low': float(kline[3]), # Low
'price': float(kline[4]), # Close
'volume': float(kline[5]), # Volume
'type': 'kline'
}
logger.info(f"Processed kline data: {kline_data}")
return kline_data
return None
except websockets.exceptions.ConnectionClosed:
logger.warning("WebSocket connection closed")
self.running = False
return None
except json.JSONDecodeError as e:
logger.error(f"JSON decode error: {str(e)}, message: {message[:200]}...")
return None
except Exception as e:
logger.error(f"Error receiving message: {str(e)}")
return None
async def unsubscribe(self):
"""Unsubscribe from all channels"""
if self.ws:
for msg in self.subscribe_msgs:
unsub_msg = {
"method": "UNSUBSCRIPTION",
"params": msg["params"]
}
try:
await self.ws.send(json.dumps(unsub_msg))
except:
pass
async def close(self):
"""Close the WebSocket connection"""
if self.ws:
await self.unsubscribe()
await self.ws.close()
self.running = False
logger.info("WebSocket connection closed")
class BinanceWebSocket:
"""Binance WebSocket implementation for real-time tick data"""
def __init__(self, symbol: str):
self.symbol = symbol.replace('/', '').lower()
self.ws = None
self.running = False
self.reconnect_delay = 1
self.max_reconnect_delay = 60
self.message_count = 0
# Binance WebSocket configuration
self.ws_url = f"wss://stream.binance.com:9443/ws/{self.symbol}@trade"
logger.info(f"Initialized Binance WebSocket for symbol: {self.symbol}")
async def connect(self):
while True:
try:
logger.info(f"Attempting to connect to {self.ws_url}")
self.ws = await websockets.connect(self.ws_url)
logger.info("WebSocket connection established")
self.running = True
self.reconnect_delay = 1
logger.info(f"Successfully connected to Binance WebSocket for {self.symbol}")
return True
except Exception as e:
logger.error(f"WebSocket connection error: {str(e)}")
await asyncio.sleep(self.reconnect_delay)
self.reconnect_delay = min(self.reconnect_delay * 2, self.max_reconnect_delay)
continue
async def receive(self) -> Optional[Dict]:
if not self.ws:
return None
try:
message = await self.ws.recv()
self.message_count += 1
if self.message_count % 100 == 0: # Log every 100th message to avoid spam
logger.info(f"Received message #{self.message_count}")
logger.debug(f"Raw message: {message[:200]}...")
data = json.loads(message)
# Process trade data
if 'e' in data and data['e'] == 'trade':
trade_data = {
'timestamp': data['T'], # Trade time
'price': float(data['p']), # Price
'volume': float(data['q']), # Quantity
'type': 'trade'
}
logger.debug(f"Processed trade data: {trade_data}")
return trade_data
return None
except websockets.exceptions.ConnectionClosed:
logger.warning("WebSocket connection closed")
self.running = False
return None
except json.JSONDecodeError as e:
logger.error(f"JSON decode error: {str(e)}, message: {message[:200]}...")
return None
except Exception as e:
logger.error(f"Error receiving message: {str(e)}")
return None
async def close(self):
"""Close the WebSocket connection"""
if self.ws:
await self.ws.close()
self.running = False
logger.info("WebSocket connection closed")
class ExchangeWebSocket:
"""Generic WebSocket interface for cryptocurrency exchanges"""
def __init__(self, symbol: str, exchange: str = "binance"):
self.symbol = symbol
self.exchange = exchange.lower()
self.ws = None
# Initialize the appropriate WebSocket implementation
if self.exchange == "binance":
self.ws = BinanceWebSocket(symbol)
elif self.exchange == "mexc":
self.ws = MEXCWebSocket(symbol)
else:
raise ValueError(f"Unsupported exchange: {exchange}")
async def connect(self):
"""Connect to the exchange WebSocket"""
return await self.ws.connect()
async def receive(self) -> Optional[Dict]:
"""Receive data from the WebSocket"""
return await self.ws.receive()
async def close(self):
"""Close the WebSocket connection"""
await self.ws.close()
@property
def running(self):
"""Check if the WebSocket is running"""
return self.ws.running if self.ws else False
class RealTimeChart:
def __init__(self, symbol: str):
self.symbol = symbol
self.app = dash.Dash(__name__)
self.candlestick_data = CandlestickData()
self.tick_storage = TradeTickStorage(max_age_seconds=300) # Store 5 minutes of ticks
logger.info(f"Initializing RealTimeChart for {symbol}")
# Initialize the layout with improved styling
self.app.layout = html.Div([
html.H1(f"{symbol} Real-Time Price", style={
'textAlign': 'center',
'color': '#2c3e50',
'fontFamily': 'Arial, sans-serif',
'marginTop': '20px'
}),
html.Div([
html.Button('1s', id='btn-1s', n_clicks=0, style={'margin': '5px'}),
html.Button('5s', id='btn-5s', n_clicks=0, style={'margin': '5px'}),
html.Button('15s', id='btn-15s', n_clicks=0, style={'margin': '5px'}),
html.Button('30s', id='btn-30s', n_clicks=0, style={'margin': '5px'}),
html.Button('1m', id='btn-1m', n_clicks=0, style={'margin': '5px'}),
], style={'textAlign': 'center', 'margin': '10px'}),
dcc.Store(id='interval-store', data={'interval': 1}), # Store for current interval
dcc.Graph(
id='live-chart',
style={'height': '80vh'}
),
dcc.Interval(
id='interval-component',
interval=500, # Update every 500ms for smoother display
n_intervals=0
)
], style={
'backgroundColor': '#f8f9fa',
'padding': '20px'
})
# Callback to update interval based on button clicks
@self.app.callback(
Output('interval-store', 'data'),
[Input('btn-1s', 'n_clicks'),
Input('btn-5s', 'n_clicks'),
Input('btn-15s', 'n_clicks'),
Input('btn-30s', 'n_clicks'),
Input('btn-1m', 'n_clicks')],
[dash.dependencies.State('interval-store', 'data')]
)
def update_interval(n1, n5, n15, n30, n60, data):
ctx = dash.callback_context
if not ctx.triggered:
return data
button_id = ctx.triggered[0]['prop_id'].split('.')[0]
if button_id == 'btn-1s':
return {'interval': 1}
elif button_id == 'btn-5s':
return {'interval': 5}
elif button_id == 'btn-15s':
return {'interval': 15}
elif button_id == 'btn-30s':
return {'interval': 30}
elif button_id == 'btn-1m':
return {'interval': 60}
return data
# Callback to update the chart
@self.app.callback(
Output('live-chart', 'figure'),
[Input('interval-component', 'n_intervals'),
Input('interval-store', 'data')]
)
def update_chart(n, interval_data):
interval = interval_data.get('interval', 1)
fig = make_subplots(
rows=2, cols=1,
shared_xaxis=True,
vertical_spacing=0.03,
subplot_titles=(f'{self.symbol} Price ({interval}s)', 'Volume'),
row_heights=[0.7, 0.3]
)
# Get candlesticks from tick storage
df = self.tick_storage.get_candles(interval_seconds=interval)
if not df.empty:
# Add candlestick chart
fig.add_trace(
go.Candlestick(
x=df['timestamp'],
open=df['open'],
high=df['high'],
low=df['low'],
close=df['close'],
name='Price',
increasing_line_color='#33CC33', # Green
decreasing_line_color='#FF4136' # Red
),
row=1, col=1
)
# Add volume bars
colors = ['#33CC33' if close >= open else '#FF4136'
for close, open in zip(df['close'], df['open'])]
fig.add_trace(
go.Bar(
x=df['timestamp'],
y=df['volume'],
name='Volume',
marker_color=colors
),
row=2, col=1
)
# Add latest price line
if len(df) > 0:
latest_price = df['close'].iloc[-1]
fig.add_shape(
type="line",
x0=df['timestamp'].min(),
y0=latest_price,
x1=df['timestamp'].max(),
y1=latest_price,
line=dict(color="yellow", width=1, dash="dash"),
row=1, col=1
)
# Add price label
fig.add_annotation(
x=df['timestamp'].max(),
y=latest_price,
text=f"{latest_price:.2f}",
showarrow=False,
font=dict(size=14, color="yellow"),
xshift=50,
row=1, col=1
)
# Update layout with improved styling
fig.update_layout(
title_text=f"{self.symbol} Real-Time Data ({interval}s candles)",
title_x=0.5, # Center the title
xaxis_rangeslider_visible=False,
height=800,
template='plotly_dark',
paper_bgcolor='rgba(0,0,0,0)',
plot_bgcolor='rgba(0,0,0,0)',
font=dict(family="Arial, sans-serif", size=12, color="#2c3e50"),
showlegend=True,
legend=dict(
yanchor="top",
y=0.99,
xanchor="left",
x=0.01
)
)
# Update axes styling
fig.update_xaxes(showgrid=True, gridwidth=1, gridcolor='rgba(128,128,128,0.2)')
fig.update_yaxes(showgrid=True, gridwidth=1, gridcolor='rgba(128,128,128,0.2)')
return fig
async def start_websocket(self):
ws = ExchangeWebSocket(self.symbol)
while True: # Keep trying to maintain connection
if not await ws.connect():
logger.error(f"Failed to connect to exchange for {self.symbol}")
await asyncio.sleep(5)
continue
try:
while True:
if not ws.running:
logger.warning("WebSocket not running, breaking loop")
break
data = await ws.receive()
if data:
if data.get('type') == 'kline':
# Use kline data directly for candlestick
trade_data = {
'timestamp': data['timestamp'],
'price': data['price'],
'volume': data['volume'],
'open': data['open'],
'high': data['high'],
'low': data['low']
}
else:
# Use trade data
trade_data = {
'timestamp': data['timestamp'],
'price': data['price'],
'volume': data['volume']
}
logger.debug(f"Updating candlestick with data: {trade_data}")
# Store raw tick in the tick storage
self.tick_storage.add_tick(trade_data)
# Also update the old candlestick data for backward compatibility
self.candlestick_data.update_from_trade(trade_data)
await asyncio.sleep(0.01)
except Exception as e:
logger.error(f"Error in WebSocket loop: {str(e)}")
finally:
await ws.close()
logger.info("Waiting 5 seconds before reconnecting...")
await asyncio.sleep(5)
def run(self, host='localhost', port=8050):
logger.info(f"Starting Dash server on {host}:{port}")
self.app.run(debug=False, host=host, port=port)
async def main():
symbols = ["ETH/USDT", "BTC/USDT"]
logger.info(f"Starting application for symbols: {symbols}")
charts = []
websocket_tasks = []
# Create a chart and websocket task for each symbol
for symbol in symbols:
chart = RealTimeChart(symbol)
charts.append(chart)
websocket_tasks.append(asyncio.create_task(chart.start_websocket()))
# Run Dash in a separate thread to not block the event loop
server_threads = []
for i, chart in enumerate(charts):
port = 8050 + i # Use different ports for each chart
thread = Thread(target=lambda c=chart, p=8050+i: c.run(port=p)) # Fix lambda capture
thread.daemon = True
thread.start()
server_threads.append(thread)
try:
# Keep the main task running
while True:
await asyncio.sleep(1)
except KeyboardInterrupt:
logger.info("Shutting down...")
except Exception as e:
logger.error(f"Unexpected error: {str(e)}")
finally:
for task in websocket_tasks:
task.cancel()
try:
await task
except asyncio.CancelledError:
pass
if __name__ == "__main__":
try:
asyncio.run(main())
except KeyboardInterrupt:
logger.info("Application terminated by user")

39618
realtime_chart.log Normal file

File diff suppressed because it is too large Load Diff

View File

@ -1,12 +1,6 @@
numpy>=1.21.0
pandas>=1.3.0
matplotlib>=3.4.0
mplfinance>=0.12.7
torch>=1.9.0
python-dotenv>=0.19.0
ccxt>=2.0.0
websockets>=10.0
tensorboard>=2.6.0
scikit-learn>=1.0.0
Pillow>=9.0.0
asyncio>=3.4.3
plotly>=5.18.0
dash>=2.14.0
pandas>=2.0.0
numpy>=1.24.0
python-dotenv>=1.0.0

View File

@ -1,6 +1,5 @@
#!/usr/bin/env python
import asyncio
import argparse
import logging
from main import live_trading, setup_logging
@ -9,32 +8,22 @@ setup_logging()
logger = logging.getLogger(__name__)
async def main():
parser = argparse.ArgumentParser(description='Run live trading in demo mode')
parser.add_argument('--symbol', type=str, default='ETH/USDT', help='Trading pair symbol')
parser.add_argument('--timeframe', type=str, default='1m', help='Timeframe for trading')
parser.add_argument('--model_path', type=str, default='data/best_model.pth', help='Path to the trained model')
parser.add_argument('--initial_balance', type=float, default=1000, help='Initial balance')
parser.add_argument('--update_interval', type=int, default=30, help='Interval to update data in seconds')
"""Run a simplified demo trading session with mock data"""
logger.info("Starting simplified demo trading session")
args = parser.parse_args()
logger.info(f"Starting live trading demo with {args.symbol} on {args.timeframe} timeframe")
# Run live trading in demo mode
# Run live trading in demo mode with simplified parameters
await live_trading(
symbol=args.symbol,
timeframe=args.timeframe,
model_path=args.model_path,
demo=True, # Always use demo mode in this script
initial_balance=args.initial_balance,
update_interval=args.update_interval,
# Using default values for other parameters
symbol="ETH/USDT",
timeframe="1m",
model_path="models/trading_agent_best_pnl.pt",
demo=True,
initial_balance=1000,
update_interval=10, # Update every 10 seconds for faster feedback
max_position_size=0.1,
risk_per_trade=0.02,
stop_loss_pct=0.02,
take_profit_pct=0.04,
)
if __name__ == "__main__":
try:
asyncio.run(main())
except KeyboardInterrupt:
logger.info("Live trading demo stopped by user")
except Exception as e:
logger.error(f"Error in live trading demo: {e}")
asyncio.run(main())

128
tests/test_websocket.py Normal file
View File

@ -0,0 +1,128 @@
import asyncio
import json
import logging
import unittest
from typing import Optional, Dict
import websockets
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)
class TestMEXCWebSocket(unittest.TestCase):
async def test_websocket_connection(self):
"""Test basic WebSocket connection and subscription"""
uri = "wss://stream.mexc.com/ws"
symbol = "ethusdt"
async with websockets.connect(uri) as ws:
# Test subscription to deals
sub_msg = {
"op": "sub",
"id": "test1",
"topic": f"spot.deals.{symbol}"
}
# Send subscription
await ws.send(json.dumps(sub_msg))
# Wait for subscription confirmation and first message
messages_received = 0
trades_received = 0
while messages_received < 5: # Get at least 5 messages
try:
response = await asyncio.wait_for(ws.recv(), timeout=10)
messages_received += 1
logger.info(f"Received message: {response[:200]}...")
data = json.loads(response)
# Check message structure
if isinstance(data, dict):
if 'channel' in data:
if data['channel'] == 'spot.deals':
trades = data.get('data', [])
if trades:
trades_received += 1
logger.info(f"Received trade data: {trades[0]}")
# Verify trade data structure
trade = trades[0]
self.assertIn('t', trade) # timestamp
self.assertIn('p', trade) # price
self.assertIn('v', trade) # volume
self.assertIn('S', trade) # side
except asyncio.TimeoutError:
self.fail("Timeout waiting for WebSocket messages")
# Verify we received some trades
self.assertGreater(trades_received, 0, "No trades received")
# Test unsubscribe
unsub_msg = {
"op": "unsub",
"id": "test1",
"topic": f"spot.deals.{symbol}"
}
await ws.send(json.dumps(unsub_msg))
async def test_kline_subscription(self):
"""Test subscription to kline (candlestick) data"""
uri = "wss://stream.mexc.com/ws"
symbol = "ethusdt"
async with websockets.connect(uri) as ws:
# Subscribe to 1m klines
sub_msg = {
"op": "sub",
"id": "test2",
"topic": f"spot.klines.{symbol}_1m"
}
await ws.send(json.dumps(sub_msg))
messages_received = 0
klines_received = 0
while messages_received < 5:
try:
response = await asyncio.wait_for(ws.recv(), timeout=10)
messages_received += 1
logger.info(f"Received kline message: {response[:200]}...")
data = json.loads(response)
if isinstance(data, dict):
if data.get('channel') == 'spot.klines':
kline_data = data.get('data', [])
if kline_data:
klines_received += 1
logger.info(f"Received kline data: {kline_data[0]}")
# Verify kline data structure (should be an array)
kline = kline_data[0]
self.assertEqual(len(kline), 6) # Should have 6 elements
except asyncio.TimeoutError:
self.fail("Timeout waiting for kline data")
self.assertGreater(klines_received, 0, "No kline data received")
def run_tests():
"""Run the WebSocket tests"""
async def run_async_tests():
# Create test suite
suite = unittest.TestSuite()
suite.addTest(TestMEXCWebSocket('test_websocket_connection'))
suite.addTest(TestMEXCWebSocket('test_kline_subscription'))
# Run tests
runner = unittest.TextTestRunner(verbosity=2)
runner.run(suite)
# Run tests in asyncio loop
asyncio.run(run_async_tests())
if __name__ == "__main__":
run_tests()

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff