popov/gogo2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

working training

Browse Source
This commit is contained in:
Dobromir Popov
2025-03-29 02:18:25 +02:00
parent 0b2000e3e7
commit 2255a8363a
4 changed files with 314 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files

300
NN/utils/realtime_analyzer.py
View File

@@ -10,6 +10,12 @@ import numpy as np
from threading import Thread
from queue import Queue
from datetime import datetime
import asyncio
import websockets
import json
import os
import pandas as pd
from collections import deque
logger = logging.getLogger(__name__)
@@ -19,7 +25,8 @@ class RealtimeAnalyzer:
Features:
- Connects to real-time data sources (websockets)
- Processes incoming data through the neural network
- Processes tick data into multiple timeframes (1s, 1m, 1h, 1d)
- Uses trained models to analyze all timeframes
- Generates trading signals
- Manages risk and position sizing
- Logs all trading decisions
@@ -33,17 +40,26 @@ class RealtimeAnalyzer:
data_interface (DataInterface): Preconfigured data interface
model: Trained neural network model
symbol (str): Trading pair symbol
timeframes (list): List of timeframes to monitor
timeframes (list): List of timeframes to monitor (default: ['1s', '1m', '1h', '1d'])
"""
self.data_interface = data_interface
self.model = model
self.symbol = symbol
self.timeframes = timeframes or ['1h']
self.timeframes = timeframes or ['1s', '1m', '1h', '1d']
self.running = False
self.data_queue = Queue()
self.prediction_interval = 60 # Seconds between predictions
self.prediction_interval = 10 # Seconds between predictions
self.ws_url = f"wss://stream.binance.com:9443/ws/{symbol.replace('/', '').lower()}@trade"
self.ws = None
self.tick_storage = deque(maxlen=10000) # Store up to 10,000 ticks
self.candle_cache = {
'1s': deque(maxlen=5000),
'1m': deque(maxlen=5000),
'1h': deque(maxlen=5000),
'1d': deque(maxlen=5000)
}
logger.info(f"RealtimeAnalyzer initialized for {symbol}")
logger.info(f"RealtimeAnalyzer initialized for {symbol} with timeframes: {self.timeframes}")
def start(self):
"""Start the realtime analysis process."""
@@ -53,9 +69,13 @@ class RealtimeAnalyzer:
self.running = True
# Start data collection thread
self.data_thread = Thread(target=self._collect_data, daemon=True)
self.data_thread.start()
# Start WebSocket connection thread
self.ws_thread = Thread(target=self._run_websocket, daemon=True)
self.ws_thread.start()
# Start data processing thread
self.processing_thread = Thread(target=self._process_data, daemon=True)
self.processing_thread.start()
# Start analysis thread
self.analysis_thread = Thread(target=self._analyze_data, daemon=True)
@@ -66,42 +86,128 @@ class RealtimeAnalyzer:
def stop(self):
"""Stop the realtime analysis process."""
self.running = False
if hasattr(self, 'data_thread'):
self.data_thread.join(timeout=1)
if self.ws:
asyncio.run(self.ws.close())
if hasattr(self, 'ws_thread'):
self.ws_thread.join(timeout=1)
if hasattr(self, 'processing_thread'):
self.processing_thread.join(timeout=1)
if hasattr(self, 'analysis_thread'):
self.analysis_thread.join(timeout=1)
logger.info("Realtime analysis stopped")
def _collect_data(self):
"""Thread function for collecting real-time data."""
logger.info("Starting data collection thread")
# In a real implementation, this would connect to websockets/API
# For now, we'll simulate data collection from the data interface
def _run_websocket(self):
"""Thread function for running WebSocket connection."""
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(self._connect_websocket())
async def _connect_websocket(self):
"""Connect to WebSocket and receive data."""
while self.running:
try:
# Get latest data for each timeframe
for timeframe in self.timeframes:
# Get recent data (simulating real-time updates)
X, timestamp = self.data_interface.prepare_realtime_input(
timeframe=timeframe,
n_candles=30,
window_size=self.data_interface.window_size
)
logger.info(f"Connecting to WebSocket: {self.ws_url}")
async with websockets.connect(self.ws_url) as ws:
self.ws = ws
logger.info("WebSocket connected")
if X is not None:
self.data_queue.put({
'timeframe': timeframe,
'data': X,
'timestamp': timestamp
})
while self.running:
try:
message = await ws.recv()
data = json.loads(message)
if 'e' in data and data['e'] == 'trade':
tick = {
'timestamp': data['T'],
'price': float(data['p']),
'volume': float(data['q']),
'symbol': self.symbol
}
self.tick_storage.append(tick)
self.data_queue.put(tick)
except websockets.exceptions.ConnectionClosed:
logger.warning("WebSocket connection closed")
break
except Exception as e:
logger.error(f"Error receiving WebSocket message: {str(e)}")
time.sleep(1)
except Exception as e:
logger.error(f"WebSocket connection error: {str(e)}")
time.sleep(5) # Wait before reconnecting
def _process_data(self):
"""Process incoming tick data into candles for all timeframes."""
logger.info("Starting data processing thread")
while self.running:
try:
# Process any new ticks
while not self.data_queue.empty():
tick = self.data_queue.get()
# Convert timestamp to datetime
timestamp = datetime.fromtimestamp(tick['timestamp'] / 1000)
# Process for each timeframe
for timeframe in self.timeframes:
interval = self._get_interval_seconds(timeframe)
if interval is None:
continue
# Round timestamp to nearest candle interval
candle_ts = int(tick['timestamp'] // (interval * 1000)) * (interval * 1000)
# Get or create candle for this timeframe
if not self.candle_cache[timeframe]:
# First candle for this timeframe
candle = {
'timestamp': candle_ts,
'open': tick['price'],
'high': tick['price'],
'low': tick['price'],
'close': tick['price'],
'volume': tick['volume']
}
self.candle_cache[timeframe].append(candle)
else:
# Update existing candle
last_candle = self.candle_cache[timeframe][-1]
if last_candle['timestamp'] == candle_ts:
# Update current candle
last_candle['high'] = max(last_candle['high'], tick['price'])
last_candle['low'] = min(last_candle['low'], tick['price'])
last_candle['close'] = tick['price']
last_candle['volume'] += tick['volume']
else:
# New candle
candle = {
'timestamp': candle_ts,
'open': tick['price'],
'high': tick['price'],
'low': tick['price'],
'close': tick['price'],
'volume': tick['volume']
}
self.candle_cache[timeframe].append(candle)
# Throttle data collection
time.sleep(1)
time.sleep(0.1)
except Exception as e:
logger.error(f"Error in data collection: {str(e)}")
time.sleep(5) # Wait before retrying
logger.error(f"Error in data processing: {str(e)}")
time.sleep(1)
def _get_interval_seconds(self, timeframe):
"""Convert timeframe string to seconds."""
intervals = {
'1s': 1,
'1m': 60,
'1h': 3600,
'1d': 86400
}
return intervals.get(timeframe)
def _analyze_data(self):
"""Thread function for analyzing data and generating signals."""
@@ -118,27 +224,56 @@ class RealtimeAnalyzer:
time.sleep(0.1)
continue
# Get latest data from queue
if not self.data_queue.empty():
data_item = self.data_queue.get()
# Prepare input data from all timeframes
input_data = {}
valid = True
for timeframe in self.timeframes:
if not self.candle_cache[timeframe]:
logger.warning(f"No data available for timeframe {timeframe}")
valid = False
break
# Get last N candles for this timeframe
candles = list(self.candle_cache[timeframe])[-self.data_interface.window_size:]
# Make prediction
prediction = self.model.predict(data_item['data'])
# Convert to numpy array
ohlcv = np.array([
[c['open'], c['high'], c['low'], c['close'], c['volume']]
for c in candles
])
# Normalize data
ohlcv_normalized = (ohlcv - ohlcv.mean(axis=0)) / (ohlcv.std(axis=0) + 1e-8)
input_data[timeframe] = ohlcv_normalized
if not valid:
time.sleep(0.1)
continue
# Make prediction using the model
try:
prediction = self.model.predict(input_data)
# Get latest timestamp from 1s timeframe
latest_ts = self.candle_cache['1s'][-1]['timestamp'] if self.candle_cache['1s'] else int(time.time() * 1000)
# Process prediction
self._process_prediction(
prediction=prediction,
timeframe=data_item['timeframe'],
timestamp=data_item['timestamp']
timeframe='multi',
timestamp=latest_ts
)
last_prediction_time = current_time
except Exception as e:
logger.error(f"Error making prediction: {str(e)}")
time.sleep(0.1)
except Exception as e:
logger.error(f"Error in analysis: {str(e)}")
time.sleep(1) # Wait before retrying
time.sleep(1)
def _process_prediction(self, prediction, timeframe, timestamp):
"""
@@ -146,17 +281,23 @@ class RealtimeAnalyzer:
Args:
prediction: Model prediction output
timeframe (str): Timeframe the prediction is for
timestamp: Timestamp of the prediction
timeframe (str): Timeframe the prediction is for ('multi' for combined)
timestamp: Timestamp of the prediction (ms)
"""
# Convert prediction to trading signal
signal = self._prediction_to_signal(prediction)
signal, confidence = self._prediction_to_signal(prediction)
# Log the signal
# Convert timestamp to datetime
try:
dt = datetime.fromtimestamp(timestamp / 1000)
except:
dt = datetime.now()
# Log the signal with all timeframes
logger.info(
f"Signal generated - Timeframe: {timeframe}, "
f"Timestamp: {timestamp}, "
f"Signal: {signal}"
f"Signal generated - Timeframes: {', '.join(self.timeframes)}, "
f"Timestamp: {dt}, "
f"Signal: {signal} (Confidence: {confidence:.2f})"
)
# In a real implementation, we would execute trades here
@@ -164,19 +305,60 @@ class RealtimeAnalyzer:
def _prediction_to_signal(self, prediction):
"""
Convert model prediction to trading signal.
Convert model prediction to trading signal and confidence.
Args:
prediction: Model prediction output
prediction: Model prediction output (can be dict for multi-timeframe)
Returns:
str: Trading signal (BUY, SELL, HOLD)
tuple: (signal, confidence) where signal is BUY/SELL/HOLD,
confidence is probability (0-1)
"""
# Simple threshold-based signal generation
if len(prediction.shape) == 1:
# Binary classification
return "BUY" if prediction[0] > 0.5 else "SELL"
if isinstance(prediction, dict):
# Multi-timeframe prediction - combine signals
signals = []
confidences = []
for tf, pred in prediction.items():
if len(pred.shape) == 1:
# Binary classification
signal = "BUY" if pred[0] > 0.5 else "SELL"
confidence = pred[0] if signal == "BUY" else 1 - pred[0]
else:
# Multi-class
class_idx = np.argmax(pred)
signal = ["SELL", "HOLD", "BUY"][class_idx]
confidence = pred[class_idx]
signals.append(signal)
confidences.append(confidence)
# Simple voting system - count BUY/SELL signals
buy_count = signals.count("BUY")
sell_count = signals.count("SELL")
if buy_count > sell_count:
final_signal = "BUY"
final_confidence = np.mean([c for s, c in zip(signals, confidences) if s == "BUY"])
elif sell_count > buy_count:
final_signal = "SELL"
final_confidence = np.mean([c for s, c in zip(signals, confidences) if s == "SELL"])
else:
final_signal = "HOLD"
final_confidence = np.mean(confidences)
return final_signal, final_confidence
else:
# Multi-class classification (3 outputs)
class_idx = np.argmax(prediction)
return ["SELL", "HOLD", "BUY"][class_idx]
# Single prediction
if len(prediction.shape) == 1:
# Binary classification
signal = "BUY" if prediction[0] > 0.5 else "SELL"
confidence = prediction[0] if signal == "BUY" else 1 - prediction[0]
else:
# Multi-class
class_idx = np.argmax(prediction)
signal = ["SELL", "HOLD", "BUY"][class_idx]
confidence = prediction[class_idx]
return signal, confidence