BIG CLEANUP
This commit is contained in:
Dobromir Popov
@ -1,13 +0,0 @@
|
||||
"""
|
||||
Neural Network Utilities
|
||||
======================
|
||||
|
||||
This package contains utility functions and classes used in the neural network trading system:
|
||||
- Data Interface: Connects to realtime trading data and processes it for the neural network models
|
||||
"""
|
||||
|
||||
from .data_interface import DataInterface
|
||||
from .trading_env import TradingEnvironment
|
||||
from .signal_interpreter import SignalInterpreter
|
||||
|
||||
__all__ = ['DataInterface', 'TradingEnvironment', 'SignalInterpreter']
|
||||
@ -1,123 +0,0 @@
|
||||
"""
|
||||
Enhanced Data Interface with additional NN trading parameters
|
||||
"""
|
||||
from typing import List, Optional, Tuple
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from datetime import datetime
|
||||
from .data_interface import DataInterface
|
||||
|
||||
class MultiDataInterface(DataInterface):
|
||||
"""
|
||||
Enhanced data interface that supports window_size and output_size parameters
|
||||
for neural network trading models.
|
||||
"""
|
||||
|
||||
def __init__(self, symbol: str,
|
||||
timeframes: List[str],
|
||||
window_size: int = 20,
|
||||
output_size: int = 3,
|
||||
data_dir: str = "NN/data"):
|
||||
"""
|
||||
Initialize with window_size and output_size for NN predictions.
|
||||
"""
|
||||
super().__init__(symbol, timeframes, data_dir)
|
||||
self.window_size = window_size
|
||||
self.output_size = output_size
|
||||
self.scalers = {} # Store scalers for each timeframe
|
||||
self.min_window_threshold = 100 # Minimum candles needed for training
|
||||
|
||||
def get_feature_count(self) -> int:
|
||||
"""
|
||||
Get number of features (OHLCV) for NN input.
|
||||
"""
|
||||
return 5 # open, high, low, close, volume
|
||||
|
||||
def prepare_training_data(self) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]:
|
||||
"""Prepare training data with windowed sequences"""
|
||||
# Get historical data for primary timeframe
|
||||
primary_tf = self.timeframes[0]
|
||||
df = self.get_historical_data(timeframe=primary_tf,
|
||||
n_candles=self.min_window_threshold + 1000)
|
||||
|
||||
if df is None or len(df) < self.min_window_threshold:
|
||||
raise ValueError(f"Insufficient data for training. Need at least {self.min_window_threshold} candles")
|
||||
|
||||
# Prepare OHLCV sequences
|
||||
ohlcv = df[['open', 'high', 'low', 'close', 'volume']].values
|
||||
|
||||
# Create sequences and labels
|
||||
X = []
|
||||
y = []
|
||||
|
||||
for i in range(len(ohlcv) - self.window_size - self.output_size):
|
||||
# Input sequence
|
||||
seq = ohlcv[i:i+self.window_size]
|
||||
X.append(seq)
|
||||
|
||||
# Output target (price movement direction)
|
||||
close_prices = ohlcv[i+self.window_size:i+self.window_size+self.output_size, 3] # Close prices
|
||||
price_changes = np.diff(close_prices)
|
||||
|
||||
if self.output_size == 1:
|
||||
# Binary classification (up/down)
|
||||
label = 1 if price_changes[0] > 0 else 0
|
||||
elif self.output_size == 3:
|
||||
# 3-class classification (buy/hold/sell)
|
||||
if price_changes[0] > 0.002: # Significant rise
|
||||
label = 0 # Buy
|
||||
elif price_changes[0] < -0.002: # Significant drop
|
||||
label = 2 # Sell
|
||||
else:
|
||||
label = 1 # Hold
|
||||
else:
|
||||
raise ValueError(f"Unsupported output_size: {self.output_size}")
|
||||
|
||||
y.append(label)
|
||||
|
||||
# Convert to numpy arrays
|
||||
X = np.array(X)
|
||||
y = np.array(y)
|
||||
|
||||
# Split into train/validation (80/20)
|
||||
split_idx = int(0.8 * len(X))
|
||||
X_train, y_train = X[:split_idx], y[:split_idx]
|
||||
X_val, y_val = X[split_idx:], y[split_idx:]
|
||||
|
||||
return X_train, y_train, X_val, y_val
|
||||
|
||||
def prepare_prediction_data(self) -> np.ndarray:
|
||||
"""Prepare most recent window for predictions"""
|
||||
primary_tf = self.timeframes[0]
|
||||
df = self.get_historical_data(timeframe=primary_tf,
|
||||
n_candles=self.window_size,
|
||||
use_cache=False)
|
||||
|
||||
if df is None or len(df) < self.window_size:
|
||||
raise ValueError(f"Need at least {self.window_size} candles for prediction")
|
||||
|
||||
ohlcv = df[['open', 'high', 'low', 'close', 'volume']].values[-self.window_size:]
|
||||
return np.array([ohlcv]) # Add batch dimension
|
||||
|
||||
def process_predictions(self, predictions: np.ndarray):
|
||||
"""Convert prediction probabilities to trading signals"""
|
||||
signals = []
|
||||
for pred in predictions:
|
||||
if self.output_size == 1:
|
||||
signal = "BUY" if pred[0] > 0.5 else "SELL"
|
||||
confidence = np.abs(pred[0] - 0.5) * 2 # Convert to 0-1 scale
|
||||
elif self.output_size == 3:
|
||||
action_idx = np.argmax(pred)
|
||||
signal = ["BUY", "HOLD", "SELL"][action_idx]
|
||||
confidence = pred[action_idx]
|
||||
else:
|
||||
signal = "HOLD"
|
||||
confidence = 0.0
|
||||
|
||||
signals.append({
|
||||
'action': signal,
|
||||
'confidence': confidence,
|
||||
'timestamp': datetime.now().isoformat()
|
||||
})
|
||||
|
||||
return signals
|
||||
@ -1,364 +0,0 @@
|
||||
"""
|
||||
Realtime Analyzer for Neural Network Trading System
|
||||
|
||||
This module implements real-time analysis of market data using trained neural network models.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
import numpy as np
|
||||
from threading import Thread
|
||||
from queue import Queue
|
||||
from datetime import datetime
|
||||
import asyncio
|
||||
import websockets
|
||||
import json
|
||||
import os
|
||||
import pandas as pd
|
||||
from collections import deque
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class RealtimeAnalyzer:
|
||||
"""
|
||||
Handles real-time analysis of market data using trained neural network models.
|
||||
|
||||
Features:
|
||||
- Connects to real-time data sources (websockets)
|
||||
- Processes tick data into multiple timeframes (1s, 1m, 1h, 1d)
|
||||
- Uses trained models to analyze all timeframes
|
||||
- Generates trading signals
|
||||
- Manages risk and position sizing
|
||||
- Logs all trading decisions
|
||||
"""
|
||||
|
||||
def __init__(self, data_interface, model, symbol="BTC/USDT", timeframes=None):
|
||||
"""
|
||||
Initialize the realtime analyzer.
|
||||
|
||||
Args:
|
||||
data_interface (DataInterface): Preconfigured data interface
|
||||
model: Trained neural network model
|
||||
symbol (str): Trading pair symbol
|
||||
timeframes (list): List of timeframes to monitor (default: ['1s', '1m', '1h', '1d'])
|
||||
"""
|
||||
self.data_interface = data_interface
|
||||
self.model = model
|
||||
self.symbol = symbol
|
||||
self.timeframes = timeframes or ['1s', '1m', '1h', '1d']
|
||||
self.running = False
|
||||
self.data_queue = Queue()
|
||||
self.prediction_interval = 10 # Seconds between predictions
|
||||
self.ws_url = f"wss://stream.binance.com:9443/ws/{symbol.replace('/', '').lower()}@trade"
|
||||
self.ws = None
|
||||
self.tick_storage = deque(maxlen=10000) # Store up to 10,000 ticks
|
||||
self.candle_cache = {
|
||||
'1s': deque(maxlen=5000),
|
||||
'1m': deque(maxlen=5000),
|
||||
'1h': deque(maxlen=5000),
|
||||
'1d': deque(maxlen=5000)
|
||||
}
|
||||
|
||||
logger.info(f"RealtimeAnalyzer initialized for {symbol} with timeframes: {self.timeframes}")
|
||||
|
||||
def start(self):
|
||||
"""Start the realtime analysis process."""
|
||||
if self.running:
|
||||
logger.warning("Realtime analyzer already running")
|
||||
return
|
||||
|
||||
self.running = True
|
||||
|
||||
# Start WebSocket connection thread
|
||||
self.ws_thread = Thread(target=self._run_websocket, daemon=True)
|
||||
self.ws_thread.start()
|
||||
|
||||
# Start data processing thread
|
||||
self.processing_thread = Thread(target=self._process_data, daemon=True)
|
||||
self.processing_thread.start()
|
||||
|
||||
# Start analysis thread
|
||||
self.analysis_thread = Thread(target=self._analyze_data, daemon=True)
|
||||
self.analysis_thread.start()
|
||||
|
||||
logger.info("Realtime analysis started")
|
||||
|
||||
def stop(self):
|
||||
"""Stop the realtime analysis process."""
|
||||
self.running = False
|
||||
if self.ws:
|
||||
asyncio.run(self.ws.close())
|
||||
if hasattr(self, 'ws_thread'):
|
||||
self.ws_thread.join(timeout=1)
|
||||
if hasattr(self, 'processing_thread'):
|
||||
self.processing_thread.join(timeout=1)
|
||||
if hasattr(self, 'analysis_thread'):
|
||||
self.analysis_thread.join(timeout=1)
|
||||
logger.info("Realtime analysis stopped")
|
||||
|
||||
def _run_websocket(self):
|
||||
"""Thread function for running WebSocket connection."""
|
||||
loop = asyncio.new_event_loop()
|
||||
asyncio.set_event_loop(loop)
|
||||
loop.run_until_complete(self._connect_websocket())
|
||||
|
||||
async def _connect_websocket(self):
|
||||
"""Connect to WebSocket and receive data."""
|
||||
while self.running:
|
||||
try:
|
||||
logger.info(f"Connecting to WebSocket: {self.ws_url}")
|
||||
async with websockets.connect(self.ws_url) as ws:
|
||||
self.ws = ws
|
||||
logger.info("WebSocket connected")
|
||||
|
||||
while self.running:
|
||||
try:
|
||||
message = await ws.recv()
|
||||
data = json.loads(message)
|
||||
|
||||
if 'e' in data and data['e'] == 'trade':
|
||||
tick = {
|
||||
'timestamp': data['T'],
|
||||
'price': float(data['p']),
|
||||
'volume': float(data['q']),
|
||||
'symbol': self.symbol
|
||||
}
|
||||
self.tick_storage.append(tick)
|
||||
self.data_queue.put(tick)
|
||||
|
||||
except websockets.exceptions.ConnectionClosed:
|
||||
logger.warning("WebSocket connection closed")
|
||||
break
|
||||
except Exception as e:
|
||||
logger.error(f"Error receiving WebSocket message: {str(e)}")
|
||||
time.sleep(1)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"WebSocket connection error: {str(e)}")
|
||||
time.sleep(5) # Wait before reconnecting
|
||||
|
||||
def _process_data(self):
|
||||
"""Process incoming tick data into candles for all timeframes."""
|
||||
logger.info("Starting data processing thread")
|
||||
|
||||
while self.running:
|
||||
try:
|
||||
# Process any new ticks
|
||||
while not self.data_queue.empty():
|
||||
tick = self.data_queue.get()
|
||||
|
||||
# Convert timestamp to datetime
|
||||
timestamp = datetime.fromtimestamp(tick['timestamp'] / 1000)
|
||||
|
||||
# Process for each timeframe
|
||||
for timeframe in self.timeframes:
|
||||
interval = self._get_interval_seconds(timeframe)
|
||||
if interval is None:
|
||||
continue
|
||||
|
||||
# Round timestamp to nearest candle interval
|
||||
candle_ts = int(tick['timestamp'] // (interval * 1000)) * (interval * 1000)
|
||||
|
||||
# Get or create candle for this timeframe
|
||||
if not self.candle_cache[timeframe]:
|
||||
# First candle for this timeframe
|
||||
candle = {
|
||||
'timestamp': candle_ts,
|
||||
'open': tick['price'],
|
||||
'high': tick['price'],
|
||||
'low': tick['price'],
|
||||
'close': tick['price'],
|
||||
'volume': tick['volume']
|
||||
}
|
||||
self.candle_cache[timeframe].append(candle)
|
||||
else:
|
||||
# Update existing candle
|
||||
last_candle = self.candle_cache[timeframe][-1]
|
||||
|
||||
if last_candle['timestamp'] == candle_ts:
|
||||
# Update current candle
|
||||
last_candle['high'] = max(last_candle['high'], tick['price'])
|
||||
last_candle['low'] = min(last_candle['low'], tick['price'])
|
||||
last_candle['close'] = tick['price']
|
||||
last_candle['volume'] += tick['volume']
|
||||
else:
|
||||
# New candle
|
||||
candle = {
|
||||
'timestamp': candle_ts,
|
||||
'open': tick['price'],
|
||||
'high': tick['price'],
|
||||
'low': tick['price'],
|
||||
'close': tick['price'],
|
||||
'volume': tick['volume']
|
||||
}
|
||||
self.candle_cache[timeframe].append(candle)
|
||||
|
||||
time.sleep(0.1)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in data processing: {str(e)}")
|
||||
time.sleep(1)
|
||||
|
||||
def _get_interval_seconds(self, timeframe):
|
||||
"""Convert timeframe string to seconds."""
|
||||
intervals = {
|
||||
'1s': 1,
|
||||
'1m': 60,
|
||||
'1h': 3600,
|
||||
'1d': 86400
|
||||
}
|
||||
return intervals.get(timeframe)
|
||||
|
||||
def _analyze_data(self):
|
||||
"""Thread function for analyzing data and generating signals."""
|
||||
logger.info("Starting analysis thread")
|
||||
|
||||
last_prediction_time = 0
|
||||
|
||||
while self.running:
|
||||
try:
|
||||
current_time = time.time()
|
||||
|
||||
# Only make predictions at the specified interval
|
||||
if current_time - last_prediction_time < self.prediction_interval:
|
||||
time.sleep(0.1)
|
||||
continue
|
||||
|
||||
# Prepare input data from all timeframes
|
||||
input_data = {}
|
||||
valid = True
|
||||
|
||||
for timeframe in self.timeframes:
|
||||
if not self.candle_cache[timeframe]:
|
||||
logger.warning(f"No data available for timeframe {timeframe}")
|
||||
valid = False
|
||||
break
|
||||
|
||||
# Get last N candles for this timeframe
|
||||
candles = list(self.candle_cache[timeframe])[-self.data_interface.window_size:]
|
||||
|
||||
# Convert to numpy array
|
||||
ohlcv = np.array([
|
||||
[c['open'], c['high'], c['low'], c['close'], c['volume']]
|
||||
for c in candles
|
||||
])
|
||||
|
||||
# Normalize data
|
||||
ohlcv_normalized = (ohlcv - ohlcv.mean(axis=0)) / (ohlcv.std(axis=0) + 1e-8)
|
||||
input_data[timeframe] = ohlcv_normalized
|
||||
|
||||
if not valid:
|
||||
time.sleep(0.1)
|
||||
continue
|
||||
|
||||
# Make prediction using the model
|
||||
try:
|
||||
prediction = self.model.predict(input_data)
|
||||
|
||||
# Get latest timestamp from 1s timeframe
|
||||
latest_ts = self.candle_cache['1s'][-1]['timestamp'] if self.candle_cache['1s'] else int(time.time() * 1000)
|
||||
|
||||
# Process prediction
|
||||
self._process_prediction(
|
||||
prediction=prediction,
|
||||
timeframe='multi',
|
||||
timestamp=latest_ts
|
||||
)
|
||||
|
||||
last_prediction_time = current_time
|
||||
except Exception as e:
|
||||
logger.error(f"Error making prediction: {str(e)}")
|
||||
|
||||
time.sleep(0.1)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error in analysis: {str(e)}")
|
||||
time.sleep(1)
|
||||
|
||||
def _process_prediction(self, prediction, timeframe, timestamp):
|
||||
"""
|
||||
Process model prediction and generate trading signals.
|
||||
|
||||
Args:
|
||||
prediction: Model prediction output
|
||||
timeframe (str): Timeframe the prediction is for ('multi' for combined)
|
||||
timestamp: Timestamp of the prediction (ms)
|
||||
"""
|
||||
# Convert prediction to trading signal
|
||||
signal, confidence = self._prediction_to_signal(prediction)
|
||||
|
||||
# Convert timestamp to datetime
|
||||
try:
|
||||
dt = datetime.fromtimestamp(timestamp / 1000)
|
||||
except:
|
||||
dt = datetime.now()
|
||||
|
||||
# Log the signal with all timeframes
|
||||
logger.info(
|
||||
f"Signal generated - Timeframes: {', '.join(self.timeframes)}, "
|
||||
f"Timestamp: {dt}, "
|
||||
f"Signal: {signal} (Confidence: {confidence:.2f})"
|
||||
)
|
||||
|
||||
# In a real implementation, we would execute trades here
|
||||
# For now, we'll just log the signals
|
||||
|
||||
def _prediction_to_signal(self, prediction):
|
||||
"""
|
||||
Convert model prediction to trading signal and confidence.
|
||||
|
||||
Args:
|
||||
prediction: Model prediction output (can be dict for multi-timeframe)
|
||||
|
||||
Returns:
|
||||
tuple: (signal, confidence) where signal is BUY/SELL/HOLD,
|
||||
confidence is probability (0-1)
|
||||
"""
|
||||
if isinstance(prediction, dict):
|
||||
# Multi-timeframe prediction - combine signals
|
||||
signals = []
|
||||
confidences = []
|
||||
|
||||
for tf, pred in prediction.items():
|
||||
if len(pred.shape) == 1:
|
||||
# Binary classification
|
||||
signal = "BUY" if pred[0] > 0.5 else "SELL"
|
||||
confidence = pred[0] if signal == "BUY" else 1 - pred[0]
|
||||
else:
|
||||
# Multi-class
|
||||
class_idx = np.argmax(pred)
|
||||
signal = ["SELL", "HOLD", "BUY"][class_idx]
|
||||
confidence = pred[class_idx]
|
||||
|
||||
signals.append(signal)
|
||||
confidences.append(confidence)
|
||||
|
||||
# Simple voting system - count BUY/SELL signals
|
||||
buy_count = signals.count("BUY")
|
||||
sell_count = signals.count("SELL")
|
||||
|
||||
if buy_count > sell_count:
|
||||
final_signal = "BUY"
|
||||
final_confidence = np.mean([c for s, c in zip(signals, confidences) if s == "BUY"])
|
||||
elif sell_count > buy_count:
|
||||
final_signal = "SELL"
|
||||
final_confidence = np.mean([c for s, c in zip(signals, confidences) if s == "SELL"])
|
||||
else:
|
||||
final_signal = "HOLD"
|
||||
final_confidence = np.mean(confidences)
|
||||
|
||||
return final_signal, final_confidence
|
||||
|
||||
else:
|
||||
# Single prediction
|
||||
if len(prediction.shape) == 1:
|
||||
# Binary classification
|
||||
signal = "BUY" if prediction[0] > 0.5 else "SELL"
|
||||
confidence = prediction[0] if signal == "BUY" else 1 - prediction[0]
|
||||
else:
|
||||
# Multi-class
|
||||
class_idx = np.argmax(prediction)
|
||||
signal = ["SELL", "HOLD", "BUY"][class_idx]
|
||||
confidence = prediction[class_idx]
|
||||
|
||||
return signal, confidence
|
||||
Reference in New Issue
Block a user