LLM proxy integration
This commit is contained in:
Dobromir Popov
@@ -6,6 +6,15 @@ system:
|
||||
log_level: "INFO" # DEBUG, INFO, WARNING, ERROR
|
||||
session_timeout: 3600 # Session timeout in seconds
|
||||
|
||||
# LLM Proxy Configuration
|
||||
llm_proxy:
|
||||
base_url: "http://localhost:1234" # LLM server base URL
|
||||
model: "openai/gpt-oss-20b" # Model name
|
||||
temperature: 0.7 # Response creativity (0.0-1.0)
|
||||
max_tokens: -1 # Max response tokens (-1 for unlimited)
|
||||
timeout: 30 # Request timeout in seconds
|
||||
api_key: null # API key if required
|
||||
|
||||
# Cold Start Mode Configuration
|
||||
cold_start:
|
||||
enabled: true # Enable cold start mode logic
|
||||
|
||||
383
core/llm_proxy.py
Normal file
383
core/llm_proxy.py
Normal file
@@ -0,0 +1,383 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
LLM Proxy Model - Interface for LLM-based trading signals
|
||||
Sends market data to LLM endpoint and parses responses for trade signals
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import requests
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime
|
||||
from typing import Dict, List, Optional, Any, Tuple
|
||||
from dataclasses import dataclass
|
||||
import os
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@dataclass
|
||||
class LLMTradeSignal:
|
||||
"""Trade signal from LLM"""
|
||||
symbol: str
|
||||
action: str # 'BUY', 'SELL', 'HOLD'
|
||||
confidence: float # 0.0 to 1.0
|
||||
reasoning: str
|
||||
price_target: Optional[float] = None
|
||||
stop_loss: Optional[float] = None
|
||||
timestamp: Optional[datetime] = None
|
||||
|
||||
@dataclass
|
||||
class LLMConfig:
|
||||
"""LLM configuration"""
|
||||
base_url: str = "http://localhost:1234"
|
||||
model: str = "openai/gpt-oss-20b"
|
||||
temperature: float = 0.7
|
||||
max_tokens: int = -1
|
||||
timeout: int = 30
|
||||
api_key: Optional[str] = None
|
||||
|
||||
class LLMProxy:
|
||||
"""
|
||||
LLM Proxy for trading signal generation
|
||||
|
||||
Features:
|
||||
- Configurable LLM endpoint and model
|
||||
- Processes market data from TextDataExporter files
|
||||
- Generates structured trading signals
|
||||
- Thread-safe operations
|
||||
- Error handling and retry logic
|
||||
"""
|
||||
|
||||
def __init__(self,
|
||||
config: Optional[LLMConfig] = None,
|
||||
data_dir: str = "NN/training/samples/txt"):
|
||||
"""
|
||||
Initialize LLM proxy
|
||||
|
||||
Args:
|
||||
config: LLM configuration
|
||||
data_dir: Directory to watch for market data files
|
||||
"""
|
||||
self.config = config or LLMConfig()
|
||||
self.data_dir = data_dir
|
||||
|
||||
# Processing state
|
||||
self.is_running = False
|
||||
self.processing_thread = None
|
||||
self.processed_files = set()
|
||||
|
||||
# Signal storage
|
||||
self.latest_signals: Dict[str, LLMTradeSignal] = {}
|
||||
self.signal_history: List[LLMTradeSignal] = []
|
||||
self.lock = threading.Lock()
|
||||
|
||||
# System prompt for trading
|
||||
self.system_prompt = """You are an expert cryptocurrency trading analyst.
|
||||
You will receive market data for ETH (main symbol) with reference data for BTC and SPX.
|
||||
Analyze the multi-timeframe data (1s, 1m, 1h, 1d) and provide trading recommendations.
|
||||
|
||||
Respond ONLY with valid JSON in this format:
|
||||
{
|
||||
"action": "BUY|SELL|HOLD",
|
||||
"confidence": 0.0-1.0,
|
||||
"reasoning": "brief analysis",
|
||||
"price_target": number_or_null,
|
||||
"stop_loss": number_or_null
|
||||
}
|
||||
|
||||
Consider market correlations, timeframe divergences, and risk management.
|
||||
"""
|
||||
|
||||
logger.info(f"LLM Proxy initialized - Model: {self.config.model}")
|
||||
logger.info(f"Watching directory: {self.data_dir}")
|
||||
|
||||
def start(self):
|
||||
"""Start LLM processing"""
|
||||
if self.is_running:
|
||||
logger.warning("LLM proxy already running")
|
||||
return
|
||||
|
||||
self.is_running = True
|
||||
self.processing_thread = threading.Thread(target=self._processing_loop, daemon=True)
|
||||
self.processing_thread.start()
|
||||
logger.info("LLM proxy started")
|
||||
|
||||
def stop(self):
|
||||
"""Stop LLM processing"""
|
||||
self.is_running = False
|
||||
if self.processing_thread:
|
||||
self.processing_thread.join(timeout=5)
|
||||
logger.info("LLM proxy stopped")
|
||||
|
||||
def _processing_loop(self):
|
||||
"""Main processing loop - checks for new files"""
|
||||
while self.is_running:
|
||||
try:
|
||||
self._check_for_new_files()
|
||||
time.sleep(5) # Check every 5 seconds
|
||||
except Exception as e:
|
||||
logger.error(f"Error in LLM processing loop: {e}")
|
||||
time.sleep(5)
|
||||
|
||||
def _check_for_new_files(self):
|
||||
"""Check for new market data files"""
|
||||
try:
|
||||
if not os.path.exists(self.data_dir):
|
||||
return
|
||||
|
||||
txt_files = [f for f in os.listdir(self.data_dir)
|
||||
if f.endswith('.txt') and f.startswith('market_data_')]
|
||||
|
||||
for filename in txt_files:
|
||||
if filename not in self.processed_files:
|
||||
filepath = os.path.join(self.data_dir, filename)
|
||||
self._process_file(filepath, filename)
|
||||
self.processed_files.add(filename)
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error checking for new files: {e}")
|
||||
|
||||
def _process_file(self, filepath: str, filename: str):
|
||||
"""Process a market data file"""
|
||||
try:
|
||||
logger.info(f"Processing market data file: {filename}")
|
||||
|
||||
# Read and parse market data
|
||||
market_data = self._parse_market_data(filepath)
|
||||
if not market_data:
|
||||
logger.warning(f"No valid market data in {filename}")
|
||||
return
|
||||
|
||||
# Generate LLM prompt
|
||||
prompt = self._create_trading_prompt(market_data)
|
||||
|
||||
# Send to LLM
|
||||
response = self._query_llm(prompt)
|
||||
if not response:
|
||||
logger.warning(f"No response from LLM for {filename}")
|
||||
return
|
||||
|
||||
# Parse response
|
||||
signal = self._parse_llm_response(response, market_data)
|
||||
if signal:
|
||||
with self.lock:
|
||||
self.latest_signals['ETH'] = signal
|
||||
self.signal_history.append(signal)
|
||||
# Keep only last 100 signals
|
||||
if len(self.signal_history) > 100:
|
||||
self.signal_history = self.signal_history[-100:]
|
||||
|
||||
logger.info(f"Generated signal: {signal.action} ({signal.confidence:.2f}) - {signal.reasoning}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error processing file {filename}: {e}")
|
||||
|
||||
def _parse_market_data(self, filepath: str) -> Optional[Dict[str, Any]]:
|
||||
"""Parse market data from text file"""
|
||||
try:
|
||||
with open(filepath, 'r', encoding='utf-8') as f:
|
||||
lines = f.readlines()
|
||||
|
||||
if len(lines) < 4: # Need header + data
|
||||
return None
|
||||
|
||||
# Find data line (skip headers)
|
||||
data_line = None
|
||||
for line in lines[3:]: # Skip the 3 header lines
|
||||
if line.strip() and not line.startswith('symbol'):
|
||||
data_line = line.strip()
|
||||
break
|
||||
|
||||
if not data_line:
|
||||
return None
|
||||
|
||||
# Parse tab-separated data
|
||||
parts = data_line.split('\t')
|
||||
if len(parts) < 25: # Need minimum data
|
||||
return None
|
||||
|
||||
# Extract structured data
|
||||
parsed_data = {
|
||||
'timestamp': parts[0],
|
||||
'eth_1s': self._extract_ohlcv(parts[1:7]),
|
||||
'eth_1m': self._extract_ohlcv(parts[7:13]),
|
||||
'eth_1h': self._extract_ohlcv(parts[13:19]),
|
||||
'eth_1d': self._extract_ohlcv(parts[19:25]),
|
||||
'btc_1s': self._extract_ohlcv(parts[25:31]) if len(parts) > 25 else None,
|
||||
'spx_1s': self._extract_ohlcv(parts[31:37]) if len(parts) > 31 else None
|
||||
}
|
||||
|
||||
return parsed_data
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing market data: {e}")
|
||||
return None
|
||||
|
||||
def _extract_ohlcv(self, data_parts: List[str]) -> Dict[str, float]:
|
||||
"""Extract OHLCV data from parts"""
|
||||
try:
|
||||
return {
|
||||
'open': float(data_parts[0]) if data_parts[0] != '0' else 0.0,
|
||||
'high': float(data_parts[1]) if data_parts[1] != '0' else 0.0,
|
||||
'low': float(data_parts[2]) if data_parts[2] != '0' else 0.0,
|
||||
'close': float(data_parts[3]) if data_parts[3] != '0' else 0.0,
|
||||
'volume': float(data_parts[4]) if data_parts[4] != '0' else 0.0,
|
||||
'timestamp': data_parts[5]
|
||||
}
|
||||
except (ValueError, IndexError):
|
||||
return {'open': 0.0, 'high': 0.0, 'low': 0.0, 'close': 0.0, 'volume': 0.0, 'timestamp': ''}
|
||||
|
||||
def _create_trading_prompt(self, market_data: Dict[str, Any]) -> str:
|
||||
"""Create trading prompt from market data"""
|
||||
prompt = f"""Market Data Analysis for ETH/USDT:
|
||||
|
||||
Timestamp: {market_data['timestamp']}
|
||||
|
||||
ETH Multi-timeframe Data:
|
||||
1s: O:{market_data['eth_1s']['open']:.2f} H:{market_data['eth_1s']['high']:.2f} L:{market_data['eth_1s']['low']:.2f} C:{market_data['eth_1s']['close']:.2f} V:{market_data['eth_1s']['volume']:.1f}
|
||||
1m: O:{market_data['eth_1m']['open']:.2f} H:{market_data['eth_1m']['high']:.2f} L:{market_data['eth_1m']['low']:.2f} C:{market_data['eth_1m']['close']:.2f} V:{market_data['eth_1m']['volume']:.1f}
|
||||
1h: O:{market_data['eth_1h']['open']:.2f} H:{market_data['eth_1h']['high']:.2f} L:{market_data['eth_1h']['low']:.2f} C:{market_data['eth_1h']['close']:.2f} V:{market_data['eth_1h']['volume']:.1f}
|
||||
1d: O:{market_data['eth_1d']['open']:.2f} H:{market_data['eth_1d']['high']:.2f} L:{market_data['eth_1d']['low']:.2f} C:{market_data['eth_1d']['close']:.2f} V:{market_data['eth_1d']['volume']:.1f}
|
||||
"""
|
||||
|
||||
if market_data.get('btc_1s'):
|
||||
prompt += f"\nBTC Reference (1s): O:{market_data['btc_1s']['open']:.2f} H:{market_data['btc_1s']['high']:.2f} L:{market_data['btc_1s']['low']:.2f} C:{market_data['btc_1s']['close']:.2f} V:{market_data['btc_1s']['volume']:.1f}"
|
||||
|
||||
if market_data.get('spx_1s'):
|
||||
prompt += f"\nSPX Reference (1s): O:{market_data['spx_1s']['open']:.2f} H:{market_data['spx_1s']['high']:.2f} L:{market_data['spx_1s']['low']:.2f} C:{market_data['spx_1s']['close']:.2f}"
|
||||
|
||||
prompt += "\n\nProvide trading recommendation based on this multi-timeframe analysis."
|
||||
return prompt
|
||||
|
||||
def _query_llm(self, prompt: str) -> Optional[str]:
|
||||
"""Send query to LLM endpoint"""
|
||||
try:
|
||||
url = f"{self.config.base_url}/v1/chat/completions"
|
||||
|
||||
headers = {
|
||||
"Content-Type": "application/json"
|
||||
}
|
||||
|
||||
if self.config.api_key:
|
||||
headers["Authorization"] = f"Bearer {self.config.api_key}"
|
||||
|
||||
payload = {
|
||||
"model": self.config.model,
|
||||
"messages": [
|
||||
{"role": "system", "content": self.system_prompt},
|
||||
{"role": "user", "content": prompt}
|
||||
],
|
||||
"temperature": self.config.temperature,
|
||||
"max_tokens": self.config.max_tokens,
|
||||
"stream": False
|
||||
}
|
||||
|
||||
response = requests.post(
|
||||
url,
|
||||
headers=headers,
|
||||
data=json.dumps(payload),
|
||||
timeout=self.config.timeout
|
||||
)
|
||||
|
||||
if response.status_code == 200:
|
||||
result = response.json()
|
||||
if 'choices' in result and len(result['choices']) > 0:
|
||||
return result['choices'][0]['message']['content']
|
||||
else:
|
||||
logger.error(f"LLM API error: {response.status_code} - {response.text}")
|
||||
|
||||
return None
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error querying LLM: {e}")
|
||||
return None
|
||||
|
||||
def _parse_llm_response(self, response: str, market_data: Dict[str, Any]) -> Optional[LLMTradeSignal]:
|
||||
"""Parse LLM response into trade signal"""
|
||||
try:
|
||||
# Try to extract JSON from response
|
||||
response = response.strip()
|
||||
if response.startswith('```json'):
|
||||
response = response[7:]
|
||||
if response.endswith('```'):
|
||||
response = response[:-3]
|
||||
|
||||
# Parse JSON
|
||||
data = json.loads(response)
|
||||
|
||||
# Validate required fields
|
||||
if 'action' not in data or 'confidence' not in data:
|
||||
logger.warning("LLM response missing required fields")
|
||||
return None
|
||||
|
||||
# Create signal
|
||||
signal = LLMTradeSignal(
|
||||
symbol='ETH/USDT',
|
||||
action=data['action'].upper(),
|
||||
confidence=float(data['confidence']),
|
||||
reasoning=data.get('reasoning', ''),
|
||||
price_target=data.get('price_target'),
|
||||
stop_loss=data.get('stop_loss'),
|
||||
timestamp=datetime.now()
|
||||
)
|
||||
|
||||
# Validate action
|
||||
if signal.action not in ['BUY', 'SELL', 'HOLD']:
|
||||
logger.warning(f"Invalid action: {signal.action}")
|
||||
return None
|
||||
|
||||
# Validate confidence
|
||||
signal.confidence = max(0.0, min(1.0, signal.confidence))
|
||||
|
||||
return signal
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error parsing LLM response: {e}")
|
||||
logger.debug(f"Response was: {response}")
|
||||
return None
|
||||
|
||||
def get_latest_signal(self, symbol: str = 'ETH') -> Optional[LLMTradeSignal]:
|
||||
"""Get latest trading signal for symbol"""
|
||||
with self.lock:
|
||||
return self.latest_signals.get(symbol)
|
||||
|
||||
def get_signal_history(self, limit: int = 10) -> List[LLMTradeSignal]:
|
||||
"""Get recent signal history"""
|
||||
with self.lock:
|
||||
return self.signal_history[-limit:] if self.signal_history else []
|
||||
|
||||
def update_config(self, config: LLMConfig):
|
||||
"""Update LLM configuration"""
|
||||
self.config = config
|
||||
logger.info(f"LLM config updated - Model: {self.config.model}, Base URL: {self.config.base_url}")
|
||||
|
||||
def get_status(self) -> Dict[str, Any]:
|
||||
"""Get LLM proxy status"""
|
||||
with self.lock:
|
||||
return {
|
||||
'is_running': self.is_running,
|
||||
'config': {
|
||||
'base_url': self.config.base_url,
|
||||
'model': self.config.model,
|
||||
'temperature': self.config.temperature
|
||||
},
|
||||
'processed_files': len(self.processed_files),
|
||||
'total_signals': len(self.signal_history),
|
||||
'latest_signals': {k: {
|
||||
'action': v.action,
|
||||
'confidence': v.confidence,
|
||||
'timestamp': v.timestamp.isoformat() if v.timestamp else None
|
||||
} for k, v in self.latest_signals.items()}
|
||||
}
|
||||
|
||||
# Convenience functions
|
||||
def create_llm_proxy(config: Optional[LLMConfig] = None, **kwargs) -> LLMProxy:
|
||||
"""Create LLM proxy instance"""
|
||||
return LLMProxy(config=config, **kwargs)
|
||||
|
||||
def create_llm_config(base_url: str = "http://localhost:1234",
|
||||
model: str = "openai/gpt-oss-20b",
|
||||
**kwargs) -> LLMConfig:
|
||||
"""Create LLM configuration"""
|
||||
return LLMConfig(base_url=base_url, model=model, **kwargs)
|
||||
@@ -31,6 +31,7 @@ import torch.optim as optim
|
||||
|
||||
# Text export integration
|
||||
from .text_export_integration import TextExportManager
|
||||
from .llm_proxy import LLMProxy, LLMConfig
|
||||
import pandas as pd
|
||||
from pathlib import Path
|
||||
|
||||
@@ -572,6 +573,7 @@ class TradingOrchestrator:
|
||||
self._initialize_transformer_model() # Initialize transformer model
|
||||
self._initialize_enhanced_training_system() # Initialize real-time training
|
||||
self._initialize_text_export_manager() # Initialize text data export
|
||||
self._initialize_llm_proxy() # Initialize LLM proxy for trading signals
|
||||
|
||||
def _normalize_model_name(self, name: str) -> str:
|
||||
"""Map various registry/UI names to canonical toggle keys."""
|
||||
@@ -7040,7 +7042,7 @@ class TradingOrchestrator:
|
||||
'main_symbol': self.symbol,
|
||||
'ref1_symbol': self.ref_symbols[0] if self.ref_symbols else 'BTC/USDT',
|
||||
'ref2_symbol': 'SPX', # Default to SPX for now
|
||||
'export_dir': 'data/text_exports'
|
||||
'export_dir': 'NN/training/samples/txt'
|
||||
}
|
||||
|
||||
self.text_export_manager.export_config.update(export_config)
|
||||
@@ -7053,6 +7055,35 @@ class TradingOrchestrator:
|
||||
logger.error(f"Error initializing text export manager: {e}")
|
||||
self.text_export_manager = None
|
||||
|
||||
def _initialize_llm_proxy(self):
|
||||
"""Initialize LLM proxy for trading signals"""
|
||||
try:
|
||||
# Get LLM configuration from config file or use defaults
|
||||
llm_config = self.config.get('llm_proxy', {})
|
||||
|
||||
llm_proxy_config = LLMConfig(
|
||||
base_url=llm_config.get('base_url', 'http://localhost:1234'),
|
||||
model=llm_config.get('model', 'openai/gpt-oss-20b'),
|
||||
temperature=llm_config.get('temperature', 0.7),
|
||||
max_tokens=llm_config.get('max_tokens', -1),
|
||||
timeout=llm_config.get('timeout', 30),
|
||||
api_key=llm_config.get('api_key')
|
||||
)
|
||||
|
||||
self.llm_proxy = LLMProxy(
|
||||
config=llm_proxy_config,
|
||||
data_dir='NN/training/samples/txt'
|
||||
)
|
||||
|
||||
logger.info("LLM proxy initialized")
|
||||
logger.info(f" - Model: {llm_proxy_config.model}")
|
||||
logger.info(f" - Base URL: {llm_proxy_config.base_url}")
|
||||
logger.info(f" - Temperature: {llm_proxy_config.temperature}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error initializing LLM proxy: {e}")
|
||||
self.llm_proxy = None
|
||||
|
||||
def start_text_export(self) -> bool:
|
||||
"""Start text data export"""
|
||||
try:
|
||||
@@ -7087,6 +7118,91 @@ class TradingOrchestrator:
|
||||
logger.error(f"Error getting text export status: {e}")
|
||||
return {'enabled': False, 'initialized': False, 'error': str(e)}
|
||||
|
||||
def start_llm_proxy(self) -> bool:
|
||||
"""Start LLM proxy for trading signals"""
|
||||
try:
|
||||
if not hasattr(self, 'llm_proxy') or not self.llm_proxy:
|
||||
logger.warning("LLM proxy not initialized")
|
||||
return False
|
||||
|
||||
self.llm_proxy.start()
|
||||
logger.info("LLM proxy started")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Error starting LLM proxy: {e}")
|
||||
return False
|
||||
|
||||
def stop_llm_proxy(self) -> bool:
|
||||
"""Stop LLM proxy"""
|
||||
try:
|
||||
if not hasattr(self, 'llm_proxy') or not self.llm_proxy:
|
||||
return True
|
||||
|
||||
self.llm_proxy.stop()
|
||||
logger.info("LLM proxy stopped")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Error stopping LLM proxy: {e}")
|
||||
return False
|
||||
|
||||
def get_llm_proxy_status(self) -> Dict[str, Any]:
|
||||
"""Get LLM proxy status"""
|
||||
try:
|
||||
if not hasattr(self, 'llm_proxy') or not self.llm_proxy:
|
||||
return {'enabled': False, 'initialized': False, 'error': 'Not initialized'}
|
||||
|
||||
return self.llm_proxy.get_status()
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting LLM proxy status: {e}")
|
||||
return {'enabled': False, 'initialized': False, 'error': str(e)}
|
||||
|
||||
def get_latest_llm_signal(self, symbol: str = 'ETH'):
|
||||
"""Get latest LLM trading signal"""
|
||||
try:
|
||||
if not hasattr(self, 'llm_proxy') or not self.llm_proxy:
|
||||
return None
|
||||
|
||||
return self.llm_proxy.get_latest_signal(symbol)
|
||||
except Exception as e:
|
||||
logger.error(f"Error getting LLM signal: {e}")
|
||||
return None
|
||||
|
||||
def update_llm_config(self, new_config: Dict[str, Any]) -> bool:
|
||||
"""Update LLM proxy configuration"""
|
||||
try:
|
||||
if not hasattr(self, 'llm_proxy') or not self.llm_proxy:
|
||||
logger.warning("LLM proxy not initialized")
|
||||
return False
|
||||
|
||||
# Create new config
|
||||
llm_proxy_config = LLMConfig(
|
||||
base_url=new_config.get('base_url', 'http://localhost:1234'),
|
||||
model=new_config.get('model', 'openai/gpt-oss-20b'),
|
||||
temperature=new_config.get('temperature', 0.7),
|
||||
max_tokens=new_config.get('max_tokens', -1),
|
||||
timeout=new_config.get('timeout', 30),
|
||||
api_key=new_config.get('api_key')
|
||||
)
|
||||
|
||||
# Stop current proxy
|
||||
was_running = self.llm_proxy.is_running
|
||||
if was_running:
|
||||
self.llm_proxy.stop()
|
||||
|
||||
# Update config
|
||||
self.llm_proxy.update_config(llm_proxy_config)
|
||||
|
||||
# Restart if it was running
|
||||
if was_running:
|
||||
self.llm_proxy.start()
|
||||
|
||||
logger.info("LLM proxy configuration updated")
|
||||
return True
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error updating LLM config: {e}")
|
||||
return False
|
||||
|
||||
def get_enhanced_training_stats(self) -> Dict[str, Any]:
|
||||
"""Get enhanced training system statistics with orchestrator integration"""
|
||||
try:
|
||||
|
||||
@@ -41,7 +41,7 @@ class TextDataExporter:
|
||||
|
||||
def __init__(self,
|
||||
data_provider=None,
|
||||
export_dir: str = "data/text_exports",
|
||||
export_dir: str = "NN/training/samples/txt",
|
||||
main_symbol: str = "ETH/USDT",
|
||||
ref1_symbol: str = "BTC/USDT",
|
||||
ref2_symbol: str = "SPX"):
|
||||
@@ -116,7 +116,7 @@ class TextDataExporter:
|
||||
# Check if we need a new file (new minute)
|
||||
if self.current_minute != current_minute_key:
|
||||
self.current_minute = current_minute_key
|
||||
self.current_filename = f"market_data_{current_minute_key}.csv"
|
||||
self.current_filename = f"market_data_{current_minute_key}.txt"
|
||||
logger.info(f"Starting new export file: {self.current_filename}")
|
||||
|
||||
# Gather data for all symbols and timeframes
|
||||
@@ -193,7 +193,7 @@ class TextDataExporter:
|
||||
return None
|
||||
|
||||
def _write_csv_file(self, export_data: List[Dict[str, Any]]):
|
||||
"""Write data to CSV file"""
|
||||
"""Write data to TXT file in tab-separated format"""
|
||||
if not export_data:
|
||||
return
|
||||
|
||||
@@ -201,25 +201,17 @@ class TextDataExporter:
|
||||
|
||||
with self.export_lock:
|
||||
try:
|
||||
with open(filepath, 'w', newline='', encoding='utf-8') as csvfile:
|
||||
# Create header based on the format specification
|
||||
fieldnames = self._create_csv_header()
|
||||
writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
|
||||
|
||||
# Write header
|
||||
writer.writeheader()
|
||||
|
||||
# Group data by symbol type for organized output
|
||||
grouped_data = self._group_data_by_symbol(export_data)
|
||||
|
||||
# Write data rows
|
||||
for row in self._format_csv_rows(grouped_data):
|
||||
writer.writerow(row)
|
||||
# Group data by symbol type for organized output
|
||||
grouped_data = self._group_data_by_symbol(export_data)
|
||||
|
||||
with open(filepath, 'w', encoding='utf-8') as txtfile:
|
||||
# Write in the format specified in readme.md sample
|
||||
self._write_tab_format(txtfile, grouped_data)
|
||||
|
||||
logger.debug(f"Exported {len(export_data)} data points to {filepath}")
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"Error writing CSV file {filepath}: {e}")
|
||||
logger.error(f"Error writing TXT file {filepath}: {e}")
|
||||
|
||||
def _create_csv_header(self) -> List[str]:
|
||||
"""Create CSV header based on specification"""
|
||||
@@ -288,6 +280,57 @@ class TextDataExporter:
|
||||
rows.append(row)
|
||||
return rows
|
||||
|
||||
def _write_tab_format(self, txtfile, grouped_data: Dict[str, Dict[str, Dict[str, Any]]]):
|
||||
"""Write data in tab-separated format like readme.md sample"""
|
||||
# Write header structure
|
||||
txtfile.write("symbol\tMAIN SYMBOL (ETH)\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\t\tREF1 (BTC)\t\t\t\t\t\tREF2 (SPX)\t\t\t\t\t\tREF3 (SOL)\n")
|
||||
txtfile.write("timeframe\t1s\t\t\t\t\t\t1m\t\t\t\t\t\t1h\t\t\t\t\t\t1d\t\t\t\t\t\t1s\t\t\t\t\t\t1s\t\t\t\t\t\t1s\n")
|
||||
txtfile.write("datapoint\tO\tH\tL\tC\tV\tTimestamp\tO\tH\tL\tC\tV\tTimestamp\tO\tH\tL\tC\tV\tTimestamp\tO\tH\tL\tC\tV\tTimestamp\tO\tH\tL\tC\tV\tTimestamp\tO\tH\tL\tC\tV\tTimestamp\tO\tH\tL\tC\tV\tTimestamp\n")
|
||||
|
||||
# Write data row
|
||||
row_parts = []
|
||||
current_time = datetime.now()
|
||||
|
||||
# Timestamp first
|
||||
row_parts.append(current_time.strftime("%Y-%m-%dT%H:%M:%SZ"))
|
||||
|
||||
# ETH data for all timeframes (1s, 1m, 1h, 1d)
|
||||
main_data = grouped_data.get('MAIN', {})
|
||||
for timeframe in ['1s', '1m', '1h', '1d']:
|
||||
data_point = main_data.get(timeframe)
|
||||
if data_point:
|
||||
row_parts.extend([
|
||||
f"{data_point['open']:.2f}",
|
||||
f"{data_point['high']:.2f}",
|
||||
f"{data_point['low']:.2f}",
|
||||
f"{data_point['close']:.2f}",
|
||||
f"{data_point['volume']:.1f}",
|
||||
data_point['timestamp'].strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
])
|
||||
else:
|
||||
row_parts.extend(["0", "0", "0", "0", "0", current_time.strftime("%Y-%m-%dT%H:%M:%SZ")])
|
||||
|
||||
# REF1 (BTC), REF2 (SPX), REF3 (SOL) - 1s timeframe only
|
||||
for ref_type in ['REF1', 'REF2']: # REF3 will be added by LLM proxy
|
||||
ref_data = grouped_data.get(ref_type, {})
|
||||
data_point = ref_data.get('1s')
|
||||
if data_point:
|
||||
row_parts.extend([
|
||||
f"{data_point['open']:.2f}",
|
||||
f"{data_point['high']:.2f}",
|
||||
f"{data_point['low']:.2f}",
|
||||
f"{data_point['close']:.2f}",
|
||||
f"{data_point['volume']:.1f}",
|
||||
data_point['timestamp'].strftime("%Y-%m-%dT%H:%M:%SZ")
|
||||
])
|
||||
else:
|
||||
row_parts.extend(["0", "0", "0", "0", "0", current_time.strftime("%Y-%m-%dT%H:%M:%SZ")])
|
||||
|
||||
# Add placeholder for REF3 (SOL) - will be filled by LLM proxy
|
||||
row_parts.extend(["0", "0", "0", "0", "0", current_time.strftime("%Y-%m-%dT%H:%M:%SZ")])
|
||||
|
||||
txtfile.write("\t".join(row_parts) + "\n")
|
||||
|
||||
def get_current_filename(self) -> Optional[str]:
|
||||
"""Get current export filename"""
|
||||
return self.current_filename
|
||||
@@ -308,7 +351,7 @@ class TextDataExporter:
|
||||
|
||||
# Add file count
|
||||
try:
|
||||
files = [f for f in os.listdir(self.export_dir) if f.endswith('.csv')]
|
||||
files = [f for f in os.listdir(self.export_dir) if f.endswith('.txt')]
|
||||
stats['total_files'] = len(files)
|
||||
except:
|
||||
stats['total_files'] = 0
|
||||
|
||||
@@ -33,7 +33,7 @@ class TextExportManager:
|
||||
'main_symbol': 'ETH/USDT',
|
||||
'ref1_symbol': 'BTC/USDT',
|
||||
'ref2_symbol': 'SPX', # Will need to be mapped to available data
|
||||
'export_dir': 'data/text_exports'
|
||||
'export_dir': 'NN/training/samples/txt'
|
||||
}
|
||||
|
||||
def initialize_exporter(self, config: Optional[Dict[str, Any]] = None):
|
||||
|
||||
Reference in New Issue
Block a user