wip misc; cleaup launch
This commit is contained in:
180
.vscode/launch.json
vendored
180
.vscode/launch.json
vendored
@@ -1,42 +1,7 @@
|
|||||||
{
|
{
|
||||||
"version": "0.2.0",
|
"version": "0.2.0",
|
||||||
"configurations": [
|
"configurations": [
|
||||||
{
|
|
||||||
"name": "📊 Enhanced Web Dashboard (Safe)",
|
|
||||||
"type": "python",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "main_clean.py",
|
|
||||||
"args": [
|
|
||||||
"--port",
|
|
||||||
"8051",
|
|
||||||
"--no-training"
|
|
||||||
],
|
|
||||||
"console": "integratedTerminal",
|
|
||||||
"justMyCode": false,
|
|
||||||
"env": {
|
|
||||||
"PYTHONUNBUFFERED": "1",
|
|
||||||
"ENABLE_REALTIME_CHARTS": "1"
|
|
||||||
},
|
|
||||||
"preLaunchTask": "Kill Stale Processes"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "📊 Enhanced Web Dashboard (Full)",
|
|
||||||
"type": "python",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "main_clean.py",
|
|
||||||
"args": [
|
|
||||||
"--port",
|
|
||||||
"8051"
|
|
||||||
],
|
|
||||||
"console": "integratedTerminal",
|
|
||||||
"justMyCode": false,
|
|
||||||
"env": {
|
|
||||||
"PYTHONUNBUFFERED": "1",
|
|
||||||
"ENABLE_REALTIME_CHARTS": "1",
|
|
||||||
"ENABLE_NN_MODELS": "1"
|
|
||||||
},
|
|
||||||
"preLaunchTask": "Kill Stale Processes"
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"name": "📊 Clean Dashboard (Legacy)",
|
"name": "📊 Clean Dashboard (Legacy)",
|
||||||
"type": "python",
|
"type": "python",
|
||||||
@@ -49,51 +14,7 @@
|
|||||||
"ENABLE_REALTIME_CHARTS": "1"
|
"ENABLE_REALTIME_CHARTS": "1"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"name": "🚀 Main System",
|
|
||||||
"type": "python",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "main.py",
|
|
||||||
"console": "integratedTerminal",
|
|
||||||
"justMyCode": false,
|
|
||||||
"env": {
|
|
||||||
"PYTHONUNBUFFERED": "1"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "🔬 System Test & Validation",
|
|
||||||
"type": "python",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "main.py",
|
|
||||||
"args": [
|
|
||||||
"--mode",
|
|
||||||
"test"
|
|
||||||
],
|
|
||||||
"console": "integratedTerminal",
|
|
||||||
"justMyCode": false,
|
|
||||||
"env": {
|
|
||||||
"PYTHONUNBUFFERED": "1",
|
|
||||||
"TEST_ALL_COMPONENTS": "1"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
|
|
||||||
{
|
|
||||||
"name": "🧪 CNN Live Training with Analysis",
|
|
||||||
"type": "python",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "training/enhanced_cnn_trainer.py",
|
|
||||||
"console": "integratedTerminal",
|
|
||||||
"justMyCode": false,
|
|
||||||
"env": {
|
|
||||||
"PYTHONUNBUFFERED": "1",
|
|
||||||
"ENABLE_BACKTESTING": "1",
|
|
||||||
"ENABLE_ANALYSIS": "1",
|
|
||||||
"ENABLE_LIVE_VALIDATION": "1",
|
|
||||||
"CUDA_VISIBLE_DEVICES": "0"
|
|
||||||
},
|
|
||||||
"preLaunchTask": "Kill Stale Processes",
|
|
||||||
"postDebugTask": "Start TensorBoard"
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"name": "🏗️ Python Debugger: Current File",
|
"name": "🏗️ Python Debugger: Current File",
|
||||||
"type": "debugpy",
|
"type": "debugpy",
|
||||||
@@ -119,38 +40,7 @@
|
|||||||
},
|
},
|
||||||
"preLaunchTask": "Kill Stale Processes"
|
"preLaunchTask": "Kill Stale Processes"
|
||||||
},
|
},
|
||||||
{
|
|
||||||
"name": "🔥 Real-time RL COB Trader (400M Parameters)",
|
|
||||||
"type": "python",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "run_realtime_rl_cob_trader.py",
|
|
||||||
"console": "integratedTerminal",
|
|
||||||
"justMyCode": false,
|
|
||||||
"env": {
|
|
||||||
"PYTHONUNBUFFERED": "1",
|
|
||||||
"CUDA_VISIBLE_DEVICES": "0",
|
|
||||||
"PYTORCH_CUDA_ALLOC_CONF": "max_split_size_mb:256",
|
|
||||||
"ENABLE_REALTIME_RL": "1"
|
|
||||||
},
|
|
||||||
"preLaunchTask": "Kill Stale Processes"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "🚀 Integrated COB Dashboard + RL Trading",
|
|
||||||
"type": "python",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "run_integrated_rl_cob_dashboard.py",
|
|
||||||
"console": "integratedTerminal",
|
|
||||||
"justMyCode": false,
|
|
||||||
"env": {
|
|
||||||
"PYTHONUNBUFFERED": "1",
|
|
||||||
"CUDA_VISIBLE_DEVICES": "0",
|
|
||||||
"PYTORCH_CUDA_ALLOC_CONF": "max_split_size_mb:256",
|
|
||||||
"ENABLE_REALTIME_RL": "1",
|
|
||||||
"COB_BTC_BUCKET_SIZE": "10",
|
|
||||||
"COB_ETH_BUCKET_SIZE": "1"
|
|
||||||
},
|
|
||||||
"preLaunchTask": "Kill Stale Processes"
|
|
||||||
},
|
|
||||||
{
|
{
|
||||||
"name": " *🧹 Clean Trading Dashboard (Universal Data Stream)",
|
"name": " *🧹 Clean Trading Dashboard (Universal Data Stream)",
|
||||||
"type": "python",
|
"type": "python",
|
||||||
@@ -191,52 +81,9 @@
|
|||||||
"order": 2
|
"order": 2
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
|
||||||
{
|
{
|
||||||
"name": "🌐 COBY Multi-Exchange Data Aggregation",
|
"name": "🔧 COBY Development Mode (Auto-reload) - main",
|
||||||
"type": "python",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "COBY/main.py",
|
|
||||||
"console": "integratedTerminal",
|
|
||||||
"justMyCode": false,
|
|
||||||
"env": {
|
|
||||||
"PYTHONUNBUFFERED": "1",
|
|
||||||
"COBY_API_HOST": "0.0.0.0",
|
|
||||||
"COBY_API_PORT": "8080",
|
|
||||||
"COBY_WEBSOCKET_PORT": "8081"
|
|
||||||
},
|
|
||||||
"preLaunchTask": "Kill Stale Processes",
|
|
||||||
"presentation": {
|
|
||||||
"hidden": false,
|
|
||||||
"group": "COBY System",
|
|
||||||
"order": 1
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "🔍 COBY Debug Mode",
|
|
||||||
"type": "python",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "COBY/main.py",
|
|
||||||
"args": [
|
|
||||||
"--debug"
|
|
||||||
],
|
|
||||||
"console": "integratedTerminal",
|
|
||||||
"justMyCode": false,
|
|
||||||
"env": {
|
|
||||||
"PYTHONUNBUFFERED": "1",
|
|
||||||
"COBY_API_HOST": "localhost",
|
|
||||||
"COBY_API_PORT": "8080",
|
|
||||||
"COBY_WEBSOCKET_PORT": "8081",
|
|
||||||
"COBY_LOG_LEVEL": "DEBUG"
|
|
||||||
},
|
|
||||||
"preLaunchTask": "Kill Stale Processes",
|
|
||||||
"presentation": {
|
|
||||||
"hidden": false,
|
|
||||||
"group": "COBY System",
|
|
||||||
"order": 2
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "🔧 COBY Development Mode (Auto-reload)",
|
|
||||||
"type": "python",
|
"type": "python",
|
||||||
"request": "launch",
|
"request": "launch",
|
||||||
"program": "COBY/main.py",
|
"program": "COBY/main.py",
|
||||||
@@ -259,23 +106,8 @@
|
|||||||
"group": "COBY System",
|
"group": "COBY System",
|
||||||
"order": 3
|
"order": 3
|
||||||
}
|
}
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "🏥 COBY Health Check",
|
|
||||||
"type": "python",
|
|
||||||
"request": "launch",
|
|
||||||
"program": "COBY/health_check.py",
|
|
||||||
"console": "integratedTerminal",
|
|
||||||
"justMyCode": false,
|
|
||||||
"env": {
|
|
||||||
"PYTHONUNBUFFERED": "1"
|
|
||||||
},
|
|
||||||
"presentation": {
|
|
||||||
"hidden": false,
|
|
||||||
"group": "COBY System",
|
|
||||||
"order": 4
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
],
|
],
|
||||||
"compounds": [
|
"compounds": [
|
||||||
|
74
core/coingecko_client.py
Normal file
74
core/coingecko_client.py
Normal file
@@ -0,0 +1,74 @@
|
|||||||
|
"""
|
||||||
|
CoinGecko client (public free endpoints) to fetch BTC/ETH prices for plotting.
|
||||||
|
|
||||||
|
Rules:
|
||||||
|
- No synthetic data. Return empty structures or None when unavailable.
|
||||||
|
- Use shared RateLimiter for polite access and retries.
|
||||||
|
- Default to public endpoints that do not require API key; if a key is provided via env, include it.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
from typing import Any, Dict, List, Optional, Tuple
|
||||||
|
|
||||||
|
from .api_rate_limiter import get_rate_limiter
|
||||||
|
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class CoinGeckoClient:
|
||||||
|
def __init__(self,
|
||||||
|
api_base_url: str = "https://api.coingecko.com/api/v3",
|
||||||
|
api_key_env: str = "COINGECKO_API_KEY",
|
||||||
|
user_agent: str = "gogo2-dashboard/1.0") -> None:
|
||||||
|
self.api_base_url = api_base_url.rstrip("/")
|
||||||
|
self.api_key = os.environ.get(api_key_env) or ""
|
||||||
|
self.user_agent = user_agent
|
||||||
|
self._rl = get_rate_limiter()
|
||||||
|
|
||||||
|
def get_simple_price(self, ids: List[str], vs_currency: str = "usd") -> Dict[str, Any]:
|
||||||
|
if not ids:
|
||||||
|
return {}
|
||||||
|
url = f"{self.api_base_url}/simple/price"
|
||||||
|
params = {
|
||||||
|
"ids": ",".join(ids),
|
||||||
|
"vs_currencies": vs_currency,
|
||||||
|
}
|
||||||
|
headers = {"User-Agent": self.user_agent}
|
||||||
|
# Optional key
|
||||||
|
if self.api_key:
|
||||||
|
params["x_cg_pro_api_key"] = self.api_key
|
||||||
|
resp = self._rl.make_request("coingecko_api", url, method="GET", params=params, headers=headers)
|
||||||
|
if resp is None:
|
||||||
|
return {}
|
||||||
|
try:
|
||||||
|
return resp.json() # type: ignore
|
||||||
|
except Exception as ex:
|
||||||
|
logger.error("CoinGecko simple price JSON error: %s", ex)
|
||||||
|
return {}
|
||||||
|
|
||||||
|
def get_market_chart(self, coin_id: str, vs_currency: str = "usd", days: int = 5, interval: str = "hourly") -> Dict[str, Any]:
|
||||||
|
if not coin_id:
|
||||||
|
return {}
|
||||||
|
url = f"{self.api_base_url}/coins/{coin_id}/market_chart"
|
||||||
|
params = {
|
||||||
|
"vs_currency": vs_currency,
|
||||||
|
"days": str(max(1, int(days))),
|
||||||
|
"interval": interval,
|
||||||
|
}
|
||||||
|
headers = {"User-Agent": self.user_agent}
|
||||||
|
if self.api_key:
|
||||||
|
params["x_cg_pro_api_key"] = self.api_key
|
||||||
|
resp = self._rl.make_request("coingecko_api", url, method="GET", params=params, headers=headers)
|
||||||
|
if resp is None:
|
||||||
|
return {}
|
||||||
|
try:
|
||||||
|
return resp.json() # type: ignore
|
||||||
|
except Exception as ex:
|
||||||
|
logger.error("CoinGecko market_chart JSON error: %s", ex)
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
@@ -1183,16 +1183,21 @@ class DataProvider:
|
|||||||
price = tick.get('price')
|
price = tick.get('price')
|
||||||
if not price:
|
if not price:
|
||||||
price = tick.get('mid_price') or stats.get('mid_price', 0)
|
price = tick.get('mid_price') or stats.get('mid_price', 0)
|
||||||
# Derive a volume proxy if not provided (use bid+ask volume from stats)
|
# Strict: if still falsy or non-finite, skip
|
||||||
|
try:
|
||||||
|
price = float(price)
|
||||||
|
except Exception:
|
||||||
|
price = 0.0
|
||||||
|
# Volume: do not synthesize from other stats; use provided value or 0.0
|
||||||
volume = tick.get('volume')
|
volume = tick.get('volume')
|
||||||
if volume is None:
|
try:
|
||||||
bid_vol = stats.get('bid_volume', 0) or 0
|
volume = float(volume) if volume is not None else 0.0
|
||||||
ask_vol = stats.get('ask_volume', 0) or 0
|
except Exception:
|
||||||
volume = float(bid_vol) + float(ask_vol)
|
volume = 0.0
|
||||||
# Do not create synthetic volume; keep zero if not available
|
|
||||||
else:
|
else:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
# Normalize timestamp; support seconds or milliseconds since epoch and tz-aware datetimes
|
||||||
if not timestamp or not price or price <= 0:
|
if not timestamp or not price or price <= 0:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
@@ -1200,7 +1205,16 @@ class DataProvider:
|
|||||||
if isinstance(timestamp, (int, float)):
|
if isinstance(timestamp, (int, float)):
|
||||||
import pytz
|
import pytz
|
||||||
utc = pytz.UTC
|
utc = pytz.UTC
|
||||||
tick_time = datetime.fromtimestamp(timestamp, tz=utc)
|
# Handle ms epoch inputs by thresholding reasonable ranges
|
||||||
|
try:
|
||||||
|
# If timestamp looks like milliseconds (e.g., > 10^12), convert to seconds
|
||||||
|
if timestamp > 1e12:
|
||||||
|
tick_time = datetime.fromtimestamp(timestamp / 1000.0, tz=utc)
|
||||||
|
else:
|
||||||
|
tick_time = datetime.fromtimestamp(timestamp, tz=utc)
|
||||||
|
except Exception:
|
||||||
|
# Skip bad timestamps cleanly on Windows
|
||||||
|
continue
|
||||||
# Keep in UTC to match COB WebSocket data
|
# Keep in UTC to match COB WebSocket data
|
||||||
elif isinstance(timestamp, datetime):
|
elif isinstance(timestamp, datetime):
|
||||||
import pytz
|
import pytz
|
||||||
@@ -1208,7 +1222,14 @@ class DataProvider:
|
|||||||
tick_time = timestamp
|
tick_time = timestamp
|
||||||
# If no timezone info, assume UTC and keep in UTC
|
# If no timezone info, assume UTC and keep in UTC
|
||||||
if tick_time.tzinfo is None:
|
if tick_time.tzinfo is None:
|
||||||
tick_time = utc.localize(tick_time)
|
try:
|
||||||
|
tick_time = utc.localize(tick_time)
|
||||||
|
except Exception:
|
||||||
|
# Fallback: coerce via fromtimestamp using naive seconds
|
||||||
|
try:
|
||||||
|
tick_time = datetime.fromtimestamp(tick_time.timestamp(), tz=utc)
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
# Keep in UTC (no timezone conversion)
|
# Keep in UTC (no timezone conversion)
|
||||||
else:
|
else:
|
||||||
continue
|
continue
|
||||||
@@ -1265,25 +1286,26 @@ class DataProvider:
|
|||||||
|
|
||||||
# logger.info(f"Generated {len(df)} 1s candles from {len(recent_ticks)} ticks for {symbol}")
|
# logger.info(f"Generated {len(df)} 1s candles from {len(recent_ticks)} ticks for {symbol}")
|
||||||
return df
|
return df
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
# Handle Windows-specific invalid argument (e.g., bad timestamps) gracefully
|
# Handle invalid argument or bad timestamps gracefully (Windows-safe)
|
||||||
try:
|
try:
|
||||||
import errno
|
import errno
|
||||||
if hasattr(e, 'errno') and e.errno == errno.EINVAL:
|
if hasattr(e, 'errno') and e.errno == errno.EINVAL:
|
||||||
logger.warning(f"Invalid argument while generating 1s candles for {symbol}; trimming tick buffer and falling back")
|
logger.warning(f"Invalid argument while generating 1s candles for {symbol}; trimming tick buffer and falling back")
|
||||||
try:
|
|
||||||
if hasattr(self, 'cob_raw_ticks') and symbol in getattr(self, 'cob_raw_ticks', {}):
|
|
||||||
buf = self.cob_raw_ticks[symbol]
|
|
||||||
drop = max(1, len(buf)//2)
|
|
||||||
for _ in range(drop):
|
|
||||||
buf.popleft()
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
else:
|
else:
|
||||||
logger.error(f"Error generating 1s candles from ticks for {symbol}: {e}")
|
logger.error(f"Error generating 1s candles from ticks for {symbol}: {e}")
|
||||||
except Exception:
|
except Exception:
|
||||||
logger.error(f"Error generating 1s candles from ticks for {symbol}: {e}")
|
logger.error(f"Error generating 1s candles from ticks for {symbol}: {e}")
|
||||||
|
# Always trim a small portion of tick buffer to recover from corrupt front entries
|
||||||
|
try:
|
||||||
|
if hasattr(self, 'cob_raw_ticks') and symbol in getattr(self, 'cob_raw_ticks', {}):
|
||||||
|
buf = self.cob_raw_ticks[symbol]
|
||||||
|
drop = max(1, min(50, len(buf)//10)) # drop up to 10% or 50 entries
|
||||||
|
for _ in range(drop):
|
||||||
|
buf.popleft()
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def _fetch_from_binance(self, symbol: str, timeframe: str, limit: int) -> Optional[pd.DataFrame]:
|
def _fetch_from_binance(self, symbol: str, timeframe: str, limit: int) -> Optional[pd.DataFrame]:
|
||||||
|
253
core/polymarket_client.py
Normal file
253
core/polymarket_client.py
Normal file
@@ -0,0 +1,253 @@
|
|||||||
|
"""
|
||||||
|
Polymarket client for discovering relevant BTC/ETH price markets and fetching live data.
|
||||||
|
|
||||||
|
Notes:
|
||||||
|
- Uses public Gamma API for market discovery. Endpoints can change; keep URLs configurable.
|
||||||
|
- Avoids any synthetic data. Returns empty lists when nothing is found.
|
||||||
|
- Windows-safe ASCII logging only.
|
||||||
|
|
||||||
|
Responsibilities:
|
||||||
|
- Search active markets relevant to BTC and ETH price over the next N days
|
||||||
|
- Extract scalar market metadata when available (lower/upper bounds) to derive implied price from share price
|
||||||
|
- Optionally fetch order book or last prices for outcomes using CLOB REST if available
|
||||||
|
|
||||||
|
This module focuses on read-only public data. Trading functionality is out of scope.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import time
|
||||||
|
import threading
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from typing import Any, Dict, List, Optional, Tuple
|
||||||
|
|
||||||
|
from .api_rate_limiter import get_rate_limiter
|
||||||
|
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class ScalarMarketInfo:
|
||||||
|
market_id: str
|
||||||
|
title: str
|
||||||
|
end_date: Optional[datetime]
|
||||||
|
lower_bound: Optional[float]
|
||||||
|
upper_bound: Optional[float]
|
||||||
|
last_price: Optional[float]
|
||||||
|
slug: Optional[str]
|
||||||
|
url: Optional[str]
|
||||||
|
asset: str # "BTC" or "ETH" when detected, else ""
|
||||||
|
|
||||||
|
|
||||||
|
class PolymarketClient:
|
||||||
|
"""Simple Polymarket data client using public HTTP endpoints.
|
||||||
|
|
||||||
|
The exact endpoints can change. By default, we use Gamma API for discovery.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self,
|
||||||
|
gamma_base_url: str = "https://gamma-api.polymarket.com",
|
||||||
|
clob_base_url: str = "https://clob.polymarket.com",
|
||||||
|
user_agent: str = "gogo2-dashboard/1.0",
|
||||||
|
) -> None:
|
||||||
|
self.gamma_base_url = gamma_base_url.rstrip("/")
|
||||||
|
self.clob_base_url = clob_base_url.rstrip("/")
|
||||||
|
self.user_agent = user_agent
|
||||||
|
self._rl = get_rate_limiter()
|
||||||
|
|
||||||
|
# In-memory cache; no synthetic values, just last successful responses
|
||||||
|
self._last_markets: List[Dict[str, Any]] = []
|
||||||
|
self._last_scalar_infos: List[ScalarMarketInfo] = []
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# Public API
|
||||||
|
# ----------------------------
|
||||||
|
def discover_btc_eth_scalar_markets(self, days_ahead: int = 5) -> List[ScalarMarketInfo]:
|
||||||
|
"""Discover scalar markets about BTC/ETH price ending within days_ahead.
|
||||||
|
|
||||||
|
Strategy:
|
||||||
|
- Query markets with a search filter for keywords (BTC/Bitcoin/ETH/Ethereum)
|
||||||
|
- Keep markets with endDate within now+days_ahead
|
||||||
|
- Attempt to parse scalar bounds and last price
|
||||||
|
Return empty list when nothing suitable is found.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
raw_markets = self._fetch_markets()
|
||||||
|
cutoff = datetime.now(timezone.utc) + timedelta(days=max(0, int(days_ahead)))
|
||||||
|
|
||||||
|
scalar_infos: List[ScalarMarketInfo] = []
|
||||||
|
for m in raw_markets:
|
||||||
|
try:
|
||||||
|
title = str(m.get("title") or m.get("question") or "")
|
||||||
|
if not title:
|
||||||
|
continue
|
||||||
|
title_l = title.lower()
|
||||||
|
asset = ""
|
||||||
|
if "btc" in title_l or "bitcoin" in title_l:
|
||||||
|
asset = "BTC"
|
||||||
|
elif "eth" in title_l or "ethereum" in title_l:
|
||||||
|
asset = "ETH"
|
||||||
|
else:
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Parse end date
|
||||||
|
end_dt = self._parse_datetime(m.get("endDate") or m.get("closeDate"))
|
||||||
|
if end_dt and end_dt > cutoff:
|
||||||
|
# Only next N days
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Heuristic: detect scalar markets and bounds
|
||||||
|
lower_bound, upper_bound = self._extract_bounds(m)
|
||||||
|
last_price = self._extract_last_price(m)
|
||||||
|
|
||||||
|
# Only accept if appears scalar (has bounds)
|
||||||
|
if lower_bound is None or upper_bound is None:
|
||||||
|
continue
|
||||||
|
|
||||||
|
scalar_infos.append(ScalarMarketInfo(
|
||||||
|
market_id=str(m.get("id") or m.get("_id") or ""),
|
||||||
|
title=title,
|
||||||
|
end_date=end_dt,
|
||||||
|
lower_bound=lower_bound,
|
||||||
|
upper_bound=upper_bound,
|
||||||
|
last_price=last_price,
|
||||||
|
slug=m.get("slug"),
|
||||||
|
url=self._compose_market_url(m),
|
||||||
|
asset=asset,
|
||||||
|
))
|
||||||
|
except Exception as inner_ex:
|
||||||
|
logger.debug("Polymarket market parse error: %s", inner_ex)
|
||||||
|
|
||||||
|
with self._lock:
|
||||||
|
self._last_scalar_infos = scalar_infos
|
||||||
|
return scalar_infos
|
||||||
|
except Exception as ex:
|
||||||
|
logger.error("Polymarket discovery error: %s", ex)
|
||||||
|
return []
|
||||||
|
|
||||||
|
def get_cached_scalar_markets(self) -> List[ScalarMarketInfo]:
|
||||||
|
with self._lock:
|
||||||
|
return list(self._last_scalar_infos)
|
||||||
|
|
||||||
|
def derive_implied_price(self, market: ScalarMarketInfo) -> Optional[float]:
|
||||||
|
"""For scalar markets with last_price in [0,1] and known bounds, derive implied USD price.
|
||||||
|
|
||||||
|
implied = lower + (upper - lower) * last_price
|
||||||
|
Returns None if data insufficient.
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
if market is None:
|
||||||
|
return None
|
||||||
|
if market.last_price is None:
|
||||||
|
return None
|
||||||
|
if market.lower_bound is None or market.upper_bound is None:
|
||||||
|
return None
|
||||||
|
p = float(market.last_price)
|
||||||
|
lb = float(market.lower_bound)
|
||||||
|
ub = float(market.upper_bound)
|
||||||
|
if ub <= lb:
|
||||||
|
return None
|
||||||
|
if p < 0 or p > 1:
|
||||||
|
# Some APIs might provide unscaled price; ignore if out of [0,1]
|
||||||
|
return None
|
||||||
|
return lb + (ub - lb) * p
|
||||||
|
except Exception as ex:
|
||||||
|
logger.debug("Implied price calc error: %s", ex)
|
||||||
|
return None
|
||||||
|
|
||||||
|
# ----------------------------
|
||||||
|
# Internal helpers
|
||||||
|
# ----------------------------
|
||||||
|
def _fetch_markets(self) -> List[Dict[str, Any]]:
|
||||||
|
"""Fetch active markets from Gamma API.
|
||||||
|
|
||||||
|
Uses conservative params to get recent/active markets and includes descriptions.
|
||||||
|
Returns empty list if any error occurs.
|
||||||
|
"""
|
||||||
|
url = f"{self.gamma_base_url}/markets"
|
||||||
|
params = {
|
||||||
|
"limit": 200,
|
||||||
|
"active": "true",
|
||||||
|
"withDescription": "true",
|
||||||
|
# Some gamma deployments support search param; we do broader fetch then filter locally
|
||||||
|
}
|
||||||
|
headers = {"User-Agent": self.user_agent}
|
||||||
|
resp = self._rl.make_request("polymarket_gamma", url, method="GET", params=params, headers=headers)
|
||||||
|
if resp is None:
|
||||||
|
return []
|
||||||
|
if resp.status_code != 200:
|
||||||
|
logger.warning("Polymarket markets status: %s", resp.status_code)
|
||||||
|
return []
|
||||||
|
try:
|
||||||
|
data = resp.json() # type: ignore
|
||||||
|
except Exception as ex:
|
||||||
|
logger.error("Polymarket markets JSON error: %s", ex)
|
||||||
|
return []
|
||||||
|
|
||||||
|
# Gamma may return object with `data` or direct list
|
||||||
|
markets: List[Dict[str, Any]]
|
||||||
|
if isinstance(data, dict) and isinstance(data.get("data"), list):
|
||||||
|
markets = data.get("data", [])
|
||||||
|
elif isinstance(data, list):
|
||||||
|
markets = data
|
||||||
|
else:
|
||||||
|
markets = []
|
||||||
|
|
||||||
|
with self._lock:
|
||||||
|
self._last_markets = markets
|
||||||
|
return markets
|
||||||
|
|
||||||
|
def _parse_datetime(self, value: Any) -> Optional[datetime]:
|
||||||
|
if not value:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
# Common ISO-8601 variants
|
||||||
|
s = str(value)
|
||||||
|
# Ensure Z -> +00:00 for Python <3.11 compatibility
|
||||||
|
if s.endswith("Z"):
|
||||||
|
s = s.replace("Z", "+00:00")
|
||||||
|
return datetime.fromisoformat(s)
|
||||||
|
except Exception:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _extract_bounds(self, market: Dict[str, Any]) -> Tuple[Optional[float], Optional[float]]:
|
||||||
|
"""Attempt to extract scalar bounds from various field names used in Polymarket responses."""
|
||||||
|
lb = market.get("lowerBound") or market.get("min") or market.get("lower")
|
||||||
|
ub = market.get("upperBound") or market.get("max") or market.get("upper")
|
||||||
|
try:
|
||||||
|
return (float(lb) if lb is not None else None, float(ub) if ub is not None else None)
|
||||||
|
except Exception:
|
||||||
|
return (None, None)
|
||||||
|
|
||||||
|
def _extract_last_price(self, market: Dict[str, Any]) -> Optional[float]:
|
||||||
|
"""Attempt to extract last traded share price in [0,1] for scalar markets."""
|
||||||
|
# Some APIs expose price under `price`, some under `outcomePrices` or `lastPrice`
|
||||||
|
price_candidates: List[Any] = []
|
||||||
|
price_candidates.append(market.get("lastPrice"))
|
||||||
|
price_candidates.append(market.get("price"))
|
||||||
|
# If outcomePrices is present and scalar, the first could be used as indicator
|
||||||
|
op = market.get("outcomePrices")
|
||||||
|
if isinstance(op, list) and op:
|
||||||
|
price_candidates.append(op[0])
|
||||||
|
for val in price_candidates:
|
||||||
|
try:
|
||||||
|
if val is None:
|
||||||
|
continue
|
||||||
|
f = float(val)
|
||||||
|
if 0.0 <= f <= 1.0:
|
||||||
|
return f
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
return None
|
||||||
|
|
||||||
|
def _compose_market_url(self, market: Dict[str, Any]) -> Optional[str]:
|
||||||
|
slug = market.get("slug")
|
||||||
|
if slug:
|
||||||
|
return f"https://polymarket.com/event/{slug}"
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
@@ -15,4 +15,5 @@ matplotlib>=3.7.0
|
|||||||
seaborn>=0.12.0
|
seaborn>=0.12.0
|
||||||
asyncio-compat>=0.1.2
|
asyncio-compat>=0.1.2
|
||||||
wandb>=0.16.0
|
wandb>=0.16.0
|
||||||
pybit>=5.11.0
|
pybit>=5.11.0
|
||||||
|
requests>=2.31.0
|
@@ -99,6 +99,8 @@ from NN.models.standardized_cnn import StandardizedCNN
|
|||||||
# Import layout and component managers
|
# Import layout and component managers
|
||||||
from web.layout_manager import DashboardLayoutManager
|
from web.layout_manager import DashboardLayoutManager
|
||||||
from web.component_manager import DashboardComponentManager
|
from web.component_manager import DashboardComponentManager
|
||||||
|
from core.polymarket_client import PolymarketClient
|
||||||
|
from core.coingecko_client import CoinGeckoClient
|
||||||
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
@@ -405,8 +407,116 @@ class CleanTradingDashboard:
|
|||||||
threading.Thread(target=self._delayed_training_check, daemon=True).start()
|
threading.Thread(target=self._delayed_training_check, daemon=True).start()
|
||||||
|
|
||||||
logger.debug("Clean Trading Dashboard initialized with HIGH-FREQUENCY COB integration and signal generation")
|
logger.debug("Clean Trading Dashboard initialized with HIGH-FREQUENCY COB integration and signal generation")
|
||||||
logger.info("🌙 Overnight Training Coordinator ready - call start_overnight_training() to begin")
|
logger.info("Overnight Training Coordinator ready - call start_overnight_training() to begin")
|
||||||
logger.info("✅ Universal model toggle system initialized - supports dynamic model registration")
|
logger.info("Universal model toggle system initialized - supports dynamic model registration")
|
||||||
|
|
||||||
|
# Initialize Polymarket/CoinGecko clients
|
||||||
|
try:
|
||||||
|
self.polymarket_client = PolymarketClient()
|
||||||
|
self.coingecko_client = CoinGeckoClient()
|
||||||
|
except Exception as init_ex:
|
||||||
|
logger.error("Failed to initialize external data clients: %s", init_ex)
|
||||||
|
|
||||||
|
def _build_polymarket_vs_coingecko_figure(self):
|
||||||
|
"""Build a figure comparing Polymarket scalar implied prices vs CoinGecko real prices.
|
||||||
|
|
||||||
|
- Queries Polymarket for BTC/ETH scalar markets ending in next 5 days
|
||||||
|
- Derives implied price when possible
|
||||||
|
- Fetches CoinGecko last 5 days hourly price series
|
||||||
|
- Returns a Plotly figure with two subplots (BTC and ETH)
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
# Discover markets
|
||||||
|
markets = []
|
||||||
|
if hasattr(self, 'polymarket_client') and self.polymarket_client:
|
||||||
|
markets = self.polymarket_client.discover_btc_eth_scalar_markets(days_ahead=5)
|
||||||
|
|
||||||
|
# Partition by asset
|
||||||
|
btc_markets = [m for m in markets if m.asset == 'BTC']
|
||||||
|
eth_markets = [m for m in markets if m.asset == 'ETH']
|
||||||
|
|
||||||
|
# Compute implied prices (pick most recent by end_date if multiple)
|
||||||
|
def pick_latest_implied(mkts):
|
||||||
|
implied = None
|
||||||
|
chosen = None
|
||||||
|
if not mkts:
|
||||||
|
return (None, None)
|
||||||
|
# Sort by end_date ASC and take last with valid implied
|
||||||
|
mkts_sorted = sorted(mkts, key=lambda x: (x.end_date is None, x.end_date))
|
||||||
|
for m in mkts_sorted:
|
||||||
|
ip = self.polymarket_client.derive_implied_price(m) if self.polymarket_client else None
|
||||||
|
if ip is not None:
|
||||||
|
implied = ip
|
||||||
|
chosen = m
|
||||||
|
return (implied, chosen)
|
||||||
|
|
||||||
|
btc_implied, btc_market = pick_latest_implied(btc_markets)
|
||||||
|
eth_implied, eth_market = pick_latest_implied(eth_markets)
|
||||||
|
|
||||||
|
# Fetch CoinGecko market charts
|
||||||
|
cg_btc = {}
|
||||||
|
cg_eth = {}
|
||||||
|
if hasattr(self, 'coingecko_client') and self.coingecko_client:
|
||||||
|
cg_btc = self.coingecko_client.get_market_chart('bitcoin', days=5, interval='hourly') or {}
|
||||||
|
cg_eth = self.coingecko_client.get_market_chart('ethereum', days=5, interval='hourly') or {}
|
||||||
|
|
||||||
|
# Extract prices arrays: each entry [timestamp_ms, price]
|
||||||
|
def extract_series(obj):
|
||||||
|
arr = obj.get('prices') if isinstance(obj, dict) else None
|
||||||
|
if not isinstance(arr, list):
|
||||||
|
return []
|
||||||
|
result = []
|
||||||
|
for it in arr:
|
||||||
|
if isinstance(it, list) and len(it) >= 2:
|
||||||
|
ts = it[0]
|
||||||
|
val = it[1]
|
||||||
|
try:
|
||||||
|
result.append((ts, float(val)))
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
return result
|
||||||
|
|
||||||
|
btc_series = extract_series(cg_btc)
|
||||||
|
eth_series = extract_series(cg_eth)
|
||||||
|
|
||||||
|
fig = make_subplots(rows=1, cols=2, subplot_titles=("BTC", "ETH"))
|
||||||
|
|
||||||
|
# Plot CoinGecko series
|
||||||
|
if btc_series:
|
||||||
|
fig.add_trace(
|
||||||
|
go.Scatter(x=[pd.to_datetime(ts, unit='ms') for ts, _ in btc_series],
|
||||||
|
y=[v for _, v in btc_series],
|
||||||
|
name='BTC Price (CG)',
|
||||||
|
mode='lines',
|
||||||
|
line=dict(color='royalblue')), row=1, col=1)
|
||||||
|
if eth_series:
|
||||||
|
fig.add_trace(
|
||||||
|
go.Scatter(x=[pd.to_datetime(ts, unit='ms') for ts, _ in eth_series],
|
||||||
|
y=[v for _, v in eth_series],
|
||||||
|
name='ETH Price (CG)',
|
||||||
|
mode='lines',
|
||||||
|
line=dict(color='seagreen')), row=1, col=2)
|
||||||
|
|
||||||
|
# Overlay Polymarket implied point (as marker at end_date)
|
||||||
|
def add_implied_point(asset_name, implied, market, col_idx):
|
||||||
|
if implied is None or market is None:
|
||||||
|
return
|
||||||
|
dt = market.end_date
|
||||||
|
x_val = pd.to_datetime(dt) if dt else (pd.to_datetime('now'))
|
||||||
|
label = f"{asset_name} implied: {implied:.2f}"
|
||||||
|
fig.add_trace(
|
||||||
|
go.Scatter(x=[x_val], y=[implied], name=label,
|
||||||
|
mode='markers+text', text=["PM"], textposition='top center',
|
||||||
|
marker=dict(color='orange', size=8)), row=1, col=col_idx)
|
||||||
|
|
||||||
|
add_implied_point('BTC', btc_implied, btc_market, 1)
|
||||||
|
add_implied_point('ETH', eth_implied, eth_market, 2)
|
||||||
|
|
||||||
|
fig.update_layout(margin=dict(l=20, r=20, t=30, b=20), legend=dict(orientation='h'))
|
||||||
|
return fig
|
||||||
|
except Exception as ex:
|
||||||
|
logger.error("Polymarket vs CG figure error: %s", ex)
|
||||||
|
return go.Figure().add_annotation(text="Error building figure", x=0.5, y=0.5, showarrow=False)
|
||||||
|
|
||||||
def _on_cob_data_update(self, symbol: str, cob_data: dict):
|
def _on_cob_data_update(self, symbol: str, cob_data: dict):
|
||||||
"""Handle COB data updates from data provider"""
|
"""Handle COB data updates from data provider"""
|
||||||
@@ -1303,6 +1413,21 @@ class CleanTradingDashboard:
|
|||||||
xref="paper", yref="paper",
|
xref="paper", yref="paper",
|
||||||
x=0.5, y=0.5, showarrow=False)
|
x=0.5, y=0.5, showarrow=False)
|
||||||
|
|
||||||
|
# NOTE: Removed duplicate callback registration for 'polymarket-eth-btc-chart'
|
||||||
|
|
||||||
|
# Polymarket vs CoinGecko panel (slow interval)
|
||||||
|
@self.app.callback(
|
||||||
|
Output('polymarket-eth-btc-chart', 'figure'),
|
||||||
|
[Input('slow-interval-component', 'n_intervals')]
|
||||||
|
)
|
||||||
|
def update_polymarket_vs_prices(n):
|
||||||
|
try:
|
||||||
|
return self._build_polymarket_vs_coingecko_figure()
|
||||||
|
except Exception as ex:
|
||||||
|
logger.error("Polymarket panel error: %s", ex)
|
||||||
|
return go.Figure().add_annotation(text="Polymarket panel error", x=0.5, y=0.5, showarrow=False)
|
||||||
|
|
||||||
|
|
||||||
# Display state label for pivots toggle
|
# Display state label for pivots toggle
|
||||||
@self.app.callback(
|
@self.app.callback(
|
||||||
Output('pivots-display', 'children'),
|
Output('pivots-display', 'children'),
|
||||||
|
@@ -371,6 +371,14 @@ class DashboardLayoutManager:
|
|||||||
|
|
||||||
html.Div([
|
html.Div([
|
||||||
dcc.Graph(id="price-chart", style={"height": "500px"})
|
dcc.Graph(id="price-chart", style={"height": "500px"})
|
||||||
|
]),
|
||||||
|
html.Hr(className="my-2"),
|
||||||
|
html.Div([
|
||||||
|
html.H6([
|
||||||
|
html.I(className="fas fa-chart-line me-2"),
|
||||||
|
"Polymarket vs CoinGecko (BTC/ETH, next 5 days)"
|
||||||
|
], className="card-title mb-2"),
|
||||||
|
dcc.Graph(id="polymarket-eth-btc-chart", style={"height": "350px"})
|
||||||
])
|
])
|
||||||
], className="card-body p-2")
|
], className="card-body p-2")
|
||||||
], className="card")
|
], className="card")
|
||||||
|
Reference in New Issue
Block a user