diff --git a/.vscode/launch.json b/.vscode/launch.json index bc2e2ca..bb83115 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -16,7 +16,8 @@ "PYTHONUNBUFFERED": "1", "ENABLE_REALTIME_CHARTS": "1", "ENABLE_NN_MODELS": "1", - "HSA_OVERRIDE_GFX_VERSION": "11.0.0" + "HSA_OVERRIDE_GFX_VERSION": "11.0.0", + "TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL": "1" }, "preLaunchTask": "Kill Stale Processes" }, @@ -37,7 +38,8 @@ "justMyCode": false, "env": { "PYTHONUNBUFFERED": "1", - "HSA_OVERRIDE_GFX_VERSION": "11.0.0" + "HSA_OVERRIDE_GFX_VERSION": "11.0.0", + "TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL": "1" } }, { @@ -58,7 +60,8 @@ "env": { "PYTHONUNBUFFERED": "1", "CUDA_VISIBLE_DEVICES": "0", - "HSA_OVERRIDE_GFX_VERSION": "11.0.0" + "HSA_OVERRIDE_GFX_VERSION": "11.0.0", + "TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL": "1" } }, { @@ -80,7 +83,8 @@ "justMyCode": false, "env": { "PYTHONUNBUFFERED": "1", - "HSA_OVERRIDE_GFX_VERSION": "11.0.0" + "HSA_OVERRIDE_GFX_VERSION": "11.0.0", + "TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL": "1" } }, { @@ -92,7 +96,8 @@ "justMyCode": false, "env": { "PYTHONUNBUFFERED": "1", - "HSA_OVERRIDE_GFX_VERSION": "11.0.0" + "HSA_OVERRIDE_GFX_VERSION": "11.0.0", + "TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL": "1" } }, { @@ -106,7 +111,8 @@ "PYTHONUNBUFFERED": "1", "FLASK_ENV": "development", "FLASK_DEBUG": "1", - "HSA_OVERRIDE_GFX_VERSION": "11.0.0" + "HSA_OVERRIDE_GFX_VERSION": "11.0.0", + "TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL": "1" }, "cwd": "${workspaceFolder}", "preLaunchTask": "Kill Stale Processes" @@ -122,7 +128,8 @@ "PYTHONUNBUFFERED": "1", "COB_BTC_BUCKET_SIZE": "10", "COB_ETH_BUCKET_SIZE": "1", - "HSA_OVERRIDE_GFX_VERSION": "11.0.0" + "HSA_OVERRIDE_GFX_VERSION": "11.0.0", + "TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL": "1" }, "preLaunchTask": "Kill Stale Processes" }, @@ -138,7 +145,8 @@ "CUDA_VISIBLE_DEVICES": "0", "PYTORCH_CUDA_ALLOC_CONF": "max_split_size_mb:256", "ENABLE_REALTIME_RL": "1", - "HSA_OVERRIDE_GFX_VERSION": "11.0.0" + "HSA_OVERRIDE_GFX_VERSION": "11.0.0", + "TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL": "1" }, "preLaunchTask": "Kill Stale Processes" }, @@ -156,7 +164,8 @@ "ENABLE_REALTIME_RL": "1", "COB_BTC_BUCKET_SIZE": "10", "COB_ETH_BUCKET_SIZE": "1", - "HSA_OVERRIDE_GFX_VERSION": "11.0.0" + "HSA_OVERRIDE_GFX_VERSION": "11.0.0", + "TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL": "1" }, "preLaunchTask": "Kill Stale Processes" }, @@ -169,7 +178,8 @@ "justMyCode": false, "env": { "PYTHONUNBUFFERED": "1", - "HSA_OVERRIDE_GFX_VERSION": "11.0.0" + "HSA_OVERRIDE_GFX_VERSION": "11.0.0", + "TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL": "1" } }, { @@ -181,7 +191,8 @@ "justMyCode": false, "env": { "PYTHONUNBUFFERED": "1", - "HSA_OVERRIDE_GFX_VERSION": "11.0.0" + "HSA_OVERRIDE_GFX_VERSION": "11.0.0", + "TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL": "1" } }, @@ -202,7 +213,8 @@ "COBY_API_PORT": "8080", "COBY_WEBSOCKET_PORT": "8081", "COBY_LOG_LEVEL": "DEBUG", - "HSA_OVERRIDE_GFX_VERSION": "11.0.0" + "HSA_OVERRIDE_GFX_VERSION": "11.0.0", + "TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL": "1" }, "preLaunchTask": "Kill Stale Processes", "presentation": { diff --git a/AMD_GPU_FIX.md b/AMD_GPU_FIX.md index ab97bd6..72c9c49 100644 --- a/AMD_GPU_FIX.md +++ b/AMD_GPU_FIX.md @@ -84,7 +84,8 @@ The environment variable has been automatically added to your venv activation sc ### What was done: 1. Added `export HSA_OVERRIDE_GFX_VERSION=11.0.0` to `venv/bin/activate` 2. This allows gfx1151 to use gfx1100 libraries (fully compatible) -3. All PyTorch operations now work on GPU +3. Added `export TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1` for Flash Efficient attention +4. All PyTorch operations now work on GPU with experimental optimizations ### To apply: ```bash diff --git a/ANNOTATE/core/real_training_adapter.py b/ANNOTATE/core/real_training_adapter.py index 3449c8d..b42f5e7 100644 --- a/ANNOTATE/core/real_training_adapter.py +++ b/ANNOTATE/core/real_training_adapter.py @@ -1615,41 +1615,68 @@ class RealTrainingAdapter: # FIXED: Ensure shape is [1, 1] not [1] to match BCELoss requirements trade_success = torch.tensor([[1.0 if profit_loss_pct > 0 else 0.0]], dtype=torch.float32) # [1, 1] - # REAL TREND CALCULATION from actual price data (NO MORE SYNTHETIC DATA!) - # Use last 10 candles to calculate actual trend angle, steepness, direction + # REAL TREND CALCULATION from historical + FUTURE price movement + # Calculate trend target from current price to future predicted candles + # This tells the model what the ACTUAL trend will be, not what it was - # Get price data from the batch to calculate actual trend + import math + + # Get current price (last close from historical data) price_data = price_data_1m if price_data_1m is not None else ( price_data_1s if price_data_1s is not None else price_data_1h) - if price_data is not None and price_data.shape[1] >= 10: - # price_data shape: [batch=1, seq_len=200, features=5] -> OHLCV - recent_closes = price_data[0, -10:, 3] # Last 10 close prices [10] - - # Calculate actual price change and time delta - price_start = recent_closes[0].item() - price_end = recent_closes[-1].item() - price_delta = price_end - price_start - time_delta = 9.0 # 10 candles = 9 intervals + current_price = None + if price_data is not None and price_data.shape[1] > 0: + current_price = price_data[0, -1, 3].item() # Last close price + + # Try to get future price from next candle predictions + # This represents the ACTUAL trend that will happen (ground truth) + future_price = None + timeframe_for_trend = None + + # Check all available timeframes for next candle data + if timeframes and '1s' in timeframes and '1s' in norm_params_dict: + future_candle = self._extract_next_candle(timeframes['1s'], norm_params_dict['1s']) + if future_candle is not None: + future_price = future_candle[0, 3].item() # Close price from first row + timeframe_for_trend = '1s' + + if future_price is None and timeframes and '1m' in timeframes and '1m' in norm_params_dict: + future_candle = self._extract_next_candle(timeframes['1m'], norm_params_dict['1m']) + if future_candle is not None: + future_price = future_candle[0, 3].item() # Close price from first row + timeframe_for_trend = '1m' + + # Calculate trend from current to future (what will actually happen) + if current_price and future_price and current_price > 0: + price_delta = future_price - current_price + time_delta = 1.0 # 1 candle ahead # Calculate real angle using atan2 - import math - trend_angle = math.atan2(price_delta, time_delta * price_start / 100.0) # Normalize by price scale + trend_angle = math.atan2(price_delta, time_delta * current_price / 100.0) # Calculate real steepness (magnitude of change) - if price_start > 0: - price_change_pct = abs(price_delta / price_start) - trend_steepness = min(price_change_pct * 100.0, 1.0) # Scale and cap at 1.0 - else: - trend_steepness = 0.0 + price_change_pct = abs(price_delta / current_price) + trend_steepness = min(price_change_pct * 100.0, 1.0) # Scale and cap at 1.0 # Calculate real direction trend_direction = 1.0 if price_delta > 0 else (-1.0 if price_delta < 0 else 0.0) else: - # Fallback if no price data available (should rarely happen) - trend_angle = 0.0 - trend_steepness = 0.0 - trend_direction = 0.0 + # Fallback: use recent historical trend if future data not available + if price_data is not None and price_data.shape[1] >= 5: + recent_closes = price_data[0, -5:, 3] # Last 5 closes + price_start = recent_closes[0].item() + price_end = recent_closes[-1].item() + price_delta = price_end - price_start + + if price_start > 0: + trend_angle = math.atan2(price_delta, 4.0 * price_start / 100.0) + trend_steepness = min(abs(price_delta / price_start) * 100.0, 1.0) + trend_direction = 1.0 if price_delta > 0 else (-1.0 if price_delta < 0 else 0.0) + else: + trend_angle, trend_steepness, trend_direction = 0.0, 0.0, 0.0 + else: + trend_angle, trend_steepness, trend_direction = 0.0, 0.0, 0.0 # Create trend target tensor [batch, 3]: [angle, steepness, direction] trend_target = torch.tensor([[trend_angle, trend_steepness, trend_direction]], dtype=torch.float32) # [1, 3] diff --git a/TREND_TARGET_IMPROVEMENT.md b/TREND_TARGET_IMPROVEMENT.md new file mode 100644 index 0000000..8ccc8d0 --- /dev/null +++ b/TREND_TARGET_IMPROVEMENT.md @@ -0,0 +1,183 @@ +# Trend Target Calculation - Forward-Looking Approach + +## Problem + +Trend targets were calculated from **historical price movement** (last 10 candles), not from **future price movement** (what will actually happen). + +This meant the model was learning: +- ❌ "The price WAS going up" → predict "up trend" +- ✓ Should learn: "The price WILL go up" → predict "up trend" + +## Solution + +Changed trend target calculation to use **FUTURE price** from the next candle: + +### Before (Historical): +```python +# Used last 10 candles to calculate trend +recent_closes = price_data[0, -10:, 3] +price_start = recent_closes[0] # 10 candles ago +price_end = recent_closes[-1] # Current candle +trend_angle = atan2(price_end - price_start, ...) # Past trend +``` + +**Problem**: Model learns to predict what already happened! + +### After (Forward-Looking): +```python +# Use current price and NEXT candle price +current_price = price_data[0, -1, 3] # Current close +future_candle = _extract_next_candle(...) # Next candle (ground truth) +future_price = future_candle[3] # Future close + +# Calculate trend from NOW to FUTURE +price_delta = future_price - current_price +trend_angle = atan2(price_delta, ...) # FUTURE trend! +``` + +**Benefit**: Model learns to predict what WILL happen! + +## How It Works + +### 1. Primary Method - Future Candle Data +```python +# Try 1s timeframe first (most responsive) +if future_candle_1s available: + future_price = future_candle_1s[3] # Close + +# Fallback to 1m timeframe +elif future_candle_1m available: + future_price = future_candle_1m[3] + +# Calculate trend from current to future +trend_angle = atan2(future_price - current_price, ...) +trend_steepness = abs(future_price - current_price) / current_price +trend_direction = +1 (up) or -1 (down) +``` + +### 2. Fallback Method - Recent Historical Trend +```python +# Only if future data not available +if no future data: + # Use last 5 candles as fallback + recent_closes = price_data[0, -5:, 3] + trend_angle = atan2(recent_closes[-1] - recent_closes[0], ...) +``` + +## Impact + +### Training Behavior: +- **Before**: "Model, learn that when price went up, trend was up" (obvious!) +- **After**: "Model, learn to predict when price WILL go up" (useful!) + +### Trend Line Display: +The yellow trend line now shows: +- **Prediction**: Model's forecast of future trend +- **Target**: What actually happened (for comparison) + +This lets you see if the model is learning correctly! + +## Technical Details + +### Trend Vector Components: +- **angle** (radians): Direction and steepness combined + - Positive = upward trend + - Negative = downward trend + - Magnitude = steepness + +- **steepness** (0 to 1): How steep the trend is + - Calculated as: `abs(price_delta / current_price) * 100` + - Capped at 1.0 + +- **direction** (+1, 0, -1): Simple direction indicator + - +1.0 = up + - -1.0 = down + - 0.0 = sideways + +### Loss Calculation: +```python +# NN/models/advanced_transformer_trading.py +trend_pred = [model_angle, model_steepness, model_direction] +trend_target = [actual_angle, actual_steepness, actual_direction] +trend_loss = MSE(trend_pred, trend_target) + +total_loss = action_loss + 0.1*price_loss + 0.05*trend_loss + 0.15*candle_loss +``` + +Trend loss weight = **5%** of total loss. + +## Example + +### Current State: +- Current price: $2750 +- Last 5 candles: $2740, $2745, $2748, $2749, $2750 (trending up) + +### Old Method (Historical): +``` +trend_target: + angle = atan2(2750 - 2740, ...) = 0.15 rad ≈ 8.6° + direction = +1.0 (up) +``` +→ Model learns "price was going up" + +### New Method (Forward-Looking): +``` +Next candle (ground truth): Close = $2755 + +trend_target: + angle = atan2(2755 - 2750, ...) = 0.25 rad ≈ 14.3° + direction = +1.0 (up) +``` +→ Model learns "price WILL go up by $5" + +## Why This Matters + +1. **Predictive vs Reactive**: + - Old: Reactive (learns what happened) + - New: Predictive (learns what will happen) + +2. **Trading Value**: + - Old: "Market was bullish" (too late!) + - New: "Market will be bullish" (actionable!) + +3. **Model Accuracy**: + - Old: Can achieve high accuracy by just looking at past + - New: Must actually predict future to be accurate + +## Testing + +### Before Training: +- Trend predictions will be random/wrong +- Yellow line will point in wrong direction + +### After 5-10 Epochs: +- Trend predictions should match actual future movement +- Yellow line should point correctly more often +- Trend accuracy should reach 40-60% + +### Check Console Logs: +``` +Trend loss: 0.0234 (pred=[0.12, 0.45, 1.0], target=[0.15, 0.52, 1.0]) +trend_accuracy: 78.3% (angle: 85.2%, steepness: 71.4%) +``` + +## Files Modified + +- `/ANNOTATE/core/real_training_adapter.py` (lines 1618-1682) + - Changed from historical (last 10 candles) to forward-looking (next candle) + - Added fallback to historical if future data not available + +## Related Fixes + +This complements the earlier fixes: +1. ✅ Removed synthetic fixed-angle trend data (±45°) +2. ✅ Now uses real future price movement +3. ✅ Trend line projection adjusted for better visualization + +## Status + +✅ **IMPLEMENTED** - Trend targets now forward-looking +⏳ **PENDING** - Needs testing with fresh training +📊 **EXPECTED** - Better trend predictions after training + + diff --git a/run_experimental_gpu.sh b/run_experimental_gpu.sh index 55ca6ea..70d522a 100644 --- a/run_experimental_gpu.sh +++ b/run_experimental_gpu.sh @@ -3,6 +3,7 @@ # This tells ROCm to treat gfx1151 as gfx1100 export HSA_OVERRIDE_GFX_VERSION=11.0.0 export AMD_SERIALIZE_KERNEL=3 # Enable debugging +export TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1 # Enable Flash Efficient attention cd /mnt/shared/DEV/repos/d-popov.com/gogo2 source venv/bin/activate python ANNOTATE/web/app.py "$@" diff --git a/start_with_gpu.sh b/start_with_gpu.sh index 265a695..12c258a 100644 --- a/start_with_gpu.sh +++ b/start_with_gpu.sh @@ -7,10 +7,11 @@ export HSA_OVERRIDE_GFX_VERSION=11.0.0 # Activate virtual environment source venv/bin/activate -# Optional: Enable experimental features for better performance -# export TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1 +# Enable experimental Flash Efficient attention for AMD GPU +export TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1 echo "GPU Compatibility: HSA_OVERRIDE_GFX_VERSION=11.0.0" +echo "Experimental Features: TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1" echo "Virtual environment: $(which python)" echo "" echo "Starting application..."