popov/gogo2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

use experimental features; runtime fix

Browse Source
This commit is contained in:
Dobromir Popov
2025-11-22 23:17:10 +02:00
parent f38a924b0f
commit 3eb74381a8
6 changed files with 263 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
.vscode/launch.json vendored
View File

@@ -16,7 +16,8 @@
"PYTHONUNBUFFERED": "1",
"ENABLE_REALTIME_CHARTS": "1",
"ENABLE_NN_MODELS": "1",
"HSA_OVERRIDE_GFX_VERSION": "11.0.0"
"HSA_OVERRIDE_GFX_VERSION": "11.0.0",
"TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL": "1"
},
"preLaunchTask": "Kill Stale Processes"
},
@@ -37,7 +38,8 @@
"justMyCode": false,
"env": {
"PYTHONUNBUFFERED": "1",
"HSA_OVERRIDE_GFX_VERSION": "11.0.0"
"HSA_OVERRIDE_GFX_VERSION": "11.0.0",
"TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL": "1"
}
},
{
@@ -58,7 +60,8 @@
"env": {
"PYTHONUNBUFFERED": "1",
"CUDA_VISIBLE_DEVICES": "0",
"HSA_OVERRIDE_GFX_VERSION": "11.0.0"
"HSA_OVERRIDE_GFX_VERSION": "11.0.0",
"TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL": "1"
}
},
{
@@ -80,7 +83,8 @@
"justMyCode": false,
"env": {
"PYTHONUNBUFFERED": "1",
"HSA_OVERRIDE_GFX_VERSION": "11.0.0"
"HSA_OVERRIDE_GFX_VERSION": "11.0.0",
"TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL": "1"
}
},
{
@@ -92,7 +96,8 @@
"justMyCode": false,
"env": {
"PYTHONUNBUFFERED": "1",
"HSA_OVERRIDE_GFX_VERSION": "11.0.0"
"HSA_OVERRIDE_GFX_VERSION": "11.0.0",
"TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL": "1"
}
},
{
@@ -106,7 +111,8 @@
"PYTHONUNBUFFERED": "1",
"FLASK_ENV": "development",
"FLASK_DEBUG": "1",
"HSA_OVERRIDE_GFX_VERSION": "11.0.0"
"HSA_OVERRIDE_GFX_VERSION": "11.0.0",
"TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL": "1"
},
"cwd": "${workspaceFolder}",
"preLaunchTask": "Kill Stale Processes"
@@ -122,7 +128,8 @@
"PYTHONUNBUFFERED": "1",
"COB_BTC_BUCKET_SIZE": "10",
"COB_ETH_BUCKET_SIZE": "1",
"HSA_OVERRIDE_GFX_VERSION": "11.0.0"
"HSA_OVERRIDE_GFX_VERSION": "11.0.0",
"TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL": "1"
},
"preLaunchTask": "Kill Stale Processes"
},
@@ -138,7 +145,8 @@
"CUDA_VISIBLE_DEVICES": "0",
"PYTORCH_CUDA_ALLOC_CONF": "max_split_size_mb:256",
"ENABLE_REALTIME_RL": "1",
"HSA_OVERRIDE_GFX_VERSION": "11.0.0"
"HSA_OVERRIDE_GFX_VERSION": "11.0.0",
"TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL": "1"
},
"preLaunchTask": "Kill Stale Processes"
},
@@ -156,7 +164,8 @@
"ENABLE_REALTIME_RL": "1",
"COB_BTC_BUCKET_SIZE": "10",
"COB_ETH_BUCKET_SIZE": "1",
"HSA_OVERRIDE_GFX_VERSION": "11.0.0"
"HSA_OVERRIDE_GFX_VERSION": "11.0.0",
"TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL": "1"
},
"preLaunchTask": "Kill Stale Processes"
},
@@ -169,7 +178,8 @@
"justMyCode": false,
"env": {
"PYTHONUNBUFFERED": "1",
"HSA_OVERRIDE_GFX_VERSION": "11.0.0"
"HSA_OVERRIDE_GFX_VERSION": "11.0.0",
"TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL": "1"
}
},
{
@@ -181,7 +191,8 @@
"justMyCode": false,
"env": {
"PYTHONUNBUFFERED": "1",
"HSA_OVERRIDE_GFX_VERSION": "11.0.0"
"HSA_OVERRIDE_GFX_VERSION": "11.0.0",
"TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL": "1"
}
},
@@ -202,7 +213,8 @@
"COBY_API_PORT": "8080",
"COBY_WEBSOCKET_PORT": "8081",
"COBY_LOG_LEVEL": "DEBUG",
"HSA_OVERRIDE_GFX_VERSION": "11.0.0"
"HSA_OVERRIDE_GFX_VERSION": "11.0.0",
"TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL": "1"
},
"preLaunchTask": "Kill Stale Processes",
"presentation": {

3
AMD_GPU_FIX.md
View File

@@ -84,7 +84,8 @@ The environment variable has been automatically added to your venv activation sc
### What was done:
1. Added `export HSA_OVERRIDE_GFX_VERSION=11.0.0` to `venv/bin/activate`
2. This allows gfx1151 to use gfx1100 libraries (fully compatible)
3. All PyTorch operations now work on GPU
3. Added `export TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1` for Flash Efficient attention
4. All PyTorch operations now work on GPU with experimental optimizations
### To apply:
```bash

73
ANNOTATE/core/real_training_adapter.py
View File

@@ -1615,41 +1615,68 @@ class RealTrainingAdapter:
# FIXED: Ensure shape is [1, 1] not [1] to match BCELoss requirements
trade_success = torch.tensor([[1.0 if profit_loss_pct > 0 else 0.0]], dtype=torch.float32) # [1, 1]
# REAL TREND CALCULATION from actual price data (NO MORE SYNTHETIC DATA!)
# Use last 10 candles to calculate actual trend angle, steepness, direction
# REAL TREND CALCULATION from historical + FUTURE price movement
# Calculate trend target from current price to future predicted candles
# This tells the model what the ACTUAL trend will be, not what it was
# Get price data from the batch to calculate actual trend
import math
# Get current price (last close from historical data)
price_data = price_data_1m if price_data_1m is not None else (
price_data_1s if price_data_1s is not None else price_data_1h)
if price_data is not None and price_data.shape[1] >= 10:
# price_data shape: [batch=1, seq_len=200, features=5] -> OHLCV
recent_closes = price_data[0, -10:, 3] # Last 10 close prices [10]
# Calculate actual price change and time delta
price_start = recent_closes[0].item()
price_end = recent_closes[-1].item()
price_delta = price_end - price_start
time_delta = 9.0 # 10 candles = 9 intervals
current_price = None
if price_data is not None and price_data.shape[1] > 0:
current_price = price_data[0, -1, 3].item() # Last close price
# Try to get future price from next candle predictions
# This represents the ACTUAL trend that will happen (ground truth)
future_price = None
timeframe_for_trend = None
# Check all available timeframes for next candle data
if timeframes and '1s' in timeframes and '1s' in norm_params_dict:
future_candle = self._extract_next_candle(timeframes['1s'], norm_params_dict['1s'])
if future_candle is not None:
future_price = future_candle[0, 3].item() # Close price from first row
timeframe_for_trend = '1s'
if future_price is None and timeframes and '1m' in timeframes and '1m' in norm_params_dict:
future_candle = self._extract_next_candle(timeframes['1m'], norm_params_dict['1m'])
if future_candle is not None:
future_price = future_candle[0, 3].item() # Close price from first row
timeframe_for_trend = '1m'
# Calculate trend from current to future (what will actually happen)
if current_price and future_price and current_price > 0:
price_delta = future_price - current_price
time_delta = 1.0 # 1 candle ahead
# Calculate real angle using atan2
import math
trend_angle = math.atan2(price_delta, time_delta * price_start / 100.0) # Normalize by price scale
trend_angle = math.atan2(price_delta, time_delta * current_price / 100.0)
# Calculate real steepness (magnitude of change)
if price_start > 0:
price_change_pct = abs(price_delta / price_start)
trend_steepness = min(price_change_pct * 100.0, 1.0) # Scale and cap at 1.0
else:
trend_steepness = 0.0
price_change_pct = abs(price_delta / current_price)
trend_steepness = min(price_change_pct * 100.0, 1.0) # Scale and cap at 1.0
# Calculate real direction
trend_direction = 1.0 if price_delta > 0 else (-1.0 if price_delta < 0 else 0.0)
else:
# Fallback if no price data available (should rarely happen)
trend_angle = 0.0
trend_steepness = 0.0
trend_direction = 0.0
# Fallback: use recent historical trend if future data not available
if price_data is not None and price_data.shape[1] >= 5:
recent_closes = price_data[0, -5:, 3] # Last 5 closes
price_start = recent_closes[0].item()
price_end = recent_closes[-1].item()
price_delta = price_end - price_start
if price_start > 0:
trend_angle = math.atan2(price_delta, 4.0 * price_start / 100.0)
trend_steepness = min(abs(price_delta / price_start) * 100.0, 1.0)
trend_direction = 1.0 if price_delta > 0 else (-1.0 if price_delta < 0 else 0.0)
else:
trend_angle, trend_steepness, trend_direction = 0.0, 0.0, 0.0
else:
trend_angle, trend_steepness, trend_direction = 0.0, 0.0, 0.0
# Create trend target tensor [batch, 3]: [angle, steepness, direction]
trend_target = torch.tensor([[trend_angle, trend_steepness, trend_direction]], dtype=torch.float32) # [1, 3]

183
TREND_TARGET_IMPROVEMENT.md Normal file
View File

@@ -0,0 +1,183 @@
# Trend Target Calculation - Forward-Looking Approach
## Problem
Trend targets were calculated from **historical price movement** (last 10 candles), not from **future price movement** (what will actually happen).
This meant the model was learning:
- ❌ "The price WAS going up" → predict "up trend"
- ✓ Should learn: "The price WILL go up" → predict "up trend"
## Solution
Changed trend target calculation to use **FUTURE price** from the next candle:
### Before (Historical):
```python
# Used last 10 candles to calculate trend
recent_closes = price_data[0, -10:, 3]
price_start = recent_closes[0] # 10 candles ago
price_end = recent_closes[-1] # Current candle
trend_angle = atan2(price_end - price_start, ...) # Past trend
```
**Problem**: Model learns to predict what already happened!
### After (Forward-Looking):
```python
# Use current price and NEXT candle price
current_price = price_data[0, -1, 3] # Current close
future_candle = _extract_next_candle(...) # Next candle (ground truth)
future_price = future_candle[3] # Future close
# Calculate trend from NOW to FUTURE
price_delta = future_price - current_price
trend_angle = atan2(price_delta, ...) # FUTURE trend!
```
**Benefit**: Model learns to predict what WILL happen!
## How It Works
### 1. Primary Method - Future Candle Data
```python
# Try 1s timeframe first (most responsive)
if future_candle_1s available:
future_price = future_candle_1s[3] # Close
# Fallback to 1m timeframe
elif future_candle_1m available:
future_price = future_candle_1m[3]
# Calculate trend from current to future
trend_angle = atan2(future_price - current_price, ...)
trend_steepness = abs(future_price - current_price) / current_price
trend_direction = +1 (up) or -1 (down)
```
### 2. Fallback Method - Recent Historical Trend
```python
# Only if future data not available
if no future data:
# Use last 5 candles as fallback
recent_closes = price_data[0, -5:, 3]
trend_angle = atan2(recent_closes[-1] - recent_closes[0], ...)
```
## Impact
### Training Behavior:
- **Before**: "Model, learn that when price went up, trend was up" (obvious!)
- **After**: "Model, learn to predict when price WILL go up" (useful!)
### Trend Line Display:
The yellow trend line now shows:
- **Prediction**: Model's forecast of future trend
- **Target**: What actually happened (for comparison)
This lets you see if the model is learning correctly!
## Technical Details
### Trend Vector Components:
- **angle** (radians): Direction and steepness combined
- Positive = upward trend
- Negative = downward trend
- Magnitude = steepness
- **steepness** (0 to 1): How steep the trend is
- Calculated as: `abs(price_delta / current_price) * 100`
- Capped at 1.0
- **direction** (+1, 0, -1): Simple direction indicator
- +1.0 = up
- -1.0 = down
- 0.0 = sideways
### Loss Calculation:
```python
# NN/models/advanced_transformer_trading.py
trend_pred = [model_angle, model_steepness, model_direction]
trend_target = [actual_angle, actual_steepness, actual_direction]
trend_loss = MSE(trend_pred, trend_target)
total_loss = action_loss + 0.1*price_loss + 0.05*trend_loss + 0.15*candle_loss
```
Trend loss weight = **5%** of total loss.
## Example
### Current State:
- Current price: $2750
- Last 5 candles: $2740, $2745, $2748, $2749, $2750 (trending up)
### Old Method (Historical):
```
trend_target:
angle = atan2(2750 - 2740, ...) = 0.15 rad ≈ 8.6°
direction = +1.0 (up)
```
→ Model learns "price was going up"
### New Method (Forward-Looking):
```
Next candle (ground truth): Close = $2755
trend_target:
angle = atan2(2755 - 2750, ...) = 0.25 rad ≈ 14.3°
direction = +1.0 (up)
```
→ Model learns "price WILL go up by $5"
## Why This Matters
1. **Predictive vs Reactive**:
- Old: Reactive (learns what happened)
- New: Predictive (learns what will happen)
2. **Trading Value**:
- Old: "Market was bullish" (too late!)
- New: "Market will be bullish" (actionable!)
3. **Model Accuracy**:
- Old: Can achieve high accuracy by just looking at past
- New: Must actually predict future to be accurate
## Testing
### Before Training:
- Trend predictions will be random/wrong
- Yellow line will point in wrong direction
### After 5-10 Epochs:
- Trend predictions should match actual future movement
- Yellow line should point correctly more often
- Trend accuracy should reach 40-60%
### Check Console Logs:
```
Trend loss: 0.0234 (pred=[0.12, 0.45, 1.0], target=[0.15, 0.52, 1.0])
trend_accuracy: 78.3% (angle: 85.2%, steepness: 71.4%)
```
## Files Modified
- `/ANNOTATE/core/real_training_adapter.py` (lines 1618-1682)
- Changed from historical (last 10 candles) to forward-looking (next candle)
- Added fallback to historical if future data not available
## Related Fixes
This complements the earlier fixes:
1. ✅ Removed synthetic fixed-angle trend data (±45°)
2. ✅ Now uses real future price movement
3. ✅ Trend line projection adjusted for better visualization
## Status
✅ **IMPLEMENTED** - Trend targets now forward-looking
⏳ **PENDING** - Needs testing with fresh training
📊 **EXPECTED** - Better trend predictions after training

1
run_experimental_gpu.sh
View File

@@ -3,6 +3,7 @@
# This tells ROCm to treat gfx1151 as gfx1100
export HSA_OVERRIDE_GFX_VERSION=11.0.0
export AMD_SERIALIZE_KERNEL=3 # Enable debugging
export TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1 # Enable Flash Efficient attention
cd /mnt/shared/DEV/repos/d-popov.com/gogo2
source venv/bin/activate
python ANNOTATE/web/app.py "$@"

5
start_with_gpu.sh
View File

@@ -7,10 +7,11 @@ export HSA_OVERRIDE_GFX_VERSION=11.0.0
# Activate virtual environment
source venv/bin/activate
# Optional: Enable experimental features for better performance
# export TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1
# Enable experimental Flash Efficient attention for AMD GPU
export TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1
echo "GPU Compatibility: HSA_OVERRIDE_GFX_VERSION=11.0.0"
echo "Experimental Features: TORCH_ROCM_AOTRITON_ENABLE_EXPERIMENTAL=1"
echo "Virtual environment: $(which python)"
echo ""
echo "Starting application..."