popov/gogo2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

infinite scroll fix

Browse Source
This commit is contained in:
Dobromir Popov
2025-10-24 22:46:44 +03:00
parent 6e58f4d88f
commit 07b82f0a1f
6 changed files with 352 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
ANNOTATE/core/data_loader.py
View File

@@ -44,7 +44,8 @@ class HistoricalDataLoader:
def get_data(self, symbol: str, timeframe: str,
start_time: Optional[datetime] = None,
end_time: Optional[datetime] = None,
limit: int = 500) -> Optional[pd.DataFrame]:
limit: int = 500,
direction: str = 'latest') -> Optional[pd.DataFrame]:
"""
Get historical data for symbol and timeframe
@@ -54,12 +55,13 @@ class HistoricalDataLoader:
start_time: Start time for data range
end_time: End time for data range
limit: Maximum number of candles to return
direction: 'latest' (most recent), 'before' (older data), 'after' (newer data)
Returns:
DataFrame with OHLCV data or None if unavailable
"""
# Check memory cache first
cache_key = f"{symbol}_{timeframe}_{start_time}_{end_time}_{limit}"
cache_key = f"{symbol}_{timeframe}_{start_time}_{end_time}_{limit}_{direction}"
if cache_key in self.memory_cache:
cached_data, cached_time = self.memory_cache[cache_key]
if datetime.now() - cached_time < self.cache_ttl:
@@ -152,8 +154,7 @@ class HistoricalDataLoader:
logger.info(f"Loaded {len(df)} candles for {symbol} {timeframe}")
return df
# Fallback: fetch from DataProvider's historical data method
# During startup, allow stale cache to avoid slow API calls
# Fallback: Try DuckDB first, then fetch from API if needed
if self.startup_mode:
logger.info(f"Loading data for {symbol} {timeframe} (startup mode: allow stale cache)")
df = self.data_provider.get_historical_data(
@@ -163,11 +164,33 @@ class HistoricalDataLoader:
allow_stale_cache=True
)
else:
logger.info(f"Fetching fresh data for {symbol} {timeframe}")
# Check DuckDB first for historical data
if self.data_provider.duckdb_storage and (start_time or end_time):
logger.info(f"Checking DuckDB for {symbol} {timeframe} historical data (direction={direction})")
df = self.data_provider.duckdb_storage.get_ohlcv_data(
symbol=symbol,
timeframe=timeframe,
start_time=start_time,
end_time=end_time,
limit=limit,
direction=direction
)
if df is not None and not df.empty:
logger.info(f"✅ Loaded {len(df)} candles from DuckDB for {symbol} {timeframe}")
# Cache in memory
self.memory_cache[cache_key] = (df.copy(), datetime.now())
return df
else:
logger.info(f"No data in DuckDB, fetching from API for {symbol} {timeframe}")
# Fetch from API and store in DuckDB
logger.info(f"Fetching data from API for {symbol} {timeframe}")
df = self.data_provider.get_historical_data(
symbol=symbol,
timeframe=timeframe,
limit=limit
limit=limit,
refresh=True # Force API fetch
)
if df is not None and not df.empty:

72
ANNOTATE/data/annotations/annotations_db.json
View File

@@ -1,69 +1,23 @@
{
"annotations": [
{
"annotation_id": "967f91f4-5f01-4608-86af-4a006d55bd3c",
"annotation_id": "dc35c362-6174-4db4-b4db-8cc58a4ba8e5",
"symbol": "ETH/USDT",
"timeframe": "1m",
"timeframe": "1h",
"entry": {
"timestamp": "2025-10-23 14:15",
"price": 3821.57,
"index": 421
"timestamp": "2025-10-07 13:00",
"price": 4755,
"index": 28
},
"exit": {
"timestamp": "2025-10-23 15:32",
"price": 3874.23,
"index": 498
"timestamp": "2025-10-11 21:00",
"price": 3643.33,
"index": 63
},
"direction": "LONG",
"profit_loss_pct": 1.377967693905904,
"direction": "SHORT",
"profit_loss_pct": 23.378969505783388,
"notes": "",
"created_at": "2025-10-23T18:36:05.807749",
"market_context": {
"entry_state": {},
"exit_state": {}
}
},
{
"annotation_id": "a16f92c0-7228-4293-a188-baf74ca9b284",
"symbol": "ETH/USDT",
"timeframe": "1m",
"entry": {
"timestamp": "2025-10-24 08:31",
"price": 3930.54,
"index": 101
},
"exit": {
"timestamp": "2025-10-24 08:46",
"price": 3965,
"index": 104
},
"direction": "LONG",
"profit_loss_pct": 0.8767243177782196,
"notes": "",
"created_at": "2025-10-24T12:56:59.390345",
"market_context": {
"entry_state": {},
"exit_state": {}
}
},
{
"annotation_id": "ee34388a-fcad-4c7e-8a0a-2e8d205d5357",
"symbol": "ETH/USDT",
"timeframe": "1m",
"entry": {
"timestamp": "2025-10-24 04:06",
"price": 3895.67,
"index": 18
},
"exit": {
"timestamp": "2025-10-24 05:24",
"price": 3965,
"index": 36
},
"direction": "LONG",
"profit_loss_pct": 1.7796681957147276,
"notes": "",
"created_at": "2025-10-24T13:55:31.843201",
"created_at": "2025-10-24T22:33:26.187249",
"market_context": {
"entry_state": {},
"exit_state": {}
@@ -71,7 +25,7 @@
}
],
"metadata": {
"total_annotations": 5,
"last_updated": "2025-10-24T13:55:31.843201"
"total_annotations": 1,
"last_updated": "2025-10-24T22:33:26.194492"
}
}

11
ANNOTATE/web/app.py
View File

@@ -423,13 +423,15 @@ class AnnotationDashboard:
@self.server.route('/api/chart-data', methods=['POST'])
def get_chart_data():
"""Get chart data for specified symbol and timeframes"""
"""Get chart data for specified symbol and timeframes with infinite scroll support"""
try:
data = request.get_json()
symbol = data.get('symbol', 'ETH/USDT')
timeframes = data.get('timeframes', ['1s', '1m', '1h', '1d'])
start_time_str = data.get('start_time')
end_time_str = data.get('end_time')
limit = data.get('limit', 500) # Allow client to request more data
direction = data.get('direction', 'latest') # 'latest', 'before', or 'after'
if not self.data_loader:
return jsonify({
@@ -445,6 +447,10 @@ class AnnotationDashboard:
end_time = datetime.fromisoformat(end_time_str.replace('Z', '+00:00')) if end_time_str else None
# Fetch data for each timeframe using data loader
# This will automatically:
# 1. Check DuckDB first
# 2. Fetch from API if not in cache
# 3. Store in DuckDB for future use
chart_data = {}
for timeframe in timeframes:
df = self.data_loader.get_data(
@@ -452,7 +458,8 @@ class AnnotationDashboard:
timeframe=timeframe,
start_time=start_time,
end_time=end_time,
limit=500
limit=limit,
direction=direction
)
if df is not None and not df.empty:

235
ANNOTATE/web/static/js/chart_manager.js
View File

@@ -334,6 +334,11 @@ class ChartManager {
this.updateChartInfo(timeframe, eventData);
});
// Add relayout handler for infinite scroll (load more data when zooming/panning)
plotElement.on('plotly_relayout', (eventData) => {
this.handleChartRelayout(timeframe, eventData);
});
console.log(`Chart created for ${timeframe} with ${data.timestamps.length} candles`);
}
@@ -1088,4 +1093,234 @@ class ChartManager {
infoElement.textContent = `O: ${open.toFixed(2)} H: ${high.toFixed(2)} L: ${low.toFixed(2)} C: ${close.toFixed(2)}`;
}
}
/**
* Handle chart relayout for infinite scroll
* Detects when user scrolls/zooms to edges and loads more data
*/
handleChartRelayout(timeframe, eventData) {
const chart = this.charts[timeframe];
if (!chart || !chart.data) return;
// Check if this is a range change (zoom/pan)
if (!eventData['xaxis.range[0]'] && !eventData['xaxis.range']) return;
// Get current visible range
const xRange = eventData['xaxis.range'] || [eventData['xaxis.range[0]'], eventData['xaxis.range[1]']];
if (!xRange || xRange.length !== 2) return;
const visibleStart = new Date(xRange[0]);
const visibleEnd = new Date(xRange[1]);
// Get data boundaries
const dataStart = new Date(chart.data.timestamps[0]);
const dataEnd = new Date(chart.data.timestamps[chart.data.timestamps.length - 1]);
// Calculate threshold (10% of visible range from edge)
const visibleRange = visibleEnd - visibleStart;
const threshold = visibleRange * 0.1;
// Check if we're near the left edge (need older data)
const nearLeftEdge = (visibleStart - dataStart) < threshold;
// Check if we're near the right edge (need newer data)
const nearRightEdge = (dataEnd - visibleEnd) < threshold;
console.log(`Relayout ${timeframe}: visible=${visibleStart.toISOString()} to ${visibleEnd.toISOString()}, data=${dataStart.toISOString()} to ${dataEnd.toISOString()}, nearLeft=${nearLeftEdge}, nearRight=${nearRightEdge}`);
// Load more data if near edges
if (nearLeftEdge) {
this.loadMoreData(timeframe, 'before', dataStart);
} else if (nearRightEdge) {
this.loadMoreData(timeframe, 'after', dataEnd);
}
}
/**
* Load more historical data for a timeframe
*/
async loadMoreData(timeframe, direction, referenceTime) {
const chart = this.charts[timeframe];
if (!chart) return;
// Prevent multiple simultaneous loads
if (chart.loading) {
console.log(`Already loading data for ${timeframe}, skipping...`);
return;
}
chart.loading = true;
this.showLoadingIndicator(timeframe, direction);
try {
// Calculate time range to fetch
const limit = 500; // Fetch 500 more candles
let startTime, endTime;
if (direction === 'before') {
// Load older data
endTime = referenceTime.toISOString();
startTime = null; // Let backend calculate based on limit
} else {
// Load newer data
startTime = referenceTime.toISOString();
endTime = null;
}
console.log(`Loading ${limit} more candles ${direction} ${referenceTime.toISOString()} for ${timeframe}`);
// Fetch more data from backend
const response = await fetch('/api/chart-data', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
symbol: window.appState?.currentSymbol || 'ETH/USDT',
timeframes: [timeframe],
start_time: startTime,
end_time: endTime,
limit: limit,
direction: direction
})
});
const result = await response.json();
if (result.success && result.chart_data && result.chart_data[timeframe]) {
const newData = result.chart_data[timeframe];
// Merge with existing data
this.mergeChartData(timeframe, newData, direction);
console.log(`Loaded ${newData.timestamps.length} new candles for ${timeframe}`);
window.showSuccess(`Loaded ${newData.timestamps.length} more candles`);
} else {
console.warn(`No more data available for ${timeframe} ${direction}`);
window.showWarning('No more historical data available');
}
} catch (error) {
console.error(`Error loading more data for ${timeframe}:`, error);
window.showError('Failed to load more data');
} finally {
chart.loading = false;
this.hideLoadingIndicator(timeframe);
}
}
/**
* Merge new data with existing chart data
*/
mergeChartData(timeframe, newData, direction) {
const chart = this.charts[timeframe];
if (!chart || !chart.data) return;
const existingData = chart.data;
let mergedData;
if (direction === 'before') {
// Prepend older data
mergedData = {
timestamps: [...newData.timestamps, ...existingData.timestamps],
open: [...newData.open, ...existingData.open],
high: [...newData.high, ...existingData.high],
low: [...newData.low, ...existingData.low],
close: [...newData.close, ...existingData.close],
volume: [...newData.volume, ...existingData.volume],
pivot_markers: { ...newData.pivot_markers, ...existingData.pivot_markers }
};
} else {
// Append newer data
mergedData = {
timestamps: [...existingData.timestamps, ...newData.timestamps],
open: [...existingData.open, ...newData.open],
high: [...existingData.high, ...newData.high],
low: [...existingData.low, ...newData.low],
close: [...existingData.close, ...newData.close],
volume: [...existingData.volume, ...newData.volume],
pivot_markers: { ...existingData.pivot_markers, ...newData.pivot_markers }
};
}
// Update stored data
chart.data = mergedData;
// Update the chart with merged data
this.updateSingleChart(timeframe, mergedData);
}
/**
* Update a single chart with new data
*/
updateSingleChart(timeframe, data) {
const chart = this.charts[timeframe];
if (!chart) return;
const plotId = chart.plotId;
// Create volume colors
const volumeColors = data.close.map((close, i) => {
if (i === 0) return '#3b82f6';
return close >= data.open[i] ? '#10b981' : '#ef4444';
});
// Update traces
const update = {
x: [data.timestamps, data.timestamps],
open: [data.open],
high: [data.high],
low: [data.low],
close: [data.close],
y: [undefined, data.volume],
'marker.color': [undefined, volumeColors]
};
Plotly.restyle(plotId, update, [0, 1]);
console.log(`Updated ${timeframe} chart with ${data.timestamps.length} candles`);
}
/**
* Show loading indicator on chart
*/
showLoadingIndicator(timeframe, direction) {
const chart = this.charts[timeframe];
if (!chart) return;
const plotElement = chart.element;
const loadingDiv = document.createElement('div');
loadingDiv.id = `loading-${timeframe}`;
loadingDiv.className = 'chart-loading-indicator';
loadingDiv.innerHTML = `
<div class="spinner-border spinner-border-sm text-primary" role="status">
<span class="visually-hidden">Loading...</span>
</div>
<span class="ms-2">Loading ${direction === 'before' ? 'older' : 'newer'} data...</span>
`;
loadingDiv.style.cssText = `
position: absolute;
top: 10px;
${direction === 'before' ? 'left' : 'right'}: 10px;
background: rgba(31, 41, 55, 0.9);
color: #f8f9fa;
padding: 8px 12px;
border-radius: 4px;
font-size: 12px;
z-index: 1000;
display: flex;
align-items: center;
`;
plotElement.parentElement.style.position = 'relative';
plotElement.parentElement.appendChild(loadingDiv);
}
/**
* Hide loading indicator
*/
hideLoadingIndicator(timeframe) {
const loadingDiv = document.getElementById(`loading-${timeframe}`);
if (loadingDiv) {
loadingDiv.remove();
}
}
}

59
ANNOTATE/web/templates/annotation_dashboard.html
View File

@@ -62,7 +62,7 @@
window.appState = {
currentSymbol: '{{ current_symbol }}',
currentTimeframes: {{ timeframes | tojson }},
annotations: {{ annotations | tojson }},
annotations: { { annotations | tojson } },
pendingAnnotation: null,
chartManager: null,
annotationManager: null,
@@ -299,24 +299,44 @@
toast.addEventListener('hidden.bs.toast', () => toast.remove());
}
function showWarning(message) {
const toast = document.createElement('div');
toast.className = 'toast align-items-center text-white bg-warning border-0';
toast.setAttribute('role', 'alert');
toast.innerHTML = `
<div class="d-flex">
<div class="toast-body">
<i class="fas fa-exclamation-triangle"></i>
${message}
</div>
<button type="button" class="btn-close btn-close-white me-2 m-auto" data-bs-dismiss="toast"></button>
</div>
`;
document.body.appendChild(toast);
const bsToast = new bootstrap.Toast(toast);
bsToast.show();
toast.addEventListener('hidden.bs.toast', () => toast.remove());
}
function deleteAnnotation(annotationId) {
console.log('=== deleteAnnotation called ===');
console.log('Annotation ID:', annotationId);
console.log('window.appState:', window.appState);
console.log('window.appState.annotations:', window.appState?.annotations);
if (!annotationId) {
console.error('No annotation ID provided');
showError('No annotation ID provided');
return;
}
if (!window.appState || !window.appState.annotations) {
console.error('appState not initialized');
showError('Application state not initialized. Please refresh the page.');
return;
}
// Check if annotation exists
const annotation = window.appState.annotations.find(a => a.annotation_id === annotationId);
if (!annotation) {
@@ -324,7 +344,7 @@
showError('Annotation not found');
return;
}
console.log('Found annotation:', annotation);
console.log('Current annotations count:', window.appState.annotations.length);
@@ -351,7 +371,7 @@
if (data.success) {
console.log('Delete successful, updating UI...');
// Remove from app state
const originalCount = window.appState.annotations.length;
window.appState.annotations = window.appState.annotations.filter(
@@ -404,10 +424,12 @@
console.log('renderAnnotationsList function exists:', typeof renderAnnotationsList);
console.log('showError function exists:', typeof showError);
console.log('showSuccess function exists:', typeof showSuccess);
console.log('showWarning function exists:', typeof showWarning);
// Make functions globally available
window.showError = showError;
window.showSuccess = showSuccess;
window.showWarning = showWarning;
window.renderAnnotationsList = renderAnnotationsList;
window.deleteAnnotation = deleteAnnotation;
window.highlightAnnotation = highlightAnnotation;
@@ -418,8 +440,9 @@
console.log(' - window.renderAnnotationsList:', typeof window.renderAnnotationsList);
console.log(' - window.showError:', typeof window.showError);
console.log(' - window.showSuccess:', typeof window.showSuccess);
console.log(' - window.showWarning:', typeof window.showWarning);
console.log(' - window.highlightAnnotation:', typeof window.highlightAnnotation);
// Test call
console.log('Testing window.deleteAnnotation availability...');
if (typeof window.deleteAnnotation === 'function') {
@@ -427,10 +450,10 @@
} else {
console.error('✗ window.deleteAnnotation is NOT a function!');
}
// Add a test button to verify functionality (temporary)
console.log('Adding test delete function to window for debugging...');
window.testDeleteFunction = function() {
window.testDeleteFunction = function () {
console.log('Test delete function called');
console.log('window.deleteAnnotation type:', typeof window.deleteAnnotation);
if (typeof window.deleteAnnotation === 'function') {
@@ -446,7 +469,7 @@
console.error('window.deleteAnnotation is NOT available');
}
};
// Add test button to page (temporary debugging)
const testButton = document.createElement('button');
testButton.textContent = 'Test Delete Function';
@@ -455,7 +478,7 @@
testButton.style.top = '10px';
testButton.style.right = '10px';
testButton.style.zIndex = '9999';
testButton.onclick = function() {
testButton.onclick = function () {
console.log('Test button clicked');
window.testDeleteFunction();
};
@@ -493,23 +516,23 @@
</div>
</div>
`;
// Add event listeners
item.querySelector('.highlight-btn').addEventListener('click', function(e) {
item.querySelector('.highlight-btn').addEventListener('click', function (e) {
e.stopPropagation();
console.log('Highlight button clicked for:', annotation.annotation_id);
if (typeof window.highlightAnnotation === 'function') {
window.highlightAnnotation(annotation.annotation_id);
}
});
item.querySelector('.delete-btn').addEventListener('click', function(e) {
item.querySelector('.delete-btn').addEventListener('click', function (e) {
e.stopPropagation();
console.log('=== Delete button clicked ===');
console.log('Annotation ID:', annotation.annotation_id);
console.log('window.deleteAnnotation type:', typeof window.deleteAnnotation);
console.log('window object keys containing delete:', Object.keys(window).filter(k => k.includes('delete')));
if (typeof window.deleteAnnotation === 'function') {
console.log('Calling window.deleteAnnotation...');
try {
@@ -524,7 +547,7 @@
showError('Delete function not available. Please refresh the page.');
}
});
listElement.appendChild(item);
});
}

35
core/duckdb_storage.py
View File

@@ -189,7 +189,8 @@ class DuckDBStorage:
def get_ohlcv_data(self, symbol: str, timeframe: str,
start_time: Optional[datetime] = None,
end_time: Optional[datetime] = None,
limit: Optional[int] = None) -> Optional[pd.DataFrame]:
limit: Optional[int] = None,
direction: str = 'latest') -> Optional[pd.DataFrame]:
"""
Query OHLCV data directly from DuckDB table
@@ -199,6 +200,7 @@ class DuckDBStorage:
start_time: Start time filter
end_time: End time filter
limit: Maximum number of candles
direction: 'latest' (most recent), 'before' (older data), 'after' (newer data)
Returns:
DataFrame with OHLCV data
@@ -212,15 +214,28 @@ class DuckDBStorage:
"""
params = [symbol, timeframe]
if start_time:
query += " AND timestamp >= ?"
params.append(int(start_time.timestamp() * 1000))
if end_time:
query += " AND timestamp <= ?"
# Handle different direction modes
if direction == 'before' and end_time:
# Get older data: candles BEFORE end_time
query += " AND timestamp < ?"
params.append(int(end_time.timestamp() * 1000))
query += " ORDER BY timestamp DESC"
query += " ORDER BY timestamp DESC"
elif direction == 'after' and start_time:
# Get newer data: candles AFTER start_time
query += " AND timestamp > ?"
params.append(int(start_time.timestamp() * 1000))
query += " ORDER BY timestamp ASC"
else:
# Default: get most recent data in range
if start_time:
query += " AND timestamp >= ?"
params.append(int(start_time.timestamp() * 1000))
if end_time:
query += " AND timestamp <= ?"
params.append(int(end_time.timestamp() * 1000))
query += " ORDER BY timestamp DESC"
if limit:
query += f" LIMIT {limit}"
@@ -236,7 +251,7 @@ class DuckDBStorage:
df = df.set_index('timestamp')
df = df.sort_index()
logger.debug(f"Retrieved {len(df)} candles for {symbol} {timeframe} from DuckDB")
logger.debug(f"Retrieved {len(df)} candles for {symbol} {timeframe} from DuckDB (direction={direction})")
return df
except Exception as e: