gogo2/COBY/docker-compose.portainer.yml

# Docker Compose for COBY Multi-Exchange Data Aggregation System
# Optimized for Portainer deployment with Git repository integration
version: '3.8'

services:
  # TimescaleDB Database
  timescaledb:
    image: timescale/timescaledb:latest-pg15
    container_name: coby-timescaledb
    environment:
      POSTGRES_DB: market_data
      POSTGRES_USER: market_user
      POSTGRES_PASSWORD: market_data_secure_pass_2024
      TIMESCALEDB_TELEMETRY: 'off'
    ports:
      - "5432:5432"
    volumes:
      - timescale_data:/var/lib/postgresql/data
      - ./COBY/docker/init-scripts:/docker-entrypoint-initdb.d:ro
    networks:
      - coby-network
    restart: unless-stopped
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U market_user -d market_data"]
      interval: 30s
      timeout: 10s
      retries: 5
      start_period: 60s
    deploy:
      resources:
        limits:
          memory: 1G
          cpus: '1.0'
        reservations:
          memory: 512M
          cpus: '0.5'
    logging:
      driver: "json-file"
      options:
        max-size: "10m"
        max-file: "3"

  # Redis Cache
  redis:
    image: redis:7-alpine
    container_name: coby-redis
    command: >
      redis-server 
      --requirepass market_data_redis_2024
      --maxmemory 256mb
      --maxmemory-policy allkeys-lru
      --save 900 1
      --save 300 10
      --save 60 10000
    ports:
      - "6379:6379"
    volumes:
      - redis_data:/data
    networks:
      - coby-network
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "redis-cli", "--no-auth-warning", "-a", "market_data_redis_2024", "ping"]
      interval: 30s
      timeout: 5s
      retries: 3
    deploy:
      resources:
        limits:
          memory: 512M
          cpus: '0.5'
        reservations:
          memory: 128M
          cpus: '0.1'
    logging:
      driver: "json-file"
      options:
        max-size: "10m"
        max-file: "3"

  # COBY Main Application
  coby-app:
    build:
      context: ./COBY
      dockerfile: Dockerfile
      target: production
    container_name: coby-app
    environment:
      # Database configuration
      DB_HOST: timescaledb
      DB_PORT: 5432
      DB_NAME: market_data
      DB_USER: market_user
      DB_PASSWORD: market_data_secure_pass_2024
      DB_SCHEMA: market_data
      DB_POOL_SIZE: 10
      DB_MAX_OVERFLOW: 20
      DB_POOL_TIMEOUT: 30
      
      # Redis configuration
      REDIS_HOST: redis
      REDIS_PORT: 6379
      REDIS_PASSWORD: market_data_redis_2024
      REDIS_DB: 0
      REDIS_MAX_CONNECTIONS: 50
      REDIS_SOCKET_TIMEOUT: 5
      REDIS_CONNECT_TIMEOUT: 5
      
      # Application configuration
      ENVIRONMENT: production
      DEBUG: false
      LOG_LEVEL: INFO
      LOG_FILE: logs/coby.log
      LOG_MAX_SIZE: 100
      LOG_BACKUP_COUNT: 5
      ENABLE_CORRELATION_ID: true
      
      # API configuration
      API_HOST: 0.0.0.0
      API_PORT: 8080
      WS_PORT: 8081
      CORS_ORIGINS: "*"
      RATE_LIMIT: 100
      MAX_WS_CONNECTIONS: 1000
      
      # Exchange configuration
      MAX_CONNECTIONS_PER_EXCHANGE: 5
      RECONNECT_DELAY: 5
      MAX_RECONNECT_ATTEMPTS: 10
      HEARTBEAT_INTERVAL: 30
      EXCHANGES: "binance,coinbase,kraken,bybit,okx,huobi,kucoin,gateio,bitfinex,mexc"
      SYMBOLS: "BTCUSDT,ETHUSDT"
      
      # Aggregation configuration
      BUCKET_SIZE: 1.0
      HEATMAP_DEPTH: 50
      UPDATE_FREQUENCY: 0.5
      VOLUME_THRESHOLD: 0.01
      
      # Performance configuration
      DATA_BUFFER_SIZE: 10000
      BATCH_WRITE_SIZE: 1000
      MAX_MEMORY_USAGE: 2048
      GC_THRESHOLD: 0.8
      PROCESSING_TIMEOUT: 10
      MAX_QUEUE_SIZE: 50000
      
      # Monitoring configuration
      METRICS_COLLECTION_INTERVAL: 1.0
      MAX_CPU_USAGE: 80.0
      MAX_MEMORY_USAGE_PERCENT: 85.0
      MIN_MEMORY_AVAILABLE_GB: 1.0
      MAX_LATENCY_MS: 100.0
      MAX_ERROR_RATE_PERCENT: 5.0
      MIN_THROUGHPUT_OPS: 100.0
    ports:
      - "8080:8080"
      - "8081:8081"
    volumes:
      - coby_logs:/app/logs
      - coby_data:/app/data
    networks:
      - coby-network
    depends_on:
      timescaledb:
        condition: service_healthy
      redis:
        condition: service_healthy
    restart: unless-stopped
    healthcheck:
      test: ["CMD", "python", "-c", "import requests; requests.get('http://localhost:8080/health', timeout=5)"]
      interval: 30s
      timeout: 10s
      retries: 3
      start_period: 120s
    deploy:
      resources:
        limits:
          memory: 2G
          cpus: '2.0'
        reservations:
          memory: 1G
          cpus: '1.0'
    logging:
      driver: "json-file"
      options:
        max-size: "50m"
        max-file: "5"



  # Prometheus (Optional - for monitoring)
  prometheus:
    image: prom/prometheus:latest
    container_name: coby-prometheus
    ports:
      - "9090:9090"
    volumes:
      - prometheus_data:/prometheus
    configs:
      - source: prometheus_config
        target: /etc/prometheus/prometheus.yml
      - source: alert_rules
        target: /etc/prometheus/alert_rules.yml
    networks:
      - coby-network
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--web.console.libraries=/etc/prometheus/console_libraries'
      - '--web.console.templates=/etc/prometheus/consoles'
      - '--storage.tsdb.retention.time=200h'
      - '--web.enable-lifecycle'
      - '--web.enable-admin-api'
    restart: unless-stopped
    profiles:
      - monitoring
    deploy:
      resources:
        limits:
          memory: 512M
          cpus: '0.5'
        reservations:
          memory: 256M
          cpus: '0.2'
    logging:
      driver: "json-file"
      options:
        max-size: "10m"
        max-file: "3"

  # Grafana (Optional - for visualization)
  grafana:
    image: grafana/grafana:latest
    container_name: coby-grafana
    ports:
      - "3001:3000"
    environment:
      GF_SECURITY_ADMIN_PASSWORD: admin123
      GF_USERS_ALLOW_SIGN_UP: false
      GF_INSTALL_PLUGINS: grafana-clock-panel,grafana-simple-json-datasource
      GF_SECURITY_ALLOW_EMBEDDING: true
    volumes:
      - grafana_data:/var/lib/grafana
    networks:
      - coby-network
    depends_on:
      - prometheus
    restart: unless-stopped
    profiles:
      - monitoring
    deploy:
      resources:
        limits:
          memory: 256M
          cpus: '0.3'
        reservations:
          memory: 128M
          cpus: '0.1'
    logging:
      driver: "json-file"
      options:
        max-size: "10m"
        max-file: "3"

  # Node Exporter for system metrics
  node-exporter:
    image: prom/node-exporter:latest
    container_name: coby-node-exporter
    ports:
      - "9100:9100"
    volumes:
      - /proc:/host/proc:ro
      - /sys:/host/sys:ro
      - /:/rootfs:ro
    command:
      - '--path.procfs=/host/proc'
      - '--path.rootfs=/rootfs'
      - '--path.sysfs=/host/sys'
      - '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)'
    networks:
      - coby-network
    restart: unless-stopped
    profiles:
      - monitoring
    deploy:
      resources:
        limits:
          memory: 128M
          cpus: '0.1'
    logging:
      driver: "json-file"
      options:
        max-size: "5m"
        max-file: "2"

# Network configuration
networks:
  coby-network:
    driver: bridge
    ipam:
      config:
        - subnet: 172.20.0.0/16
    labels:
      - "com.coby.network=main"

# Volume configuration
volumes:
  timescale_data:
    driver: local
    labels:
      - "com.coby.volume=database"
  redis_data:
    driver: local
    labels:
      - "com.coby.volume=cache"
  prometheus_data:
    driver: local
    labels:
      - "com.coby.volume=monitoring"
  grafana_data:
    driver: local
    labels:
      - "com.coby.volume=monitoring"
  coby_logs:
    driver: local
    labels:
      - "com.coby.volume=logs"
  coby_data:
    driver: local
    labels:
      - "com.coby.volume=data"

# Configuration files
configs:
  prometheus_config:
    content: |
      global:
        scrape_interval: 15s
        evaluation_interval: 15s
      rule_files:
        - "/etc/prometheus/alert_rules.yml"
      scrape_configs:
        - job_name: 'coby-app'
          static_configs:
            - targets: ['coby-app:8080']
          metrics_path: '/metrics'
          scrape_interval: 10s
          scrape_timeout: 5s
        - job_name: 'prometheus'
          static_configs:
            - targets: ['localhost:9090']
        - job_name: 'node'
          static_configs:
            - targets: ['node-exporter:9100']
          scrape_interval: 30s

  alert_rules:
    content: |
      groups:
        - name: coby_alerts
          rules:
            - alert: HighCPUUsage
              expr: system_cpu_usage > 80
              for: 2m
              labels:
                severity: warning
              annotations:
                summary: "High CPU usage detected"
                description: "CPU usage is above 80% for more than 2 minutes"
            - alert: HighMemoryUsage
              expr: system_memory_usage > 85
              for: 2m
              labels:
                severity: warning
              annotations:
                summary: "High memory usage detected"
                description: "Memory usage is above 85% for more than 2 minutes"
            - alert: ServiceDown
              expr: up == 0
              for: 1m
              labels:
                severity: critical
              annotations:
                summary: "Service is down"
                description: "{{ $$labels.job }} service is down"

# Labels for the entire stack
x-labels: &default-labels
  com.coby.project: "multi-exchange-data-aggregation"
  com.coby.version: "1.0.0"
  com.coby.environment: "production"