Upload files.

backend/src/binance_collector.rs

@@ -0,0 +1,709 @@
+use crate::error::*;
+use crate::structs::*;
+use crate::historical_downloader::HistoricalDownloader;
+
+use binance_sdk::{
+    config::{ConfigurationRestApi, ProxyConfig, ProxyAuth},
+    derivatives_trading_usds_futures::{
+        DerivativesTradingUsdsFuturesRestApi,
+        rest_api::{
+            RestApi, 
+            KlineCandlestickDataIntervalEnum, KlineCandlestickDataParams, KlineCandlestickDataResponseItemInner,
+        },
+        websocket_streams::KlineCandlestickStreamsResponseK,
+    },
+};
+
+use futures::StreamExt;
+use log::{debug, error, info, warn};
+use std::collections::HashMap;
+use std::sync::Arc;
+use tokio::sync::{mpsc, RwLock};
+use reqwest::Client;
+use tokio_tungstenite::{connect_async, tungstenite::Message};
+use rand::Rng;
+use serde::Deserialize;
+
+/// Read proxy configuration from environment variables.
+/// Returns None if PROXY_HOST is not set (direct connection).
+fn proxy_config_from_env(port: u16) -> Option<ProxyConfig> {
+    let host = std::env::var("PROXY_HOST").ok()?;
+    if host.is_empty() {
+        return None;
+    }
+    let username = std::env::var("PROXY_USERNAME").unwrap_or_default();
+    let password = std::env::var("PROXY_PASSWORD").unwrap_or_default();
+    let protocol = std::env::var("PROXY_PROTOCOL").unwrap_or_else(|_| "https".to_string());
+
+    let auth = if username.is_empty() {
+        None
+    } else {
+        Some(ProxyAuth {
+            username,
+            password,
+        })
+    };
+
+    Some(ProxyConfig {
+        host,
+        port,
+        protocol: Some(protocol),
+        auth,
+    })
+}
+
+// Combined stream response wrapper
+#[derive(Debug, Deserialize)]
+struct CombinedStreamWrapper {
+    #[allow(dead_code)]
+    stream: String,
+    data: CombinedStreamData,
+}
+
+#[derive(Debug, Deserialize)]
+struct CombinedStreamData {
+    s: Option<String>,
+    k: Option<KlineCandlestickStreamsResponseK>,
+}
+
+const BATCH_SIZE: i64 = 1000;
+const ONE_MINUTE_MS: i64 = 60_000;
+const MAX_SYMBOLS_PER_WS: usize = 50;  // Binance limit per WebSocket connection
+// Rate limit: 2400 weight/min, limit=1000 costs 5 weight
+// 2400/5 = 480 requests/min = 8 req/sec = 125ms interval
+// Use 150ms for safety margin
+const REQUEST_INTERVAL_MS: u64 = 150;
+
+pub struct BinanceCollector {
+    pub symbols: Vec<String>,
+    rest_client: RestApi,
+    last_closed_timestamps: Arc<RwLock<HashMap<String, i64>>>,
+}
+
+impl BinanceCollector {
+    pub fn new(symbols: Vec<String>) -> Self {
+        assert!(!symbols.is_empty(), "symbols cannot be empty");
+
+        let symbols: Vec<String> = symbols.into_iter().map(|s| s.to_lowercase()).collect();
+
+        let port: u16 = rand::rng().random_range(10036..=10066);
+
+        let mut config_builder = ConfigurationRestApi::builder()
+            .timeout(10000);
+
+        if let Some(proxy) = proxy_config_from_env(port) {
+            config_builder = config_builder.proxy(proxy);
+        }
+
+        let rest_client_config = config_builder
+            .build()
+            .expect("Failed to initialize the rest api client");
+
+        let rest_client = DerivativesTradingUsdsFuturesRestApi::production(rest_client_config);
+
+        Self {
+            symbols,
+            rest_client,
+            last_closed_timestamps: Arc::new(RwLock::new(HashMap::new())),
+        }
+    }
+
+    /// Backfill K-line data for a specific symbol from start_time to end_time.
+    /// Uses limit=1000 per request (weight=5, ~480 requests/min allowed).
+    /// Returns total number of candles fetched.
+    pub async fn backfill(
+        &self,
+        symbol: &str,
+        start_time: i64,
+        end_time: i64,
+        candle_tx: &mpsc::Sender<CandleData>,
+    ) -> Result<u64, CollectorError> {
+        let symbol_upper = symbol.to_uppercase();
+        let symbol_lower = symbol.to_lowercase();
+        let mut current_start = start_time;
+        let mut total_count: u64 = 0;
+
+        info!(
+            "Starting backfill for {} from {} to {}",
+            symbol_upper, start_time, end_time
+        );
+
+        while current_start < end_time {
+            let params = KlineCandlestickDataParams::builder(
+                symbol_upper.clone(),
+                KlineCandlestickDataIntervalEnum::Interval1m,
+            )
+            .start_time(Some(current_start))
+            .end_time(Some(end_time))
+            .limit(Some(BATCH_SIZE))
+            .build()
+            .map_err(|e| CollectorError::RestApiError(e.to_string()))?;
+
+            let response = self.rest_client
+                .kline_candlestick_data(params)
+                .await
+                .map_err(|e| CollectorError::RestApiError(e.to_string()))?;
+
+            let klines = response.data().await
+                .map_err(|e| CollectorError::RestApiError(e.to_string()))?;
+
+            let batch_count = klines.len();
+            if batch_count == 0 {
+                break;
+            }
+
+            let mut last_timestamp = current_start;
+
+            for kline in &klines {
+                if let Some(candle) = Self::parse_rest_kline(&symbol_upper, kline) {
+                    last_timestamp = candle.timestamp;
+
+                    if let Err(e) = candle_tx.send(candle).await {
+                        warn!("Failed to send candle: {}", e);
+                    }
+
+                    total_count += 1;
+                }
+            }
+
+            // Update last_closed_timestamp for this symbol
+            {
+                let mut timestamps = self.last_closed_timestamps.write().await;
+                timestamps.insert(symbol_lower.clone(), last_timestamp);
+            }
+
+            debug!(
+                "Fetched {} klines for {}, total: {}, last_ts: {}",
+                batch_count, symbol_upper, total_count, last_timestamp
+            );
+
+            current_start = last_timestamp + ONE_MINUTE_MS;
+
+            if batch_count < BATCH_SIZE as usize {
+                break;
+            }
+
+            tokio::time::sleep(tokio::time::Duration::from_millis(REQUEST_INTERVAL_MS)).await;
+        }
+
+        info!(
+            "Backfill completed for {}: {} candles",
+            symbol_upper, total_count
+        );
+
+        Ok(total_count)
+    }
+
+    /// Backfill all symbols in the collector.
+    pub async fn backfill_all(
+        &self,
+        start_time: i64,
+        end_time: i64,
+        candle_tx: &mpsc::Sender<CandleData>,
+    ) -> Result<u64, CollectorError> {
+        let mut total_count: u64 = 0;
+
+        for symbol in &self.symbols.clone() {
+            let count = self.backfill(symbol, start_time, end_time, candle_tx).await?;
+            total_count += count;
+        }
+
+        Ok(total_count)
+    }
+
+    /// Start WebSocket streams for real-time K-line data on all symbols.
+    /// Creates multiple connections if symbols > MAX_SYMBOLS_PER_WS.
+    /// Automatically reconnects and backfills missed data on disconnect.
+    pub async fn start_stream(
+        &self,
+        candle_tx: mpsc::Sender<CandleData>,
+    ) -> Result<(), CollectorError> {
+        let total_symbols = self.symbols.len();
+        let num_connections = (total_symbols + MAX_SYMBOLS_PER_WS - 1) / MAX_SYMBOLS_PER_WS;
+        
+        info!(
+            "Starting {} WebSocket connections for {} symbols",
+            num_connections, total_symbols
+        );
+
+        // Split symbols into batches
+        let batches: Vec<Vec<String>> = self.symbols
+            .chunks(MAX_SYMBOLS_PER_WS)
+            .map(|chunk| chunk.to_vec())
+            .collect();
+
+        loop {
+            // Start all WebSocket connections in parallel
+            let mut handles = Vec::new();
+            
+            for (i, batch) in batches.iter().enumerate() {
+                let batch_symbols = batch.clone();
+                let tx = candle_tx.clone();
+                let last_ts = self.last_closed_timestamps.clone();
+                
+                let handle = tokio::spawn(async move {
+                    Self::run_single_ws_connection(i, batch_symbols, tx, last_ts).await
+                });
+                handles.push(handle);
+            }
+
+            // Wait for any connection to fail
+            let mut all_ok = true;
+            for (i, handle) in handles.into_iter().enumerate() {
+                match handle.await {
+                    Ok(Ok(())) => info!("WS connection {} ended normally", i),
+                    Ok(Err(e)) => {
+                        error!("WS connection {} error: {}", i, e);
+                        all_ok = false;
+                    }
+                    Err(e) => {
+                        error!("WS connection {} join error: {}", i, e);
+                        all_ok = false;
+                    }
+                }
+            }
+
+            if all_ok {
+                break;
+            }
+
+            // Reconnect after error
+            error!("WebSocket error. Reconnecting in 5 seconds...");
+            tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;
+
+            // Backfill missed data
+            let now = std::time::SystemTime::now()
+                .duration_since(std::time::UNIX_EPOCH)
+                .unwrap()
+                .as_millis() as i64;
+
+            let timestamps = self.last_closed_timestamps.read().await.clone();
+            
+            for symbol in &self.symbols {
+                if let Some(&last_ts) = timestamps.get(symbol) {
+                    if now - last_ts > ONE_MINUTE_MS {
+                        info!("Backfilling missed data for {} from {}", symbol, last_ts);
+                        if let Err(e) = self.backfill(symbol, last_ts, now, &candle_tx).await {
+                            warn!("Backfill failed for {}: {}", symbol, e);
+                        }
+                    }
+                }
+            }
+        }
+        Ok(())
+    }
+
+    /// Run a single WebSocket connection for a batch of symbols (combined streams)
+    async fn run_single_ws_connection(
+        connection_id: usize,
+        symbols: Vec<String>,
+        candle_tx: mpsc::Sender<CandleData>,
+        last_closed_timestamps: Arc<RwLock<HashMap<String, i64>>>,
+    ) -> Result<(), CollectorError> {
+        // Build combined stream URL: wss://fstream.binance.com/stream?streams=symbol1@kline_1m/symbol2@kline_1m
+        let streams: Vec<String> = symbols.iter()
+            .map(|s| format!("{}@kline_1m", s.to_lowercase()))
+            .collect();
+        let url = format!("wss://fstream.binance.com/stream?streams={}", streams.join("/"));
+
+        let (ws_stream, _) = connect_async(&url)
+            .await
+            .map_err(|e| CollectorError::ConnectionFailed(e.to_string()))?;
+
+        info!("WS {} connected with {} symbols (combined stream)", connection_id, symbols.len());
+
+        let (_, mut read) = ws_stream.split();
+
+        // Process incoming messages
+        while let Some(msg_result) = read.next().await {
+            match msg_result {
+                Ok(Message::Text(text)) => {
+                    // Combined stream format: {"stream":"btcusdt@kline_1m","data":{...}}
+                    if let Ok(wrapper) = serde_json::from_str::<CombinedStreamWrapper>(&text) {
+                        if let Some(k) = wrapper.data.k {
+                            let symbol = wrapper.data.s.unwrap_or_default();
+                            let candle = Self::parse_ws_kline(&symbol, &k);
+
+                            if let Err(e) = candle_tx.send(candle.clone()).await {
+                                warn!("Failed to send candle: {}", e);
+                            }
+
+                            if candle.is_closed {
+                                let mut timestamps = last_closed_timestamps.write().await;
+                                timestamps.insert(candle.symbol.to_lowercase(), candle.timestamp);
+
+                                debug!(
+                                    "Kline closed: {} ts={} c={:.2} net_vol={:.4}",
+                                    candle.symbol, candle.timestamp, candle.close, candle.net_volume
+                                );
+                            }
+                        }
+                    }
+                }
+                Ok(Message::Ping(data)) => {
+                    debug!("WS {} received ping", connection_id);
+                    // tungstenite auto-responds to pings
+                    let _ = data;
+                }
+                Ok(Message::Close(_)) => {
+                    info!("WS {} received close", connection_id);
+                    break;
+                }
+                Ok(_) => {}
+                Err(e) => {
+                    error!("WS {} error: {}", connection_id, e);
+                    break;
+                }
+            }
+        }
+
+        Err(CollectorError::ConnectionFailed("Disconnected".to_string()))
+    }
+
+    fn parse_rest_kline(symbol: &str, kline: &Vec<KlineCandlestickDataResponseItemInner>) -> Option<CandleData> {
+        if kline.len() < 12 {
+            return None;
+        }
+
+        let timestamp = match &kline[0] {
+            KlineCandlestickDataResponseItemInner::Integer(v) => *v,
+            _ => return None,
+        };
+
+        let open = Self::parse_string_field(&kline[1])?;
+        let high = Self::parse_string_field(&kline[2])?;
+        let low = Self::parse_string_field(&kline[3])?;
+        let close = Self::parse_string_field(&kline[4])?;
+        let volume = Self::parse_string_field(&kline[5])?;
+        let taker_buy_volume = Self::parse_string_field(&kline[9])?;
+
+        let net_volume = CandleData::calculate_net_volume(volume, taker_buy_volume);
+
+        Some(CandleData {
+            symbol: symbol.to_uppercase(),
+            timestamp,
+            open,
+            high,
+            low,
+            close,
+            volume,
+            taker_buy_volume,
+            net_volume,
+            is_closed: true,
+        })
+    }
+
+    fn parse_string_field(item: &KlineCandlestickDataResponseItemInner) -> Option<f64> {
+        match item {
+            KlineCandlestickDataResponseItemInner::String(s) => s.parse().ok(),
+            _ => None,
+        }
+    }
+
+    fn parse_ws_kline(symbol: &str, k: &KlineCandlestickStreamsResponseK) -> CandleData {
+        let timestamp = k.t.unwrap_or(0);
+        let open = k.o.as_ref().and_then(|s| s.parse().ok()).unwrap_or(0.0);
+        let high = k.h.as_ref().and_then(|s| s.parse().ok()).unwrap_or(0.0);
+        let low = k.l.as_ref().and_then(|s| s.parse().ok()).unwrap_or(0.0);
+        let close = k.c.as_ref().and_then(|s| s.parse().ok()).unwrap_or(0.0);
+        let volume = k.v.as_ref().and_then(|s| s.parse().ok()).unwrap_or(0.0);
+        let taker_buy_volume = k.v_uppercase.as_ref().and_then(|s| s.parse().ok()).unwrap_or(0.0);
+        let is_closed = k.x.unwrap_or(false);
+
+        let net_volume = CandleData::calculate_net_volume(volume, taker_buy_volume);
+
+        CandleData {
+            symbol: symbol.to_uppercase(),
+            timestamp,
+            open,
+            high,
+            low,
+            close,
+            volume,
+            taker_buy_volume,
+            net_volume,
+            is_closed,
+        }
+    }
+
+
+    /// Get all tradable symbol in current market
+    pub async fn get_symbol() -> Result<Vec<Symbol>, CollectorError> {
+
+        let port: u16 = rand::rng().random_range(10036..=10066);
+
+        let mut config_builder = ConfigurationRestApi::builder()
+            .timeout(10000);
+
+        if let Some(proxy) = proxy_config_from_env(port) {
+            config_builder = config_builder.proxy(proxy);
+        }
+
+        let rest_client_config = config_builder
+            .build()
+            .expect("Failed to initialize the rest api client");
+
+        let rest_client = DerivativesTradingUsdsFuturesRestApi::production(rest_client_config);
+
+        let mut symbol_list = Vec::new();
+
+        let response = rest_client
+            .exchange_information()
+            .await
+            .map_err(|e| CollectorError::GetSymbolError(e.to_string()))?;
+
+        let data = response.data().await.unwrap();
+
+        let symbol_vec = data.symbols.unwrap();
+
+        for item in symbol_vec {
+
+            if item.contract_type.unwrap() == "PERPETUAL" {
+
+                let symbol = Symbol { 
+
+                    symbol: item.symbol.unwrap(),
+                    start_timestamp: item.onboard_date.unwrap(),
+                };
+
+                symbol_list.push(symbol);
+            }
+        }                                            
+                        
+        Ok(symbol_list)
+    }
+
+
+    /// Build proxy list (port 10000-10099) and create clients
+    pub async fn build_clients() -> Vec<Arc<RestApi>> {
+        let mut handles = Vec::new();
+
+        let port_start: u16 = std::env::var("PROXY_PORT_START")
+            .ok().and_then(|v| v.parse().ok()).unwrap_or(10000);
+        let port_end: u16 = std::env::var("PROXY_PORT_END")
+            .ok().and_then(|v| v.parse().ok()).unwrap_or(10099);
+
+        for port in port_start..=port_end {
+            let handle = tokio::spawn(async move {
+                let mut config_builder = ConfigurationRestApi::builder()
+                    .timeout(5000);
+
+                if let Some(proxy) = proxy_config_from_env(port) {
+                    config_builder = config_builder.proxy(proxy);
+                }
+
+                let config = config_builder
+                    .build()
+                    .ok()?;
+
+                let client = DerivativesTradingUsdsFuturesRestApi::production(config);
+                
+                // Test connection
+                if client.check_server_time().await.ok()?.data().await.is_ok() {
+                    Some(Arc::new(client))
+                } else {
+                    None
+                }
+            });
+            handles.push(handle);
+        }
+
+        let mut clients = Vec::new();
+        for handle in handles {
+            if let Ok(Some(client)) = handle.await {
+                clients.push(client);
+            }
+        }
+
+        info!("Built {} working clients", clients.len());
+        clients
+    }
+
+    /// Comprehensive sync: Monthly ZIP → Daily ZIP → API (fastest to slowest)
+    pub async fn sync_comprehensive(
+        symbol: String,
+        start_time: i64,
+        api_clients: Vec<Arc<RestApi>>,
+        http_clients: &[Arc<Client>],
+        candle_tx: mpsc::Sender<CandleData>,
+    ) -> Result<u64, CollectorError> {
+        let mut total = 0u64;
+        let mut current_start = start_time;
+        
+        // Phase 1: Monthly ZIP download (complete months only)
+        let month_end = HistoricalDownloader::last_complete_month_end();
+        if current_start < month_end && !http_clients.is_empty() {
+            info!("{}: Phase 1 - Monthly ZIP download", symbol);
+            match HistoricalDownloader::download_symbol_with_clients(
+                &symbol,
+                current_start,
+                http_clients,
+                candle_tx.clone(),
+            ).await {
+                Ok(count) => {
+                    total += count;
+                    if count > 0 {
+                        current_start = month_end;
+                        info!("{}: Monthly ZIP done, {} candles", symbol, count);
+                    }
+                }
+                Err(e) => warn!("{}: Monthly ZIP failed: {}", symbol, e),
+            }
+        }
+
+        // Phase 2: Daily ZIP download (current month's completed days)
+        let now = std::time::SystemTime::now()
+            .duration_since(std::time::UNIX_EPOCH)
+            .unwrap()
+            .as_millis() as i64;
+        let today_start = (now / (24 * 60 * 60 * 1000)) * (24 * 60 * 60 * 1000);
+        
+        if current_start < today_start && !http_clients.is_empty() {
+            info!("{}: Phase 2 - Daily ZIP download", symbol);
+            match HistoricalDownloader::download_days_with_clients(
+                &symbol,
+                current_start,
+                http_clients,
+                candle_tx.clone(),
+            ).await {
+                Ok(count) => {
+                    total += count;
+                    if count > 0 {
+                        current_start = today_start;
+                        info!("{}: Daily ZIP done, {} candles", symbol, count);
+                    }
+                }
+                Err(e) => warn!("{}: Daily ZIP failed: {}", symbol, e),
+            }
+        }
+
+        // Phase 3: API for remaining (today's data)
+        if current_start < now && !api_clients.is_empty() {
+            info!("{}: Phase 3 - API sync for today", symbol);
+            match Self::sync_from_scratch(
+                symbol.clone(),
+                current_start,
+                api_clients,
+                candle_tx,
+            ).await {
+                Ok(count) => {
+                    total += count;
+                    info!("{}: API sync done, {} candles", symbol, count);
+                }
+                Err(e) => warn!("{}: API sync failed: {}", symbol, e),
+            }
+        }
+
+        info!("{}: Comprehensive sync complete, total {} candles", symbol, total);
+        Ok(total)
+    }
+
+    /// Sync single symbol using ALL clients in parallel (each client handles a time segment)
+    pub async fn sync_from_scratch(
+        symbol: String,
+        start_time: i64,
+        clients: Vec<Arc<RestApi>>,
+        candle_tx: mpsc::Sender<CandleData>,
+    ) -> Result<u64, CollectorError> {
+        let now = std::time::SystemTime::now()
+            .duration_since(std::time::UNIX_EPOCH)
+            .unwrap()
+            .as_millis() as i64;
+
+        let num_clients = clients.len();
+        if num_clients == 0 {
+            return Err(CollectorError::RestApiError("No clients".to_string()));
+        }
+
+        let total_duration = now - start_time;
+        let segment_size = total_duration / num_clients as i64;
+
+        info!(
+            "Syncing {} with {} clients, {} -> {} ({} ms per segment)",
+            symbol, num_clients, start_time, now, segment_size
+        );
+
+        // Each client gets a time segment
+        let mut handles = Vec::new();
+
+        for (i, client) in clients.into_iter().enumerate() {
+            let seg_start = start_time + (i as i64 * segment_size);
+            let seg_end = if i == num_clients - 1 {
+                now // Last segment goes to now
+            } else {
+                start_time + ((i + 1) as i64 * segment_size)
+            };
+
+            let sym = symbol.clone();
+            let tx = candle_tx.clone();
+
+            handles.push(tokio::spawn(async move {
+                Self::sync_segment(sym, seg_start, seg_end, client, tx).await
+            }));
+        }
+
+        // Wait and sum results
+        let mut total = 0u64;
+        for (i, h) in handles.into_iter().enumerate() {
+            match h.await {
+                Ok(Ok(count)) => {
+                    total += count;
+                    debug!("Client {} done: {} candles", i, count);
+                }
+                Ok(Err(e)) => error!("Client {} error: {}", i, e),
+                Err(e) => error!("Client {} join error: {}", i, e),
+            }
+        }
+
+        info!("{} sync complete: {} candles", symbol, total);
+        Ok(total)
+    }
+
+    /// Sync a specific time segment
+    async fn sync_segment(
+        symbol: String,
+        start_time: i64,
+        end_time: i64,
+        client: Arc<RestApi>,
+        candle_tx: mpsc::Sender<CandleData>,
+    ) -> Result<u64, CollectorError> {
+        let mut current = start_time;
+        let mut count = 0u64;
+
+        while current < end_time {
+            let params = KlineCandlestickDataParams::builder(
+                symbol.clone(),
+                KlineCandlestickDataIntervalEnum::Interval1m,
+            )
+            .start_time(Some(current))
+            .end_time(Some(end_time))
+            .limit(Some(BATCH_SIZE))
+            .build()
+            .map_err(|e| CollectorError::RestApiError(e.to_string()))?;
+
+            let res = client
+                .kline_candlestick_data(params)
+                .await
+                .map_err(|e| CollectorError::RestApiError(e.to_string()))?;
+
+            let klines = res.data().await
+                .map_err(|e| CollectorError::RestApiError(e.to_string()))?;
+
+            if klines.is_empty() { break; }
+
+            let batch_len = klines.len();
+            for kline in &klines {
+                if let Some(candle) = Self::parse_rest_kline(&symbol, kline) {
+                    current = candle.timestamp + ONE_MINUTE_MS;
+                    let _ = candle_tx.send(candle).await;
+                    count += 1;
+                }
+            }
+
+            if batch_len < BATCH_SIZE as usize { break; }
+            tokio::time::sleep(tokio::time::Duration::from_millis(REQUEST_INTERVAL_MS)).await;
+        }
+
+        Ok(count)
+    }
+}

backend/src/database.rs

@@ -0,0 +1,479 @@
+use crate::structs::*;
+
+use log::{debug, error, info};
+use sqlx::{PgPool, postgres::PgPoolOptions};
+use tokio::sync::mpsc;
+use tokio::time::{Duration, interval};
+
+const BUFFER_SIZE: usize = 100;
+const FLUSH_INTERVAL_SECS: u64 = 5;
+
+pub struct DatabaseHandler {
+    pool: PgPool,
+}
+
+impl DatabaseHandler {
+    pub async fn new(database_url: &str) -> Result<Self, sqlx::Error> {
+        let pool = PgPoolOptions::new()
+            .max_connections(20)
+            .min_connections(5)
+            .connect(database_url)
+            .await?;
+
+        info!("Database connected (pool: 5-20 connections)");
+
+        Ok(Self { pool })
+    }
+
+    /// Start consuming from rx and batch insert into database
+    pub async fn start_consumer(&self, mut rx: mpsc::Receiver<CandleData>) {
+        let mut buffer: Vec<CandleData> = Vec::with_capacity(BUFFER_SIZE);
+        let mut flush_timer = interval(Duration::from_secs(FLUSH_INTERVAL_SECS));
+
+        info!("Database consumer started");
+
+        loop {
+            tokio::select! {
+                // Receive candle data
+                candle = rx.recv() => {
+                    match candle {
+                        Some(c) => {
+                            buffer.push(c);
+                            if buffer.len() >= BUFFER_SIZE {
+                                self.flush_buffer(&mut buffer).await;
+                            }
+                        }
+                        None => {
+                            // Channel closed, flush remaining and exit
+                            if !buffer.is_empty() {
+                                self.flush_buffer(&mut buffer).await;
+                            }
+                            info!("Database consumer stopped (channel closed)");
+                            break;
+                        }
+                    }
+                }
+                // Flush on timer
+                _ = flush_timer.tick() => {
+                    if !buffer.is_empty() {
+                        self.flush_buffer(&mut buffer).await;
+                    }
+                }
+            }
+        }
+    }
+
+    async fn flush_buffer(&self, buffer: &mut Vec<CandleData>) {
+        if buffer.is_empty() {
+            return;
+        }
+
+        let count = buffer.len();
+        
+        match self.batch_insert(buffer).await {
+            Ok(_) => {
+                debug!("Inserted {} candles", count);
+            }
+            Err(e) => {
+                error!("Failed to insert {} candles: {}", count, e);
+            }
+        }
+        
+        buffer.clear();
+    }
+
+    async fn batch_insert(&self, candles: &[CandleData]) -> Result<(), sqlx::Error> {
+        if candles.is_empty() {
+            return Ok(());
+        }
+
+        // Deduplicate by (symbol, timestamp) - keep last occurrence
+        use std::collections::HashMap;
+        let mut dedup_map: HashMap<(String, i64), &CandleData> = HashMap::new();
+        for c in candles {
+            dedup_map.insert((c.symbol.clone(), c.timestamp), c);
+        }
+        let unique_candles: Vec<&CandleData> = dedup_map.into_values().collect();
+
+        if unique_candles.is_empty() {
+            return Ok(());
+        }
+
+        // Build batch insert query
+        let mut query = String::from(
+            "INSERT INTO klines_1m (symbol, timestamp, open, high, low, close, volume, taker_buy_volume, net_volume) VALUES "
+        );
+
+        let mut values: Vec<String> = Vec::with_capacity(unique_candles.len());
+        
+        for (i, _) in unique_candles.iter().enumerate() {
+            let idx = i * 9;
+            values.push(format!(
+                "(${}, ${}, ${}, ${}, ${}, ${}, ${}, ${}, ${})",
+                idx + 1, idx + 2, idx + 3, idx + 4, idx + 5, idx + 6, idx + 7, idx + 8, idx + 9
+            ));
+        }
+        
+        query.push_str(&values.join(", "));
+        query.push_str(" ON CONFLICT (symbol, timestamp) DO UPDATE SET open = EXCLUDED.open, high = EXCLUDED.high, low = EXCLUDED.low, close = EXCLUDED.close, volume = EXCLUDED.volume, taker_buy_volume = EXCLUDED.taker_buy_volume, net_volume = EXCLUDED.net_volume");
+
+        let mut query_builder = sqlx::query(&query);
+        
+        for c in unique_candles {
+            query_builder = query_builder
+                .bind(&c.symbol)
+                .bind(c.timestamp)
+                .bind(c.open)
+                .bind(c.high)
+                .bind(c.low)
+                .bind(c.close)
+                .bind(c.volume)
+                .bind(c.taker_buy_volume)
+                .bind(c.net_volume);
+        }
+
+        query_builder.execute(&self.pool).await?;
+        
+        Ok(())
+    }
+
+    /// Get latest timestamp for a symbol
+    pub async fn get_latest_timestamp(&self, symbol: &str) -> Result<Option<i64>, sqlx::Error> {
+        let row: Option<(Option<i64>,)> = sqlx::query_as(
+            "SELECT MAX(timestamp) FROM klines_1m WHERE symbol = $1"
+        )
+        .bind(symbol.to_uppercase())
+        .fetch_optional(&self.pool)
+        .await?;
+
+        Ok(row.and_then(|r| r.0))
+    }
+
+    /// Get today's UTC 00:00 open price for all tracked symbols
+    pub async fn get_daily_opens(&self) -> Result<std::collections::HashMap<String, f64>, sqlx::Error> {
+        use chrono::Utc;
+        let now = Utc::now();
+        let today_start = now.date_naive().and_hms_opt(0, 0, 0).unwrap().and_utc().timestamp_millis();
+        
+        let rows: Vec<(String, f64)> = sqlx::query_as(
+            "SELECT symbol, open FROM klines_1m WHERE timestamp = $1"
+        )
+        .bind(today_start)
+        .fetch_all(&self.pool)
+        .await?;
+
+        Ok(rows.into_iter().collect())
+    }
+
+    /// Get all active symbols from tracked_symbols table
+    pub async fn get_active_symbols(&self) -> Result<Vec<String>, sqlx::Error> {
+        let rows: Vec<(String,)> = sqlx::query_as(
+            "SELECT symbol FROM tracked_symbols WHERE is_active = TRUE"
+        )
+        .fetch_all(&self.pool)
+        .await?;
+
+        Ok(rows.into_iter().map(|r| r.0).collect())
+    }
+
+    /// Get tracked symbols from TRACKED_SYMBOL env variable
+    /// Format: TRACKED_SYMBOL=[BTCUSDT,ETHUSDT,BNBUSDT]
+    pub fn get_symbols_from_env() -> Vec<String> {
+        std::env::var("TRACKED_SYMBOL")
+            .unwrap_or_default()
+            .trim_start_matches('[')
+            .trim_end_matches(']')
+            .split(',')
+            .map(|s| s.trim().to_uppercase())
+            .filter(|s| !s.is_empty())
+            .collect()
+    }
+
+    /// Insert candle directly (for single inserts)
+    pub async fn insert_candle(&self, candle: &CandleData) -> Result<(), sqlx::Error> {
+        sqlx::query(
+            "INSERT INTO klines_1m (symbol, timestamp, open, high, low, close, volume, taker_buy_volume, net_volume) 
+             VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
+             ON CONFLICT (symbol, timestamp) DO UPDATE SET 
+             open = EXCLUDED.open, high = EXCLUDED.high, low = EXCLUDED.low, 
+             close = EXCLUDED.close, volume = EXCLUDED.volume, 
+             taker_buy_volume = EXCLUDED.taker_buy_volume, net_volume = EXCLUDED.net_volume"
+        )
+        .bind(&candle.symbol)
+        .bind(candle.timestamp)
+        .bind(candle.open)
+        .bind(candle.high)
+        .bind(candle.low)
+        .bind(candle.close)
+        .bind(candle.volume)
+        .bind(candle.taker_buy_volume)
+        .bind(candle.net_volume)
+        .execute(&self.pool)
+        .await?;
+
+        Ok(())
+    }
+
+    /// Add a new symbol to tracked_symbols
+    pub async fn add_symbol(&self, symbol: &str) -> Result<(), sqlx::Error> {
+        let symbol_lower = symbol.to_lowercase();
+        
+        sqlx::query(
+            "INSERT INTO tracked_symbols (symbol, is_active) 
+             VALUES ($1, TRUE)
+             ON CONFLICT (symbol) DO UPDATE SET is_active = TRUE"
+        )
+        .bind(&symbol_lower)
+        .execute(&self.pool)
+        .await?;
+
+        info!("Added symbol to tracking: {}", symbol_lower);
+        Ok(())
+    }
+
+    /// Remove (deactivate) a symbol from tracking
+    pub async fn remove_symbol(&self, symbol: &str) -> Result<(), sqlx::Error> {
+        let symbol_lower = symbol.to_lowercase();
+        
+        sqlx::query(
+            "UPDATE tracked_symbols SET is_active = FALSE WHERE symbol = $1"
+        )
+        .bind(&symbol_lower)
+        .execute(&self.pool)
+        .await?;
+
+        info!("Removed symbol from tracking: {}", symbol_lower);
+        Ok(())
+    }
+
+    /// Check if a symbol is currently being tracked
+    pub async fn is_symbol_tracked(&self, symbol: &str) -> Result<bool, sqlx::Error> {
+        let symbol_lower = symbol.to_lowercase();
+        
+        let row: Option<(bool,)> = sqlx::query_as(
+            "SELECT is_active FROM tracked_symbols WHERE symbol = $1"
+        )
+        .bind(&symbol_lower)
+        .fetch_optional(&self.pool)
+        .await?;
+
+        Ok(row.map(|r| r.0).unwrap_or(false))
+    }
+
+    /// Delete all kline data for a symbol (use with caution)
+    pub async fn delete_symbol_data(&self, symbol: &str) -> Result<u64, sqlx::Error> {
+        let symbol_upper = symbol.to_uppercase();
+        
+        let result = sqlx::query(
+            "DELETE FROM klines_1m WHERE symbol = $1"
+        )
+        .bind(&symbol_upper)
+        .execute(&self.pool)
+        .await?;
+
+        let deleted = result.rows_affected();
+        info!("Deleted {} candles for symbol: {}", deleted, symbol_upper);
+        Ok(deleted)
+    }
+
+    /// Get klines for a symbol with limit and optional end_time
+    pub async fn get_klines(&self, symbol: &str, limit: i64, end_time: Option<i64>) -> Result<Vec<CandleData>, sqlx::Error> {
+        let symbol_upper = symbol.to_uppercase();
+        
+        let rows: Vec<(String, i64, f64, f64, f64, f64, f64, f64, f64)> = if let Some(et) = end_time {
+            sqlx::query_as(
+                "SELECT symbol, timestamp, open, high, low, close, volume, taker_buy_volume, net_volume 
+                 FROM klines_1m 
+                 WHERE symbol = $1 AND timestamp <= $3
+                 ORDER BY timestamp DESC 
+                 LIMIT $2"
+            )
+            .bind(&symbol_upper)
+            .bind(limit)
+            .bind(et)
+            .fetch_all(&self.pool)
+            .await?
+        } else {
+            sqlx::query_as(
+                "SELECT symbol, timestamp, open, high, low, close, volume, taker_buy_volume, net_volume 
+                 FROM klines_1m 
+                 WHERE symbol = $1 
+                 ORDER BY timestamp DESC 
+                 LIMIT $2"
+            )
+            .bind(&symbol_upper)
+            .bind(limit)
+            .fetch_all(&self.pool)
+            .await?
+        };
+
+        let mut candles: Vec<CandleData> = rows.into_iter().map(|r| CandleData {
+            symbol: r.0,
+            timestamp: r.1,
+            open: r.2,
+            high: r.3,
+            low: r.4,
+            close: r.5,
+            volume: r.6,
+            taker_buy_volume: r.7,
+            net_volume: r.8,
+            is_closed: true,
+        }).collect();
+
+        // Reverse to chronological order (oldest first)
+        candles.reverse();
+        
+        Ok(candles)
+    }
+
+    /// Get klines aggregated to a specific interval using TimescaleDB time_bucket
+    pub async fn get_klines_aggregated(
+        &self, 
+        symbol: &str, 
+        interval: crate::Interval, 
+        limit: i64,
+        end_time: Option<i64>
+    ) -> Result<Vec<CandleData>, sqlx::Error> {
+        use crate::Interval;
+        
+        // For 1m interval, just return raw data
+        if interval == Interval::Min1 {
+            return self.get_klines(symbol, limit, end_time).await;
+        }
+
+        let symbol_upper = symbol.to_uppercase();
+        
+        // Calculate interval in milliseconds for time range filtering
+        let interval_ms: i64 = match interval {
+            Interval::Min1 => 60_000,
+            Interval::Min5 => 5 * 60_000,
+            Interval::Min15 => 15 * 60_000,
+            Interval::Hour1 => 60 * 60_000,
+            Interval::Hour4 => 4 * 60 * 60_000,
+            Interval::Day1 => 24 * 60 * 60_000,
+            Interval::Week1 => 7 * 24 * 60 * 60_000,
+            Interval::Month1 => 30 * 24 * 60 * 60_000,
+        };
+        
+        // Convert interval to PostgreSQL interval string
+        let interval_str = match interval {
+            Interval::Min1 => "1 minute",
+            Interval::Min5 => "5 minutes",
+            Interval::Min15 => "15 minutes",
+            Interval::Hour1 => "1 hour",
+            Interval::Hour4 => "4 hours",
+            Interval::Day1 => "1 day",
+            Interval::Week1 => "1 week",
+            Interval::Month1 => "1 month",
+        };
+
+        // Calculate time range to limit scan (add 10% buffer)
+        let et = end_time.unwrap_or_else(|| chrono::Utc::now().timestamp_millis());
+        let time_range_needed = interval_ms * limit * 11 / 10;
+        let start_time = et - time_range_needed;
+
+        // Use TimescaleDB time_bucket for efficient aggregation with time range filter
+        let query = format!(
+            r#"
+            SELECT 
+                $1 as symbol,
+                (EXTRACT(EPOCH FROM time_bucket('{interval}', to_timestamp(timestamp/1000.0))) * 1000)::bigint as bucket_ts,
+                (array_agg(open ORDER BY timestamp ASC))[1] as open,
+                max(high) as high,
+                min(low) as low,
+                (array_agg(close ORDER BY timestamp DESC))[1] as close,
+                sum(volume) as volume,
+                sum(taker_buy_volume) as taker_buy_volume,
+                sum(net_volume) as net_volume
+            FROM klines_1m
+            WHERE symbol = $1 AND timestamp >= $3 AND timestamp <= $4
+            GROUP BY bucket_ts
+            ORDER BY bucket_ts DESC
+            LIMIT $2
+            "#,
+            interval = interval_str,
+        );
+
+        let rows: Vec<(String, i64, f64, f64, f64, f64, f64, f64, f64)> = sqlx::query_as(&query)
+            .bind(&symbol_upper)
+            .bind(limit)
+            .bind(start_time)
+            .bind(et)
+            .fetch_all(&self.pool)
+            .await?;
+
+        let mut candles: Vec<CandleData> = rows.into_iter().map(|r| CandleData {
+            symbol: r.0,
+            timestamp: r.1,
+            open: r.2,
+            high: r.3,
+            low: r.4,
+            close: r.5,
+            volume: r.6,
+            taker_buy_volume: r.7,
+            net_volume: r.8,
+            is_closed: true,
+        }).collect();
+
+        // Reverse to chronological order (oldest first)
+        candles.reverse();
+
+        Ok(candles)
+    }
+
+    /// Find gaps in 1-minute kline data for a symbol.
+    /// Returns a list of (start_timestamp, end_timestamp) pairs representing gaps.
+    /// Each gap represents missing data from start_timestamp to end_timestamp (exclusive).
+    /// Only considers data after DATA_CUTOFF_TIMESTAMP (2024-01-01).
+    pub async fn find_gaps(&self, symbol: &str, cutoff_timestamp: i64) -> Result<Vec<(i64, i64)>, sqlx::Error> {
+        let symbol_upper = symbol.to_uppercase();
+
+        // Query to find gaps using window function
+        // We look for cases where the next timestamp is more than 1 minute away
+        // Only consider data after cutoff_timestamp
+        let rows: Vec<(i64, i64)> = sqlx::query_as(
+            r#"
+            WITH ordered_klines AS (
+                SELECT timestamp,
+                       LEAD(timestamp) OVER (ORDER BY timestamp) as next_timestamp
+                FROM klines_1m
+                WHERE symbol = $1 AND timestamp >= $2
+            )
+            SELECT timestamp + 60000 as gap_start, next_timestamp as gap_end
+            FROM ordered_klines
+            WHERE next_timestamp - timestamp > 60000
+            ORDER BY timestamp
+            "#
+        )
+        .bind(&symbol_upper)
+        .bind(cutoff_timestamp)
+        .fetch_all(&self.pool)
+        .await?;
+
+        Ok(rows)
+    }
+
+    /// Get the earliest timestamp for a symbol
+    pub async fn get_earliest_timestamp(&self, symbol: &str) -> Result<Option<i64>, sqlx::Error> {
+        let row: Option<(Option<i64>,)> = sqlx::query_as(
+            "SELECT MIN(timestamp) FROM klines_1m WHERE symbol = $1"
+        )
+        .bind(symbol.to_uppercase())
+        .fetch_optional(&self.pool)
+        .await?;
+
+        Ok(row.and_then(|r| r.0))
+    }
+
+    /// Get count of klines for a symbol
+    pub async fn get_kline_count(&self, symbol: &str) -> Result<i64, sqlx::Error> {
+        let row: (i64,) = sqlx::query_as(
+            "SELECT COUNT(*) FROM klines_1m WHERE symbol = $1"
+        )
+        .bind(symbol.to_uppercase())
+        .fetch_one(&self.pool)
+        .await?;
+
+        Ok(row.0)
+    }
+}

backend/src/error.rs

@@ -0,0 +1,47 @@
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum CollectorError {
+    #[error("WebSocket connection failed: {0}")]
+    ConnectionFailed(String),
+
+    #[error("REST API request failed: {0}")]
+    RestApiError(String),
+
+    #[error("Data parsing error: {0}")]
+    ParseError(String),
+
+    #[error("Invalid kline data: {0}")]
+    InvalidKlineData(String),
+
+    #[error("Failed to get opened symbol, with error {0}")]
+    GetSymbolError(String),
+    
+}
+
+#[derive(Error, Debug)]
+pub enum SchedulerError {
+    #[error("Database error: {0}")]
+    DatabaseError(String),
+
+    #[error("Collector error: {0}")]
+    CollectorError(String),
+
+    #[error("Backfill error: {0}")]
+    BackfillError(String),
+
+    #[error("No active symbols")]
+    NoActiveSymbols,
+}
+
+impl From<sqlx::Error> for SchedulerError {
+    fn from(err: sqlx::Error) -> Self {
+        SchedulerError::DatabaseError(err.to_string())
+    }
+}
+
+impl From<CollectorError> for SchedulerError {
+    fn from(err: CollectorError) -> Self {
+        SchedulerError::CollectorError(err.to_string())
+    }
+}

backend/src/historical_downloader.rs

@@ -0,0 +1,522 @@
+use crate::structs::CandleData;
+use crate::error::CollectorError;
+
+use chrono::{Datelike, NaiveDate, Utc};
+use log::{info, warn};
+use std::io::{Cursor, Read};
+use std::sync::Arc;
+use tokio::sync::mpsc;
+use futures::future::join_all;
+use reqwest::Client;
+
+const BASE_URL: &str = "https://data.binance.vision/data/futures/um/monthly/klines";
+const DAILY_BASE_URL: &str = "https://data.binance.vision/data/futures/um/daily/klines";
+
+/// Download historical klines from Binance data archive
+pub struct HistoricalDownloader;
+
+impl HistoricalDownloader {
+    /// Build proxy clients from env vars (PROXY_HOST, PROXY_USERNAME, PROXY_PASSWORD, PROXY_PROTOCOL)
+    /// Port range defaults to PROXY_PORT_START..PROXY_PORT_END (default 10000..10099)
+    /// Returns direct (no-proxy) clients if PROXY_HOST is not set.
+    pub async fn build_clients() -> Vec<Arc<Client>> {
+        let mut handles = Vec::new();
+
+        let proxy_host = std::env::var("PROXY_HOST").unwrap_or_default();
+        let proxy_username = std::env::var("PROXY_USERNAME").unwrap_or_default();
+        let proxy_password = std::env::var("PROXY_PASSWORD").unwrap_or_default();
+        let proxy_protocol = std::env::var("PROXY_PROTOCOL").unwrap_or_else(|_| "https".to_string());
+        let port_start: u16 = std::env::var("PROXY_PORT_START")
+            .ok().and_then(|v| v.parse().ok()).unwrap_or(10000);
+        let port_end: u16 = std::env::var("PROXY_PORT_END")
+            .ok().and_then(|v| v.parse().ok()).unwrap_or(10099);
+
+        if proxy_host.is_empty() {
+            // No proxy configured — return a single direct client
+            info!("No proxy configured (PROXY_HOST not set), using direct connection");
+            if let Ok(client) = Client::builder()
+                .timeout(std::time::Duration::from_secs(60))
+                .build()
+            {
+                return vec![Arc::new(client)];
+            }
+            return vec![];
+        }
+
+        for port in port_start..=port_end {
+            let host = proxy_host.clone();
+            let user = proxy_username.clone();
+            let pass = proxy_password.clone();
+            let proto = proxy_protocol.clone();
+
+            handles.push(tokio::spawn(async move {
+                let proxy_url = if user.is_empty() {
+                    format!("{}://{}:{}", proto, host, port)
+                } else {
+                    format!("{}://{}:{}@{}:{}", proto, user, pass, host, port)
+                };
+
+                match reqwest::Proxy::all(&proxy_url) {
+                    Ok(proxy) => {
+                        match Client::builder()
+                            .proxy(proxy)
+                            .timeout(std::time::Duration::from_secs(60))
+                            .build()
+                        {
+                            Ok(client) => Some(Arc::new(client)),
+                            Err(_) => None,
+                        }
+                    }
+                    Err(_) => None,
+                }
+            }));
+        }
+        
+        let mut clients = Vec::new();
+        for handle in handles {
+            if let Ok(Some(client)) = handle.await {
+                clients.push(client);
+            }
+        }
+        
+        info!("Built {} download clients with proxies", clients.len());
+        clients
+    }
+
+    /// Download all monthly klines for a symbol using proxy clients
+    pub async fn download_symbol_with_clients(
+        symbol: &str,
+        start_timestamp: i64,
+        clients: &[Arc<Client>],
+        candle_tx: mpsc::Sender<CandleData>,
+    ) -> Result<u64, CollectorError> {
+        let symbol_upper = symbol.to_uppercase();
+        
+        if clients.is_empty() {
+            return Err(CollectorError::RestApiError("No clients available".to_string()));
+        }
+        
+        // Calculate start and end months
+        let start_date = Self::timestamp_to_date(start_timestamp);
+        let now = Utc::now().naive_utc().date();
+        
+        // Generate list of months to download (exclude current month - incomplete)
+        let months = Self::generate_months(start_date, now);
+        
+        if months.is_empty() {
+            info!("{}: No complete months to download", symbol_upper);
+            return Ok(0);
+        }
+        
+        info!(
+            "{}: Downloading {} months ({}-{:02} to {}-{:02}) with {} clients",
+            symbol_upper,
+            months.len(),
+            months.first().unwrap().0, months.first().unwrap().1,
+            months.last().unwrap().0, months.last().unwrap().1,
+            clients.len()
+        );
+        
+        // Download all months in parallel using all clients
+        let mut handles = Vec::new();
+        
+        for (i, &(year, month)) in months.iter().enumerate() {
+            let client = clients[i % clients.len()].clone();
+            let sym = symbol_upper.clone();
+            let tx = candle_tx.clone();
+            
+            handles.push(tokio::spawn(async move {
+                Self::download_month_with_client(&sym, year, month, &client, tx).await
+            }));
+        }
+        
+        // Wait for all downloads
+        let results = join_all(handles).await;
+        
+        let mut total_candles = 0u64;
+        for result in results {
+            match result {
+                Ok(Ok(count)) => total_candles += count,
+                Ok(Err(e)) => warn!("Download error: {}", e),
+                Err(e) => warn!("Join error: {}", e),
+            }
+        }
+        
+        info!("{}: Downloaded {} candles from archive", symbol_upper, total_candles);
+        Ok(total_candles)
+    }
+
+    /// Download a single month's klines using a specific client
+    async fn download_month_with_client(
+        symbol: &str,
+        year: i32,
+        month: u32,
+        client: &Client,
+        candle_tx: mpsc::Sender<CandleData>,
+    ) -> Result<u64, CollectorError> {
+        let url = format!(
+            "{}/{}/1m/{}-1m-{}-{:02}.zip",
+            BASE_URL, symbol, symbol, year, month
+        );
+        
+        // Download ZIP file with proxy client
+        let response = client.get(&url)
+            .send()
+            .await
+            .map_err(|e| CollectorError::RestApiError(format!("Download failed: {}", e)))?;
+        
+        if !response.status().is_success() {
+            return Err(CollectorError::RestApiError(
+                format!("HTTP {}: {}", response.status(), url)
+            ));
+        }
+        
+        let bytes = response.bytes()
+            .await
+            .map_err(|e| CollectorError::RestApiError(format!("Read bytes failed: {}", e)))?;
+        
+        // Extract CSV from ZIP
+        let cursor = Cursor::new(bytes);
+        let mut archive = zip::ZipArchive::new(cursor)
+            .map_err(|e| CollectorError::RestApiError(format!("ZIP error: {}", e)))?;
+        
+        let mut csv_content = String::new();
+        {
+            let mut file = archive.by_index(0)
+                .map_err(|e| CollectorError::RestApiError(format!("ZIP file error: {}", e)))?;
+            file.read_to_string(&mut csv_content)
+                .map_err(|e| CollectorError::RestApiError(format!("Read CSV error: {}", e)))?;
+        }
+        
+        // Parse CSV and send candles
+        let count = Self::parse_csv(symbol, &csv_content, &candle_tx).await?;
+        
+        info!("{} {}-{:02}: {} candles", symbol, year, month, count);
+        Ok(count)
+    }
+    
+    /// Download all monthly klines (without proxy - original method)
+    pub async fn download_symbol(
+        symbol: &str,
+        start_timestamp: i64,
+        candle_tx: mpsc::Sender<CandleData>,
+        max_parallel: usize,
+    ) -> Result<u64, CollectorError> {
+        let symbol_upper = symbol.to_uppercase();
+        
+        let start_date = Self::timestamp_to_date(start_timestamp);
+        let now = Utc::now().naive_utc().date();
+        let months = Self::generate_months(start_date, now);
+        
+        if months.is_empty() {
+            info!("{}: No complete months to download", symbol_upper);
+            return Ok(0);
+        }
+        
+        info!(
+            "{}: Downloading {} months from {}-{:02} to {}-{:02}",
+            symbol_upper,
+            months.len(),
+            months.first().unwrap().0, months.first().unwrap().1,
+            months.last().unwrap().0, months.last().unwrap().1
+        );
+        
+        let mut total_candles = 0u64;
+        
+        for chunk in months.chunks(max_parallel) {
+            let mut handles = Vec::new();
+            
+            for &(year, month) in chunk {
+                let sym = symbol_upper.clone();
+                let tx = candle_tx.clone();
+                
+                handles.push(tokio::spawn(async move {
+                    Self::download_month(&sym, year, month, tx).await
+                }));
+            }
+            
+            let results = join_all(handles).await;
+            
+            for result in results {
+                match result {
+                    Ok(Ok(count)) => total_candles += count,
+                    Ok(Err(e)) => warn!("Download error: {}", e),
+                    Err(e) => warn!("Join error: {}", e),
+                }
+            }
+        }
+        
+        info!("{}: Downloaded {} candles from archive", symbol_upper, total_candles);
+        Ok(total_candles)
+    }
+
+    /// Download a single month (without proxy)
+    async fn download_month(
+        symbol: &str,
+        year: i32,
+        month: u32,
+        candle_tx: mpsc::Sender<CandleData>,
+    ) -> Result<u64, CollectorError> {
+        let url = format!(
+            "{}/{}/1m/{}-1m-{}-{:02}.zip",
+            BASE_URL, symbol, symbol, year, month
+        );
+        
+        let response = reqwest::get(&url)
+            .await
+            .map_err(|e| CollectorError::RestApiError(format!("Download failed: {}", e)))?;
+        
+        if !response.status().is_success() {
+            return Err(CollectorError::RestApiError(
+                format!("HTTP {}: {}", response.status(), url)
+            ));
+        }
+        
+        let bytes = response.bytes()
+            .await
+            .map_err(|e| CollectorError::RestApiError(format!("Read bytes failed: {}", e)))?;
+        
+        let cursor = Cursor::new(bytes);
+        let mut archive = zip::ZipArchive::new(cursor)
+            .map_err(|e| CollectorError::RestApiError(format!("ZIP error: {}", e)))?;
+        
+        let mut csv_content = String::new();
+        {
+            let mut file = archive.by_index(0)
+                .map_err(|e| CollectorError::RestApiError(format!("ZIP file error: {}", e)))?;
+            file.read_to_string(&mut csv_content)
+                .map_err(|e| CollectorError::RestApiError(format!("Read CSV error: {}", e)))?;
+        }
+        
+        let count = Self::parse_csv(symbol, &csv_content, &candle_tx).await?;
+        
+        info!("{} {}-{:02}: {} candles", symbol, year, month, count);
+        Ok(count)
+    }
+
+    /// Parse CSV content and send candles through channel
+    async fn parse_csv(
+        symbol: &str,
+        csv_content: &str,
+        candle_tx: &mpsc::Sender<CandleData>,
+    ) -> Result<u64, CollectorError> {
+        let mut reader = csv::ReaderBuilder::new()
+            .has_headers(true)
+            .from_reader(csv_content.as_bytes());
+        
+        let mut count = 0u64;
+        
+        for result in reader.records() {
+            let record = match result {
+                Ok(r) => r,
+                Err(_) => continue,
+            };
+            
+            if record.len() < 11 {
+                continue;
+            }
+            
+            let timestamp: i64 = record[0].parse().unwrap_or(0);
+            let open: f64 = record[1].parse().unwrap_or(0.0);
+            let high: f64 = record[2].parse().unwrap_or(0.0);
+            let low: f64 = record[3].parse().unwrap_or(0.0);
+            let close: f64 = record[4].parse().unwrap_or(0.0);
+            let volume: f64 = record[5].parse().unwrap_or(0.0);
+            let taker_buy_volume: f64 = record[9].parse().unwrap_or(0.0);
+            
+            let net_volume = taker_buy_volume * 2.0 - volume;
+            
+            let candle = CandleData {
+                symbol: symbol.to_string(),
+                timestamp,
+                open,
+                high,
+                low,
+                close,
+                volume,
+                taker_buy_volume,
+                net_volume,
+                is_closed: true,
+            };
+            
+            if candle_tx.send(candle).await.is_err() {
+                break;
+            }
+            count += 1;
+        }
+        
+        Ok(count)
+    }
+    
+    fn timestamp_to_date(ts: i64) -> NaiveDate {
+        let secs = ts / 1000;
+        chrono::DateTime::from_timestamp(secs, 0)
+            .unwrap_or_else(|| Utc::now())
+            .naive_utc()
+            .date()
+    }
+    
+    fn generate_months(start: NaiveDate, end: NaiveDate) -> Vec<(i32, u32)> {
+        let mut months = Vec::new();
+        
+        let mut year = start.year();
+        let mut month = start.month();
+        
+        let end_year = if end.month() == 1 { end.year() - 1 } else { end.year() };
+        let end_month = if end.month() == 1 { 12 } else { end.month() - 1 };
+        
+        loop {
+            if year > end_year || (year == end_year && month > end_month) {
+                break;
+            }
+            
+            months.push((year, month));
+            
+            month += 1;
+            if month > 12 {
+                month = 1;
+                year += 1;
+            }
+        }
+        
+        months
+    }
+    
+    pub fn last_complete_month_end() -> i64 {
+        let now = Utc::now().naive_utc();
+        let first_of_this_month = NaiveDate::from_ymd_opt(now.year(), now.month(), 1)
+            .unwrap()
+            .and_hms_opt(0, 0, 0)
+            .unwrap();
+        first_of_this_month.and_utc().timestamp_millis()
+    }
+
+    /// Download daily klines for a symbol (for current incomplete month)
+    pub async fn download_days_with_clients(
+        symbol: &str,
+        start_timestamp: i64,
+        clients: &[Arc<Client>],
+        candle_tx: mpsc::Sender<CandleData>,
+    ) -> Result<u64, CollectorError> {
+        let symbol_upper = symbol.to_uppercase();
+        
+        if clients.is_empty() {
+            return Err(CollectorError::RestApiError("No clients available".to_string()));
+        }
+        
+        let start_date = Self::timestamp_to_date(start_timestamp);
+        let yesterday = Utc::now().naive_utc().date().pred_opt().unwrap_or(start_date);
+        
+        let days = Self::generate_days(start_date, yesterday);
+        
+        if days.is_empty() {
+            info!("{}: No days to download", symbol_upper);
+            return Ok(0);
+        }
+        
+        info!(
+            "{}: Downloading {} days ({} to {}) with {} clients",
+            symbol_upper, days.len(),
+            days.first().unwrap(), days.last().unwrap(),
+            clients.len()
+        );
+        
+        let mut handles = Vec::new();
+        
+        for (i, date) in days.iter().enumerate() {
+            let client = clients[i % clients.len()].clone();
+            let sym = symbol_upper.clone();
+            let tx = candle_tx.clone();
+            let d = *date;
+            
+            handles.push(tokio::spawn(async move {
+                Self::download_day_with_client(&sym, d, &client, tx).await
+            }));
+        }
+        
+        let results = join_all(handles).await;
+        
+        let mut total_candles = 0u64;
+        for result in results {
+            match result {
+                Ok(Ok(count)) => total_candles += count,
+                Ok(Err(e)) => warn!("Daily download error: {}", e),
+                Err(e) => warn!("Join error: {}", e),
+            }
+        }
+        
+        info!("{}: Downloaded {} candles from daily archive", symbol_upper, total_candles);
+        Ok(total_candles)
+    }
+
+    /// Download a single day's klines
+    async fn download_day_with_client(
+        symbol: &str,
+        date: NaiveDate,
+        client: &Client,
+        candle_tx: mpsc::Sender<CandleData>,
+    ) -> Result<u64, CollectorError> {
+        let url = format!(
+            "{}/{}/1m/{}-1m-{}.zip",
+            DAILY_BASE_URL, symbol, symbol, date.format("%Y-%m-%d")
+        );
+        
+        let response = client.get(&url)
+            .send()
+            .await
+            .map_err(|e| CollectorError::RestApiError(format!("Download failed: {}", e)))?;
+        
+        if !response.status().is_success() {
+            return Err(CollectorError::RestApiError(
+                format!("HTTP {}: {}", response.status(), url)
+            ));
+        }
+        
+        let bytes = response.bytes()
+            .await
+            .map_err(|e| CollectorError::RestApiError(format!("Read bytes failed: {}", e)))?;
+        
+        let cursor = Cursor::new(bytes);
+        let mut archive = zip::ZipArchive::new(cursor)
+            .map_err(|e| CollectorError::RestApiError(format!("ZIP error: {}", e)))?;
+        
+        let mut csv_content = String::new();
+        {
+            let mut file = archive.by_index(0)
+                .map_err(|e| CollectorError::RestApiError(format!("ZIP file error: {}", e)))?;
+            file.read_to_string(&mut csv_content)
+                .map_err(|e| CollectorError::RestApiError(format!("Read CSV error: {}", e)))?;
+        }
+        
+        let count = Self::parse_csv(symbol, &csv_content, &candle_tx).await?;
+        
+        info!("{} {}: {} candles", symbol, date, count);
+        Ok(count)
+    }
+
+    /// Generate list of days between start and end (inclusive)
+    fn generate_days(start: NaiveDate, end: NaiveDate) -> Vec<NaiveDate> {
+        let mut days = Vec::new();
+        let mut current = start;
+        
+        while current <= end {
+            days.push(current);
+            current = current.succ_opt().unwrap_or(end);
+            if current == end && days.last() == Some(&end) {
+                break;
+            }
+        }
+        
+        days
+    }
+
+    /// Get timestamp for start of current month (for daily download start point)
+    pub fn current_month_start() -> i64 {
+        let now = Utc::now().naive_utc();
+        let first_of_this_month = NaiveDate::from_ymd_opt(now.year(), now.month(), 1)
+            .unwrap()
+            .and_hms_opt(0, 0, 0)
+            .unwrap();
+        first_of_this_month.and_utc().timestamp_millis()
+    }
+}

backend/src/klinechart.rs

@@ -0,0 +1,103 @@
+use crate::{
+    DatabaseHandler, SchedulerCommand,
+    CandleData, KlineQuery, ApiResponse, AddSymbolRequest, SchedulerStatus,
+};
+
+use axum::{
+    Router,
+    routing::{get, post, delete},
+    extract::{Path, Query, State},
+    response::Json,
+    http::StatusCode,
+};
+use std::sync::Arc;
+use tokio::sync::{mpsc, oneshot};
+
+pub struct KlineChartState {
+    pub db: Arc<DatabaseHandler>,
+    pub command_tx: mpsc::Sender<SchedulerCommand>,
+}
+
+pub fn klinechart_routes() -> Router<Arc<KlineChartState>> {
+    Router::new()
+        .route("/api/klines/{symbol}", get(get_klines))
+        .route("/api/symbols", get(get_symbols))
+        .route("/api/symbols", post(add_symbol))
+        .route("/api/symbols/{symbol}", delete(remove_symbol))
+        .route("/api/status", get(get_status))
+}
+
+// GET /api/klines/{symbol}?limit=800&interval=1m&end_time=...
+async fn get_klines(
+    State(state): State<Arc<KlineChartState>>,
+    Path(symbol): Path<String>,
+    Query(query): Query<KlineQuery>,
+) -> Json<ApiResponse<Vec<CandleData>>> {
+    let limit = query.limit.unwrap_or(800);
+    let interval = query.interval.unwrap_or_default();
+    let end_time = query.end_time;
+    
+    match state.db.get_klines_aggregated(&symbol, interval, limit, end_time).await {
+        Ok(candles) => Json(ApiResponse::ok(candles)),
+        Err(e) => Json(ApiResponse::err(&e.to_string())),
+    }
+}
+
+// GET /api/symbols
+async fn get_symbols(
+    State(state): State<Arc<KlineChartState>>,
+) -> Json<ApiResponse<Vec<String>>> {
+    match state.db.get_active_symbols().await {
+        Ok(symbols) => Json(ApiResponse::ok(symbols)),
+        Err(e) => Json(ApiResponse::err(&e.to_string())),
+    }
+}
+
+// POST /api/symbols
+async fn add_symbol(
+    State(state): State<Arc<KlineChartState>>,
+    Json(req): Json<AddSymbolRequest>,
+) -> (StatusCode, Json<ApiResponse<String>>) {
+    let result = state.command_tx.send(SchedulerCommand::AddSymbol {
+        symbol: req.symbol.clone(),
+        backfill_from: req.backfill_from,
+    }).await;
+
+    match result {
+        Ok(_) => (StatusCode::OK, Json(ApiResponse::ok(format!("Adding symbol: {}", req.symbol)))),
+        Err(e) => (StatusCode::INTERNAL_SERVER_ERROR, Json(ApiResponse::err(&e.to_string()))),
+    }
+}
+
+// DELETE /api/symbols/{symbol}
+async fn remove_symbol(
+    State(state): State<Arc<KlineChartState>>,
+    Path(symbol): Path<String>,
+) -> (StatusCode, Json<ApiResponse<String>>) {
+    let result = state.command_tx.send(SchedulerCommand::RemoveSymbol {
+        symbol: symbol.clone(),
+    }).await;
+
+    match result {
+        Ok(_) => (StatusCode::OK, Json(ApiResponse::ok(format!("Removing symbol: {}", symbol)))),
+        Err(e) => (StatusCode::INTERNAL_SERVER_ERROR, Json(ApiResponse::err(&e.to_string()))),
+    }
+}
+
+// GET /api/status
+async fn get_status(
+    State(state): State<Arc<KlineChartState>>,
+) -> Json<ApiResponse<SchedulerStatus>> {
+    let (reply_tx, reply_rx) = oneshot::channel();
+    
+    let send_result = state.command_tx.send(SchedulerCommand::GetStatus { reply: reply_tx }).await;
+    
+    if send_result.is_err() {
+        return Json(ApiResponse::err("Failed to send status request"));
+    }
+
+    match reply_rx.await {
+        Ok(status) => Json(ApiResponse::ok(status)),
+        Err(_) => Json(ApiResponse::err("Failed to receive status")),
+    }
+}

backend/src/lib.rs

@@ -0,0 +1,23 @@
+pub mod structs;
+pub use structs::*;
+
+pub mod error;
+pub use error::*;
+
+pub mod binance_collector;
+pub use binance_collector::*;
+
+pub mod database;
+pub use database::*;
+
+pub mod scheduler;
+pub use scheduler::*;
+
+pub mod klinechart;
+pub use klinechart::*;
+
+pub mod tradingview;
+pub use tradingview::*;
+
+pub mod historical_downloader;
+pub use historical_downloader::*;

backend/src/main.rs

@@ -0,0 +1,83 @@
+use backend::{
+    DatabaseHandler, Scheduler, create_command_channel,
+    KlineChartState, klinechart_routes,
+    TradingViewState, tradingview_routes,
+    CandleData,
+};
+
+use axum::Router;
+use tower_http::cors::{CorsLayer, Any};
+use axum::http::Method;
+use std::sync::Arc;
+use log::info;
+use dotenv::*;
+use tokio::sync::mpsc;
+
+#[tokio::main]
+async fn main() {
+
+    dotenv().ok();
+    env_logger::init();
+
+    info!("Current Version 1.4");
+
+    let database_url = std::env::var("DATABASE_URL")
+        .unwrap_or_else(|_| "postgres://quant:2Nr!Ya&oVvY5pp@172.18.0.2:5432/crypto_database".to_string());
+
+    info!("Connecting to database...");
+    let db = Arc::new(
+        DatabaseHandler::new(&database_url)
+            .await
+            .expect("Failed to connect to database")
+    );
+
+    let (command_tx, command_rx) = create_command_channel();
+
+    // Create TradingView state with broadcast channel
+    let (tradingview_state, _) = TradingViewState::new(db.clone());
+    let tradingview_state = Arc::new(tradingview_state);
+
+    // Create channel for WebSocket broadcasts
+    let (ws_broadcast_tx, mut ws_broadcast_rx) = mpsc::channel::<CandleData>(10000);
+
+    // Forward candles to TradingView broadcast
+    let tv_state_clone = tradingview_state.clone();
+    tokio::spawn(async move {
+        while let Some(candle) = ws_broadcast_rx.recv().await {
+            tv_state_clone.broadcast_candle(candle);
+        }
+    });
+
+    // Start scheduler in background with ws_broadcast_tx
+    let scheduler_db = db.clone();
+    tokio::spawn(async move {
+        let mut scheduler = Scheduler::new(scheduler_db, command_rx, Some(ws_broadcast_tx));
+        scheduler.run().await;
+    });
+
+    // KlineChart state (for klinechart frontend)
+    let klinechart_state = Arc::new(KlineChartState {
+        db: db.clone(),
+        command_tx,
+    });
+
+    // CORS for frontend
+    let cors = CorsLayer::new()
+        .allow_origin(Any)
+        .allow_methods([Method::GET, Method::POST, Method::DELETE, Method::OPTIONS])
+        .allow_headers(Any);
+
+    // Merge routes from both modules
+    let app = Router::new()
+        .merge(klinechart_routes().with_state(klinechart_state))
+        .merge(tradingview_routes().with_state(tradingview_state))
+        .layer(cors);
+
+    let addr = "0.0.0.0:3000";
+    info!("Starting API server on {}", addr);
+    info!("KlineChart API: /api/klines, /api/symbols, /api/status");
+    info!("TradingView UDF: /config, /symbols, /search, /history, /time");
+    
+    let listener = tokio::net::TcpListener::bind(addr).await.unwrap();
+    axum::serve(listener, app).await.unwrap();
+}

backend/src/scheduler.rs

@@ -0,0 +1,320 @@
+use crate::database::DatabaseHandler;
+use crate::binance_collector::BinanceCollector;
+use crate::historical_downloader::HistoricalDownloader;
+use crate::error::SchedulerError;
+use crate::structs::*;
+use crate::DATA_CUTOFF_TIMESTAMP;
+
+use log::{info, error, warn};
+use std::sync::Arc;
+use tokio::sync::{mpsc, RwLock};
+use tokio::task::JoinHandle;
+
+const ONE_MINUTE_MS: i64 = 60_000;
+
+pub struct Scheduler {
+    db: Arc<DatabaseHandler>,
+    command_rx: mpsc::Receiver<SchedulerCommand>,
+    collector_handle: Option<JoinHandle<()>>,
+    is_running: Arc<RwLock<bool>>,
+    active_symbols: Arc<RwLock<Vec<String>>>,
+    ws_broadcast_tx: Option<mpsc::Sender<CandleData>>,
+}
+
+impl Scheduler {
+    pub fn new(
+        db: Arc<DatabaseHandler>,
+        command_rx: mpsc::Receiver<SchedulerCommand>,
+        ws_broadcast_tx: Option<mpsc::Sender<CandleData>>,
+    ) -> Self {
+        Self {
+            db,
+            command_rx,
+            collector_handle: None,
+            is_running: Arc::new(RwLock::new(false)),
+            active_symbols: Arc::new(RwLock::new(Vec::new())),
+            ws_broadcast_tx,
+        }
+    }
+
+    pub async fn run(&mut self) {
+        info!("Scheduler started");
+
+        // Initial startup
+        if let Err(e) = self.start_collector().await {
+            error!("Failed to start collector on init: {}", e);
+        }
+
+        // Command processing loop
+        while let Some(cmd) = self.command_rx.recv().await {
+            match cmd {
+                SchedulerCommand::AddSymbol { symbol, backfill_from: _ } => {
+                    info!("AddSymbol command ignored - symbols managed by get_symbol()");
+                    let _ = self.db.add_symbol(&symbol).await;
+                }
+                SchedulerCommand::RemoveSymbol { symbol } => {
+                    info!("RemoveSymbol: {}", symbol);
+                    let _ = self.db.remove_symbol(&symbol).await;
+                }
+                SchedulerCommand::RestartCollector => {
+                    self.handle_restart_collector().await;
+                }
+                SchedulerCommand::GetStatus { reply } => {
+                    self.handle_get_status(reply).await;
+                }
+                SchedulerCommand::Shutdown => {
+                    info!("Shutdown command received");
+                    self.stop_collector().await;
+                    break;
+                }
+            }
+        }
+
+        info!("Scheduler stopped");
+    }
+
+    async fn handle_restart_collector(&mut self) {
+        info!("Restarting collector...");
+        self.stop_collector().await;
+        if let Err(e) = self.start_collector().await {
+            error!("Failed to restart collector: {}", e);
+        }
+    }
+
+    async fn handle_get_status(&self, reply: tokio::sync::oneshot::Sender<SchedulerStatus>) {
+        let is_running = *self.is_running.read().await;
+        let active_symbols = self.active_symbols.read().await.clone();
+        let collector_connected = self.collector_handle.is_some() && is_running;
+
+        let _ = reply.send(SchedulerStatus {
+            is_running,
+            active_symbols,
+            collector_connected,
+        });
+    }
+
+    async fn start_collector(&mut self) -> Result<(), SchedulerError> {
+        // STEP 1: Get symbols from TRACKED_SYMBOLS env variable
+        let symbol_names = DatabaseHandler::get_symbols_from_env();
+        
+        if symbol_names.is_empty() {
+            warn!("No symbols in TRACKED_SYMBOLS env variable. Set TRACKED_SYMBOLS=BTCUSDT,ETHUSDT,...");
+            return Ok(());
+        }
+        info!("Tracking {} symbols from env: {:?}", symbol_names.len(), symbol_names);
+
+        // Update active symbols
+        {
+            let mut active = self.active_symbols.write().await;
+            *active = symbol_names.clone();
+        }
+
+        // STEP 2: Start WebSocket FIRST to capture real-time data immediately
+        info!("Starting WebSocket collector FIRST (priority: real-time data)...");
+        let collector = BinanceCollector::new(symbol_names.clone());
+
+        let (candle_tx, mut candle_rx) = mpsc::channel::<CandleData>(10000);
+
+        // Consumer that both saves to DB and broadcasts to WebSocket clients
+        let db = self.db.clone();
+        let ws_tx = self.ws_broadcast_tx.clone();
+        tokio::spawn(async move {
+            while let Some(candle) = candle_rx.recv().await {
+                // Broadcast to WebSocket clients immediately (all updates for real-time charts)
+                if let Some(ref tx) = ws_tx {
+                    let _ = tx.send(candle.clone()).await;
+                }
+
+                // Only save closed candles to database
+                if candle.is_closed {
+                    if let Err(e) = db.insert_candle(&candle).await {
+                        error!("Failed to insert realtime candle: {}", e);
+                    }
+                }
+            }
+        });
+
+        let is_running = self.is_running.clone();
+        let handle = tokio::spawn(async move {
+            {
+                let mut running = is_running.write().await;
+                *running = true;
+            }
+
+            if let Err(e) = collector.start_stream(candle_tx).await {
+                error!("Collector error: {}", e);
+            }
+
+            {
+                let mut running = is_running.write().await;
+                *running = false;
+            }
+        });
+
+        self.collector_handle = Some(handle);
+        info!("WebSocket collector started! Real-time data is now being captured.");
+
+        // STEP 3: Background sync - runs in parallel with WebSocket
+        // This fills in historical data without blocking real-time updates
+        let db_for_sync = self.db.clone();
+        let symbols_for_sync = symbol_names.clone();
+
+        tokio::spawn(async move {
+            info!("Starting background historical sync...");
+
+            // Build API clients for parallel sync
+            let api_clients = BinanceCollector::build_clients().await;
+            if api_clients.is_empty() {
+                error!("No working API clients available for background sync");
+                return;
+            }
+            info!("Built {} API clients for background sync", api_clients.len());
+
+            // Build HTTP clients for ZIP downloads
+            let http_clients = HistoricalDownloader::build_clients().await;
+            info!("Built {} HTTP clients for ZIP downloads", http_clients.len());
+
+            // Create channel for background sync (lower priority, uses batching)
+            let (sync_tx, sync_rx) = mpsc::channel::<CandleData>(100000);
+
+            let db_consumer = db_for_sync.clone();
+            let consumer_handle = tokio::spawn(async move {
+                db_consumer.start_consumer(sync_rx).await;
+            });
+
+            // Sync symbols that need updates
+            let mut synced_count = 0;
+            let total_symbols = symbols_for_sync.len();
+
+            for symbol in &symbols_for_sync {
+                let latest_ts = db_for_sync.get_latest_timestamp(symbol).await.ok().flatten();
+
+                let start_time = match latest_ts {
+                    Some(ts) => ts + ONE_MINUTE_MS,
+                    None => DATA_CUTOFF_TIMESTAMP,
+                };
+
+                let now = std::time::SystemTime::now()
+                    .duration_since(std::time::UNIX_EPOCH)
+                    .unwrap()
+                    .as_millis() as i64;
+
+                // Skip if less than 2 minutes behind (WebSocket will catch up)
+                if now - start_time < ONE_MINUTE_MS * 2 {
+                    synced_count += 1;
+                    continue;
+                }
+
+                let behind_mins = (now - start_time) / ONE_MINUTE_MS;
+                info!("[{}/{}] Syncing {} ({} minutes behind)...",
+                    synced_count + 1, total_symbols, symbol, behind_mins);
+
+                // Use comprehensive sync: Monthly ZIP → Daily ZIP → API
+                match BinanceCollector::sync_comprehensive(
+                    symbol.clone(),
+                    start_time,
+                    api_clients.clone(),
+                    &http_clients,
+                    sync_tx.clone(),
+                ).await {
+                    Ok(count) => {
+                        synced_count += 1;
+                        if count > 0 {
+                            info!("[{}/{}] {} synced: {} candles", synced_count, total_symbols, symbol, count);
+                        }
+                    }
+                    Err(e) => {
+                        synced_count += 1;
+                        error!("{} sync failed: {}", symbol, e);
+                    }
+                }
+            }
+
+            // Close sync channel
+            drop(sync_tx);
+            let _ = consumer_handle.await;
+            info!("Background historical sync complete! Synced {} symbols", synced_count);
+
+            // STEP 4: Gap detection and repair (also in background)
+            info!("Starting background gap detection and repair...");
+            let (gap_tx, gap_rx) = mpsc::channel::<CandleData>(100000);
+
+            let db_gap = db_for_sync.clone();
+            let gap_consumer_handle = tokio::spawn(async move {
+                db_gap.start_consumer(gap_rx).await;
+            });
+
+            // Rebuild clients for gap repair
+            let clients = BinanceCollector::build_clients().await;
+            let mut total_gaps_repaired = 0u64;
+            let mut symbols_with_gaps = 0;
+
+            for symbol in &symbols_for_sync {
+                match db_for_sync.find_gaps(symbol, DATA_CUTOFF_TIMESTAMP).await {
+                    Ok(gaps) => {
+                        if gaps.is_empty() {
+                            continue;
+                        }
+
+                        symbols_with_gaps += 1;
+                        info!("{} - Found {} gaps to repair", symbol, gaps.len());
+
+                        for (gap_start, gap_end) in gaps {
+                            let gap_duration_mins = (gap_end - gap_start) / ONE_MINUTE_MS;
+
+                            // Skip very small gaps (less than 2 minutes)
+                            if gap_duration_mins < 2 {
+                                continue;
+                            }
+
+                            info!(
+                                "{} - Repairing gap: {} minutes",
+                                symbol, gap_duration_mins
+                            );
+
+                            match BinanceCollector::sync_from_scratch(
+                                symbol.clone(),
+                                gap_start,
+                                clients.clone(),
+                                gap_tx.clone(),
+                            ).await {
+                                Ok(count) => {
+                                    total_gaps_repaired += count;
+                                }
+                                Err(e) => {
+                                    error!("{} - Gap repair failed: {}", symbol, e);
+                                }
+                            }
+                        }
+                    }
+                    Err(e) => {
+                        warn!("{} - Failed to check gaps: {}", symbol, e);
+                    }
+                }
+            }
+
+            // Close gap repair channel
+            drop(gap_tx);
+            let _ = gap_consumer_handle.await;
+            info!("Gap detection complete! {} symbols had gaps, repaired {} candles total",
+                symbols_with_gaps, total_gaps_repaired);
+        });
+
+        Ok(())
+    }
+
+    async fn stop_collector(&mut self) {
+        if let Some(handle) = self.collector_handle.take() {
+            info!("Stopping collector...");
+            handle.abort();
+            let _ = handle.await;
+
+            let mut running = self.is_running.write().await;
+            *running = false;
+        }
+    }
+}
+
+pub fn create_command_channel() -> (mpsc::Sender<SchedulerCommand>, mpsc::Receiver<SchedulerCommand>) {
+    mpsc::channel(100)
+}

backend/src/structs.rs

@@ -0,0 +1,159 @@
+use serde::{Serialize, Deserialize};
+use tokio::sync::oneshot;
+
+/// Data cutoff timestamp: 2024-01-01 00:00:00 UTC
+/// Data before this timestamp will not be synced
+pub const DATA_CUTOFF_TIMESTAMP: i64 = 1704067200000;
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct CandleData {
+    pub symbol: String,
+    pub timestamp: i64,
+    pub open: f64,
+    pub high: f64,
+    pub low: f64,
+    pub close: f64,
+    pub volume: f64,
+    pub taker_buy_volume: f64,
+    pub net_volume: f64,
+    pub is_closed: bool,
+}
+
+impl CandleData {
+    pub fn calculate_net_volume(total_volume: f64, taker_buy_volume: f64) -> f64 {
+        2.0 * taker_buy_volume - total_volume
+    }
+}
+
+/// Commands for controlling the Scheduler
+pub enum SchedulerCommand {
+    /// Add a new symbol to track. Triggers backfill then collector restart.
+    AddSymbol { 
+        symbol: String, 
+        backfill_from: Option<i64>,  // None = use EARLIEST_TIME
+    },
+    
+    /// Remove a symbol from tracking. Triggers collector restart.
+    RemoveSymbol { symbol: String },
+    
+    /// Restart the collector with current active symbols from database
+    RestartCollector,
+    
+    /// Get current scheduler status
+    GetStatus { reply: oneshot::Sender<SchedulerStatus> },
+    
+    /// Graceful shutdown
+    Shutdown,
+}
+
+/// Scheduler status for API responses
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct SchedulerStatus {
+    pub is_running: bool,
+    pub active_symbols: Vec<String>,
+    pub collector_connected: bool,
+}
+
+// API Request/Response structs
+
+#[derive(Debug, Clone, Copy, Deserialize, PartialEq)]
+#[serde(rename_all = "lowercase")]
+pub enum Interval {
+    #[serde(rename = "1m")]
+    Min1,
+    #[serde(rename = "5m")]
+    Min5,
+    #[serde(rename = "15m")]
+    Min15,
+    #[serde(rename = "1h")]
+    Hour1,
+    #[serde(rename = "4h")]
+    Hour4,
+    #[serde(rename = "1d")]
+    Day1,
+    #[serde(rename = "1w")]
+    Week1,
+    #[serde(rename = "1M")]
+    Month1,
+}
+
+impl Default for Interval {
+    fn default() -> Self {
+        Interval::Min1
+    }
+}
+
+#[derive(Deserialize)]
+pub struct KlineQuery {
+    pub limit: Option<i64>,
+    pub interval: Option<Interval>,
+    pub end_time: Option<i64>,
+}
+
+#[derive(Serialize)]
+pub struct ApiResponse<T> {
+    pub success: bool,
+    pub data: Option<T>,
+    pub error: Option<String>,
+}
+
+impl<T> ApiResponse<T> {
+    pub fn ok(data: T) -> Self {
+        Self { success: true, data: Some(data), error: None }
+    }
+    
+    pub fn err(msg: &str) -> Self {
+        Self { success: false, data: None, error: Some(msg.to_string()) }
+    }
+}
+
+#[derive(Deserialize)]
+pub struct AddSymbolRequest {
+    pub symbol: String,
+    pub backfill_from: Option<i64>,
+}
+
+#[derive(Serialize,Deserialize,Debug,Clone)]
+pub struct Symbol{
+
+    pub symbol: String,
+    pub start_timestamp: i64,
+}
+
+// WebSocket message types
+#[derive(Debug, Clone, Serialize, Deserialize)]
+#[serde(tag = "type", content = "data")]
+pub enum WsMessage {
+    /// Kline update for a symbol
+    #[serde(rename = "kline")]
+    Kline(CandleData),
+
+    /// Ticker update with price info for watchlist
+    #[serde(rename = "ticker")]
+    Ticker(TickerUpdate),
+
+    /// Subscribe to symbols
+    #[serde(rename = "subscribe")]
+    Subscribe { symbols: Vec<String> },
+
+    /// Unsubscribe from symbols
+    #[serde(rename = "unsubscribe")]
+    Unsubscribe { symbols: Vec<String> },
+
+    /// Ping/Pong for keepalive
+    #[serde(rename = "ping")]
+    Ping,
+
+    #[serde(rename = "pong")]
+    Pong,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct TickerUpdate {
+    pub symbol: String,
+    pub price: f64,
+    pub change_24h: f64,
+    pub change_percent_24h: f64,
+    pub volume_24h: f64,
+    pub timestamp: i64,
+}

backend/src/tradingview.rs

@@ -0,0 +1,559 @@
+use crate::{DatabaseHandler, Interval, CandleData, WsMessage};
+
+use axum::{
+    Router,
+    routing::{get, post, delete},
+    extract::{Query, State, WebSocketUpgrade, ws::{Message, WebSocket}},
+    response::{Json, IntoResponse},
+    http::StatusCode,
+};
+use futures::{StreamExt, SinkExt};
+use serde::{Deserialize, Serialize};
+use std::collections::HashSet;
+use std::sync::Arc;
+use std::time::{SystemTime, UNIX_EPOCH};
+use std::path::PathBuf;
+use tokio::sync::{broadcast, RwLock};
+use tokio::fs;
+use log::{info, warn, debug, error};
+
+pub struct TradingViewState {
+    pub db: Arc<DatabaseHandler>,
+    pub candle_tx: broadcast::Sender<CandleData>,
+}
+
+impl TradingViewState {
+    pub fn new(db: Arc<DatabaseHandler>) -> (Self, broadcast::Receiver<CandleData>) {
+        let (candle_tx, candle_rx) = broadcast::channel(10000);
+        (Self { db, candle_tx }, candle_rx)
+    }
+
+    /// Send a candle update to all connected WebSocket clients
+    pub fn broadcast_candle(&self, candle: CandleData) {
+        // Ignore errors (no subscribers)
+        let _ = self.candle_tx.send(candle);
+    }
+}
+
+pub fn tradingview_routes() -> Router<Arc<TradingViewState>> {
+    Router::new()
+        .route("/config", get(get_config))
+        .route("/time", get(get_time))
+        .route("/symbols", get(get_symbol_info))
+        .route("/search", get(search_symbols))
+        .route("/tracked-symbols", get(get_tracked_symbols))
+        .route("/daily-opens", get(get_daily_opens))
+        .route("/history", get(get_history))
+        .route("/ws", get(ws_handler))
+        // Canvas API
+        .route("/canvas/list", get(canvas_list))
+        .route("/canvas/load", get(canvas_load))
+        .route("/canvas/save", post(canvas_save))
+        .route("/canvas/delete", delete(canvas_delete))
+}
+
+// ============ UDF Response Types ============
+
+#[derive(Serialize)]
+struct UdfConfig {
+    supported_resolutions: Vec<&'static str>,
+    supports_group_request: bool,
+    supports_marks: bool,
+    supports_search: bool,
+    supports_timescale_marks: bool,
+}
+
+#[derive(Serialize)]
+struct UdfSymbolInfo {
+    symbol: String,
+    ticker: String,
+    name: String,
+    full_name: String,
+    description: String,
+    exchange: String,
+    listed_exchange: String,
+    #[serde(rename = "type")]
+    symbol_type: String,
+    currency_code: String,
+    session: String,
+    timezone: String,
+    minmovement: i32,
+    minmov: i32,
+    minmovement2: i32,
+    minmov2: i32,
+    pricescale: i64,
+    supported_resolutions: Vec<&'static str>,
+    has_intraday: bool,
+    has_daily: bool,
+    has_weekly_and_monthly: bool,
+    data_status: String,
+}
+
+#[derive(Serialize)]
+struct UdfSearchResult {
+    symbol: String,
+    full_name: String,
+    description: String,
+    exchange: String,
+    ticker: String,
+    #[serde(rename = "type")]
+    symbol_type: String,
+}
+
+#[derive(Serialize)]
+#[serde(untagged)]
+enum UdfHistoryResponse {
+    Ok {
+        s: String,  // "ok"
+        t: Vec<i64>,      // timestamps (seconds)
+        o: Vec<f64>,      // open
+        h: Vec<f64>,      // high
+        l: Vec<f64>,      // low
+        c: Vec<f64>,      // close
+        v: Vec<f64>,      // volume
+        nv: Vec<f64>,     // net volume (custom)
+        tbv: Vec<f64>,    // taker buy volume (custom)
+    },
+    NoData {
+        s: String,  // "no_data"
+        #[serde(skip_serializing_if = "Option::is_none")]
+        #[serde(rename = "nextTime")]
+        next_time: Option<i64>,
+    },
+    Error {
+        s: String,  // "error"
+        errmsg: String,
+    },
+}
+
+// ============ Query Parameters ============
+
+#[derive(Deserialize)]
+struct SymbolQuery {
+    symbol: String,
+}
+
+#[allow(dead_code)]
+#[derive(Deserialize)]
+struct SearchQuery {
+    query: Option<String>,
+    #[serde(rename = "type")]
+    symbol_type: Option<String>,
+    exchange: Option<String>,
+    limit: Option<i32>,
+}
+
+#[derive(Deserialize)]
+struct HistoryQuery {
+    symbol: String,
+    resolution: String,
+    from: i64,      // unix timestamp (seconds)
+    to: i64,        // unix timestamp (seconds)
+    countback: Option<i64>,
+}
+
+// ============ Handlers ============
+
+// GET /config
+async fn get_config() -> Json<UdfConfig> {
+    Json(UdfConfig {
+        supported_resolutions: vec!["1", "5", "15", "60", "240", "1D", "1W", "1M"],
+        supports_group_request: false,
+        supports_marks: false,
+        supports_search: true,
+        supports_timescale_marks: false,
+    })
+}
+
+// GET /time
+async fn get_time() -> String {
+    let now = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap()
+        .as_secs();
+    now.to_string()
+}
+
+// GET /symbols?symbol=BTCUSDT
+async fn get_symbol_info(
+    Query(query): Query<SymbolQuery>,
+) -> Json<UdfSymbolInfo> {
+    let symbol = query.symbol.to_uppercase();
+    
+    Json(UdfSymbolInfo {
+        symbol: symbol.clone(),
+        ticker: symbol.clone(),
+        name: symbol.clone(),
+        full_name: format!("BINANCE:{}", symbol),
+        description: symbol.clone(),
+        exchange: "BINANCE".to_string(),
+        listed_exchange: "BINANCE".to_string(),
+        symbol_type: "crypto".to_string(),
+        currency_code: "USDT".to_string(),
+        session: "24x7".to_string(),
+        timezone: "Etc/UTC".to_string(),
+        minmovement: 1,
+        minmov: 1,
+        minmovement2: 0,
+        minmov2: 0,
+        pricescale: 100000000,  // 8 decimal places for crypto
+        supported_resolutions: vec!["1", "5", "15", "60", "240", "1D", "1W", "1M"],
+        has_intraday: true,
+        has_daily: true,
+        has_weekly_and_monthly: true,
+        data_status: "streaming".to_string(),
+    })
+}
+
+// GET /tracked-symbols - 返回 TRACKED_SYMBOL 环境变量中配置的 symbols
+async fn get_tracked_symbols() -> Json<Vec<String>> {
+    Json(crate::DatabaseHandler::get_symbols_from_env())
+}
+
+// GET /daily-opens - 返回所有 symbol 当天 UTC 00:00 的开盘价
+async fn get_daily_opens(
+    State(state): State<Arc<TradingViewState>>,
+) -> Json<std::collections::HashMap<String, f64>> {
+    match state.db.get_daily_opens().await {
+        Ok(opens) => Json(opens),
+        Err(_) => Json(std::collections::HashMap::new()),
+    }
+}
+
+// GET /search?query=BTC&limit=10
+async fn search_symbols(
+    State(state): State<Arc<TradingViewState>>,
+    Query(query): Query<SearchQuery>,
+) -> Json<Vec<UdfSearchResult>> {
+    let search_term = query.query.unwrap_or_default().to_uppercase();
+    let limit = query.limit.unwrap_or(30) as usize;
+
+    let symbols = match state.db.get_active_symbols().await {
+        Ok(s) => s,
+        Err(_) => return Json(vec![]),
+    };
+
+    let results: Vec<UdfSearchResult> = symbols
+        .into_iter()
+        .filter(|s| search_term.is_empty() || s.to_uppercase().contains(&search_term))
+        .take(limit)
+        .map(|s| {
+            let upper = s.to_uppercase();
+            UdfSearchResult {
+                symbol: upper.clone(),
+                full_name: format!("BINANCE:{}", upper),
+                description: upper.clone(),
+                exchange: "BINANCE".to_string(),
+                ticker: upper,
+                symbol_type: "crypto".to_string(),
+            }
+        })
+        .collect();
+
+    Json(results)
+}
+
+// GET /history?symbol=BTCUSDT&resolution=1&from=...&to=...
+async fn get_history(
+    State(state): State<Arc<TradingViewState>>,
+    Query(query): Query<HistoryQuery>,
+) -> Json<UdfHistoryResponse> {
+    let symbol = query.symbol.to_uppercase();
+    
+    // Convert resolution string to Interval enum
+    let interval = match query.resolution.as_str() {
+        "1" => Interval::Min1,
+        "5" => Interval::Min5,
+        "15" => Interval::Min15,
+        "60" => Interval::Hour1,
+        "240" => Interval::Hour4,
+        "D" | "1D" => Interval::Day1,
+        "W" | "1W" => Interval::Week1,
+        "M" | "1M" => Interval::Month1,
+        _ => Interval::Min1,
+    };
+
+    // Convert from/to (seconds) to milliseconds for database query
+    let from_ms = query.from * 1000;
+    let to_ms = query.to * 1000;
+    
+    // Calculate limit based on countback or time range
+    let limit = query.countback.unwrap_or(1000);
+
+    match state.db.get_klines_aggregated(&symbol, interval, limit, Some(to_ms)).await {
+        Ok(candles) => {
+            // Filter by from_ms and convert to UDF format
+            let filtered: Vec<_> = candles
+                .into_iter()
+                .filter(|c| c.timestamp >= from_ms && c.timestamp <= to_ms)
+                .collect();
+
+            if filtered.is_empty() {
+                return Json(UdfHistoryResponse::NoData {
+                    s: "no_data".to_string(),
+                    next_time: None,
+                });
+            }
+
+            let t: Vec<i64> = filtered.iter().map(|c| c.timestamp / 1000).collect();
+            let o: Vec<f64> = filtered.iter().map(|c| c.open).collect();
+            let h: Vec<f64> = filtered.iter().map(|c| c.high).collect();
+            let l: Vec<f64> = filtered.iter().map(|c| c.low).collect();
+            let c: Vec<f64> = filtered.iter().map(|c| c.close).collect();
+            let v: Vec<f64> = filtered.iter().map(|c| c.volume).collect();
+            let nv: Vec<f64> = filtered.iter().map(|c| c.net_volume).collect();
+            let tbv: Vec<f64> = filtered.iter().map(|c| c.taker_buy_volume).collect();
+
+            Json(UdfHistoryResponse::Ok {
+                s: "ok".to_string(),
+                t, o, h, l, c, v, nv, tbv,
+            })
+        }
+        Err(e) => Json(UdfHistoryResponse::Error {
+            s: "error".to_string(),
+            errmsg: e.to_string(),
+        }),
+    }
+}
+
+// ============ WebSocket Handler ============
+
+async fn ws_handler(
+    ws: WebSocketUpgrade,
+    State(state): State<Arc<TradingViewState>>,
+) -> impl IntoResponse {
+    ws.on_upgrade(move |socket| handle_ws_connection(socket, state))
+}
+
+async fn handle_ws_connection(socket: WebSocket, state: Arc<TradingViewState>) {
+    let (mut sender, mut receiver) = socket.split();
+
+    // Subscribe to broadcast channel
+    let mut candle_rx = state.candle_tx.subscribe();
+
+    // Subscribed symbols for this client
+    let subscribed_symbols: Arc<RwLock<HashSet<String>>> = Arc::new(RwLock::new(HashSet::new()));
+    let subscribed_symbols_clone = subscribed_symbols.clone();
+
+    info!("WebSocket client connected");
+
+    // Task to receive messages from client
+    let recv_task = tokio::spawn(async move {
+        while let Some(msg) = receiver.next().await {
+            match msg {
+                Ok(Message::Text(text)) => {
+                    // Parse incoming message
+                    if let Ok(ws_msg) = serde_json::from_str::<WsMessage>(&text) {
+                        match ws_msg {
+                            WsMessage::Subscribe { symbols } => {
+                                let mut subs = subscribed_symbols_clone.write().await;
+                                for s in symbols {
+                                    subs.insert(s.to_uppercase());
+                                }
+                                debug!("Client subscribed to {} symbols", subs.len());
+                            }
+                            WsMessage::Unsubscribe { symbols } => {
+                                let mut subs = subscribed_symbols_clone.write().await;
+                                for s in symbols {
+                                    subs.remove(&s.to_uppercase());
+                                }
+                            }
+                            WsMessage::Ping => {
+                                // Pong is handled by the send task
+                            }
+                            _ => {}
+                        }
+                    }
+                }
+                Ok(Message::Close(_)) => {
+                    info!("WebSocket client disconnected");
+                    break;
+                }
+                Err(e) => {
+                    warn!("WebSocket receive error: {}", e);
+                    break;
+                }
+                _ => {}
+            }
+        }
+    });
+
+    // Task to send messages to client
+    let send_task = tokio::spawn(async move {
+        loop {
+            tokio::select! {
+                // Forward candle updates to client
+                result = candle_rx.recv() => {
+                    match result {
+                        Ok(candle) => {
+                            let subs = subscribed_symbols.read().await;
+                            // Send to client if subscribed or if subscribed to all (empty set means all)
+                            if subs.is_empty() || subs.contains(&candle.symbol.to_uppercase()) {
+                                let msg = WsMessage::Kline(candle);
+                                if let Ok(json) = serde_json::to_string(&msg) {
+                                    if sender.send(Message::Text(json.into())).await.is_err() {
+                                        break;
+                                    }
+                                }
+                            }
+                        }
+                        Err(broadcast::error::RecvError::Lagged(n)) => {
+                            warn!("WebSocket client lagged {} messages", n);
+                        }
+                        Err(broadcast::error::RecvError::Closed) => {
+                            break;
+                        }
+                    }
+                }
+            }
+        }
+    });
+
+    // Wait for either task to finish
+    tokio::select! {
+        _ = recv_task => {}
+        _ = send_task => {}
+    }
+
+    info!("WebSocket connection closed");
+}
+
+// ============ Canvas API ============
+
+const STORAGE_DIR: &str = "storage";
+const DEFAULT_USER: &str = "default";
+
+#[derive(Deserialize)]
+struct CanvasListQuery {
+    symbol: String,
+}
+
+#[derive(Deserialize)]
+struct CanvasLoadQuery {
+    symbol: String,
+    name: String,
+}
+
+#[derive(Deserialize)]
+struct CanvasSaveBody {
+    symbol: String,
+    name: String,
+    data: serde_json::Value,
+}
+
+#[derive(Deserialize)]
+struct CanvasDeleteQuery {
+    symbol: String,
+    name: String,
+}
+
+#[derive(Serialize)]
+struct CanvasListResponse {
+    canvases: Vec<String>,
+}
+
+fn get_user_id(headers: &axum::http::HeaderMap) -> String {
+    headers.get("X-User-Id")
+        .and_then(|v| v.to_str().ok())
+        .filter(|s| !s.is_empty())
+        .unwrap_or(DEFAULT_USER)
+        .to_string()
+}
+
+fn get_canvas_dir(user_id: &str, symbol: &str) -> PathBuf {
+    PathBuf::from(STORAGE_DIR)
+        .join(user_id)
+        .join(symbol.to_uppercase())
+}
+
+fn get_canvas_path(user_id: &str, symbol: &str, name: &str) -> PathBuf {
+    get_canvas_dir(user_id, symbol).join(format!("{}.json", name))
+}
+
+// GET /canvas/list?symbol=BTCUSDT
+async fn canvas_list(headers: axum::http::HeaderMap, Query(query): Query<CanvasListQuery>) -> Json<CanvasListResponse> {
+    let user_id = get_user_id(&headers);
+    let dir = get_canvas_dir(&user_id, &query.symbol);
+    let mut canvases = Vec::new();
+    
+    if let Ok(mut entries) = fs::read_dir(&dir).await {
+        while let Ok(Some(entry)) = entries.next_entry().await {
+            if let Some(name) = entry.file_name().to_str() {
+                if name.ends_with(".json") {
+                    canvases.push(name.trim_end_matches(".json").to_string());
+                }
+            }
+        }
+    }
+    
+    canvases.sort();
+    Json(CanvasListResponse { canvases })
+}
+
+// GET /canvas/load?symbol=BTCUSDT&name=default
+async fn canvas_load(
+    headers: axum::http::HeaderMap,
+    Query(query): Query<CanvasLoadQuery>,
+) -> Result<Json<serde_json::Value>, StatusCode> {
+    let user_id = get_user_id(&headers);
+    let path = get_canvas_path(&user_id, &query.symbol, &query.name);
+    
+    match fs::read_to_string(&path).await {
+        Ok(content) => {
+            match serde_json::from_str(&content) {
+                Ok(data) => Ok(Json(data)),
+                Err(e) => {
+                    error!("Failed to parse canvas {}: {}", path.display(), e);
+                    Err(StatusCode::INTERNAL_SERVER_ERROR)
+                }
+            }
+        }
+        Err(_) => Err(StatusCode::NOT_FOUND),
+    }
+}
+
+// POST /canvas/save
+async fn canvas_save(
+    headers: axum::http::HeaderMap,
+    Json(body): Json<CanvasSaveBody>,
+) -> Result<StatusCode, StatusCode> {
+    let user_id = get_user_id(&headers);
+    let dir = get_canvas_dir(&user_id, &body.symbol);
+    let path = get_canvas_path(&user_id, &body.symbol, &body.name);
+    
+    // Create directory if not exists
+    if let Err(e) = fs::create_dir_all(&dir).await {
+        error!("Failed to create dir {}: {}", dir.display(), e);
+        return Err(StatusCode::INTERNAL_SERVER_ERROR);
+    }
+    
+    // Write canvas data
+    let content = serde_json::to_string_pretty(&body.data)
+        .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?;
+    
+    fs::write(&path, content).await
+        .map_err(|e| {
+            error!("Failed to write canvas {}: {}", path.display(), e);
+            StatusCode::INTERNAL_SERVER_ERROR
+        })?;
+    
+    info!("Saved canvas: {}", path.display());
+    Ok(StatusCode::OK)
+}
+
+// DELETE /canvas/delete?symbol=BTCUSDT&name=default
+async fn canvas_delete(
+    headers: axum::http::HeaderMap,
+    Query(query): Query<CanvasDeleteQuery>,
+) -> StatusCode {
+    let user_id = get_user_id(&headers);
+    let path = get_canvas_path(&user_id, &query.symbol, &query.name);
+    
+    match fs::remove_file(&path).await {
+        Ok(_) => {
+            info!("Deleted canvas: {}", path.display());
+            StatusCode::OK
+        }
+        Err(_) => StatusCode::NOT_FOUND,
+    }
+}