Upload files.

backend/examples/sql.txt

@@ -0,0 +1,47 @@
+-- Create klines table for 1-minute candlestick data
+CREATE TABLE IF NOT EXISTS klines_1m (
+    symbol          VARCHAR(20) NOT NULL,
+    timestamp       BIGINT NOT NULL,
+    open            DOUBLE PRECISION NOT NULL,
+    high            DOUBLE PRECISION NOT NULL,
+    low             DOUBLE PRECISION NOT NULL,
+    close           DOUBLE PRECISION NOT NULL,
+    volume          DOUBLE PRECISION NOT NULL,
+    taker_buy_volume DOUBLE PRECISION NOT NULL,
+    net_volume      DOUBLE PRECISION NOT NULL,
+    PRIMARY KEY (symbol, timestamp)
+);
+
+-- Convert to TimescaleDB hypertable for better time-series performance
+-- chunk_time_interval = 7 days (in milliseconds: 7 * 24 * 60 * 60 * 1000 = 604800000)
+SELECT create_hypertable('klines_1m', 'timestamp', 
+    chunk_time_interval => 604800000,
+    if_not_exists => TRUE
+);
+
+-- Create index for faster queries by symbol
+CREATE INDEX IF NOT EXISTS idx_klines_1m_symbol ON klines_1m (symbol, timestamp DESC);
+
+-- Table for tracking active symbols
+CREATE TABLE IF NOT EXISTS tracked_symbols (
+    symbol      VARCHAR(20) PRIMARY KEY,
+    added_at    TIMESTAMP DEFAULT NOW(),
+    is_active   BOOLEAN DEFAULT TRUE
+);
+
+-- Insert some default symbols
+INSERT INTO tracked_symbols (symbol) VALUES 
+    ('btcusdt'),
+    ('ethusdt')
+ON CONFLICT (symbol) DO NOTHING;
+
+-- Useful queries:
+
+-- Get latest timestamp for a symbol
+-- SELECT MAX(timestamp) FROM klines_1m WHERE symbol = 'BTCUSDT';
+
+-- Get latest N candles for a symbol
+-- SELECT * FROM klines_1m WHERE symbol = 'BTCUSDT' ORDER BY timestamp DESC LIMIT 100;
+
+-- Get all active symbols
+-- SELECT symbol FROM tracked_symbols WHERE is_active = TRUE;

backend/examples/sync_all.rs

@@ -0,0 +1,137 @@
+use backend::{BinanceCollector, CandleData, DatabaseHandler};
+
+use log::{info, error};
+use dotenv::dotenv;
+use tokio::sync::mpsc;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicU64, Ordering};
+
+#[tokio::main]
+async fn main() {
+    dotenv().ok();
+    env_logger::init();
+
+    // 1. Connect to database
+    let database_url = std::env::var("DATABASE_URL")
+        .expect("DATABASE_URL must be set");
+    
+    info!("Connecting to database...");
+    let db = Arc::new(
+        DatabaseHandler::new(&database_url)
+            .await
+            .expect("Failed to connect to database")
+    );
+
+    // 2. Get all symbols (filter USDT pairs only)
+    info!("Fetching all symbols from Binance...");
+    let all_symbols = BinanceCollector::get_symbol().await.unwrap();
+    let symbols: Vec<_> = all_symbols
+        .into_iter()
+        .filter(|s| s.symbol.ends_with("USDT"))
+        .collect();
+    info!("Got {} USDT symbols to sync", symbols.len());
+
+    // 3. Build clients
+    info!("Building clients...");
+    let clients = BinanceCollector::build_clients().await;
+    info!("Built {} working clients", clients.len());
+
+    if clients.is_empty() {
+        error!("No working clients, exiting");
+        return;
+    }
+
+    // 4. Sync each symbol one by one
+    let total_symbols = symbols.len();
+    
+    for (i, symbol) in symbols.into_iter().enumerate() {
+        let now = std::time::SystemTime::now()
+            .duration_since(std::time::UNIX_EPOCH)
+            .unwrap()
+            .as_millis() as i64;
+        
+        let start_time = symbol.start_timestamp;
+        let total_range_ms = now - start_time;
+        // Estimate total candles (1 candle per minute)
+        let estimated_total = (total_range_ms / 60000) as u64;
+        
+        info!("[{}/{}] Starting sync for {} (estimated {} candles)", 
+            i + 1, total_symbols, symbol.symbol, estimated_total);
+
+        // Create channels
+        let (tx, rx) = mpsc::channel::<CandleData>(100000);
+        let (db_tx, db_rx) = mpsc::channel::<CandleData>(100000);
+
+        // Progress tracking
+        let candle_count = Arc::new(AtomicU64::new(0));
+        let last_progress = Arc::new(AtomicU64::new(0));
+
+        // Progress monitor and forward task
+        let symbol_name = symbol.symbol.clone();
+        let candle_count_clone = candle_count.clone();
+        let last_progress_clone = last_progress.clone();
+        let idx = i + 1;
+        
+        let progress_handle = tokio::spawn(async move {
+            let mut rx = rx;
+            while let Some(candle) = rx.recv().await {
+                let count = candle_count_clone.fetch_add(1, Ordering::Relaxed) + 1;
+                
+                // Calculate progress based on received candles vs estimated total
+                let progress = if estimated_total > 0 {
+                    ((count as f64 / estimated_total as f64) * 100.0).min(100.0) as u64
+                } else {
+                    100
+                };
+                
+                // Report every 20%
+                let last = last_progress_clone.load(Ordering::Relaxed);
+                let milestone = (progress / 20) * 20;
+                if milestone > last && milestone <= 100 {
+                    if last_progress_clone.compare_exchange(
+                        last, milestone, Ordering::Relaxed, Ordering::Relaxed
+                    ).is_ok() {
+                        info!("[{}/{}] {} progress: {}% ({}/{} candles)", 
+                            idx, total_symbols, symbol_name, milestone, count, estimated_total);
+                    }
+                }
+                
+                // Forward to database
+                let _ = db_tx.send(candle).await;
+            }
+        });
+
+        // Database consumer
+        let db_clone = db.clone();
+        let consumer_handle = tokio::spawn(async move {
+            db_clone.start_consumer(db_rx).await;
+        });
+
+        // Sync this symbol
+        let result = BinanceCollector::sync_from_scratch(
+            symbol.symbol.clone(),
+            symbol.start_timestamp,
+            clients.clone(),
+            tx,
+        ).await;
+
+        // Wait for tasks to finish
+        let _ = progress_handle.await;
+        let _ = consumer_handle.await;
+
+        let final_count = candle_count.load(Ordering::Relaxed);
+        
+        match result {
+            Ok(count) => {
+                info!("[{}/{}] {} completed: {} candles", 
+                    i + 1, total_symbols, symbol.symbol, count);
+            }
+            Err(e) => {
+                error!("[{}/{}] {} failed: {} (received: {} candles)", 
+                    i + 1, total_symbols, symbol.symbol, e, final_count);
+            }
+        }
+    }
+
+    info!("All symbols synced!");
+}

backend/examples/sync_all_fast.rs

@@ -0,0 +1,162 @@
+use backend::{BinanceCollector, CandleData, DatabaseHandler, HistoricalDownloader};
+
+use log::{info, error};
+use dotenv::dotenv;
+use tokio::sync::mpsc;
+use std::sync::Arc;
+use std::sync::atomic::{AtomicU64, Ordering};
+
+#[tokio::main]
+async fn main() {
+    dotenv().ok();
+    env_logger::init();
+
+    // 1. Connect to database
+    let database_url = std::env::var("DATABASE_URL")
+        .expect("DATABASE_URL must be set");
+    
+    info!("Connecting to database...");
+    let db = Arc::new(
+        DatabaseHandler::new(&database_url)
+            .await
+            .expect("Failed to connect to database")
+    );
+
+    // 2. Get all USDT symbols
+    info!("Fetching all symbols from Binance...");
+    let all_symbols = BinanceCollector::get_symbol().await.unwrap();
+    let symbols: Vec<_> = all_symbols
+        .into_iter()
+        .filter(|s| s.symbol.ends_with("USDT"))
+        .collect();
+    info!("Got {} USDT symbols to sync", symbols.len());
+
+    // 3. Build clients
+    info!("Building download clients (with proxies)...");
+    let download_clients = HistoricalDownloader::build_clients().await;
+    
+    info!("Building API clients (for recent data)...");
+    let api_clients = BinanceCollector::build_clients().await;
+    info!("Built {} download clients, {} API clients", download_clients.len(), api_clients.len());
+
+    // Get last complete month timestamp
+    let archive_end = HistoricalDownloader::last_complete_month_end();
+    info!("Archive data available until: {}", archive_end);
+
+    // Use the global cutoff timestamp
+    info!("Data cutoff: 2024-01-01 00:00:00 UTC (timestamp: {})", backend::DATA_CUTOFF_TIMESTAMP);
+
+    // 4. Sync each symbol
+    let total_symbols = symbols.len();
+
+    for (i, symbol) in symbols.into_iter().enumerate() {
+        info!("========================================");
+        info!("[{}/{}] {} - Starting fast sync", i + 1, total_symbols, symbol.symbol);
+
+        // Check if we have data already
+        let latest_ts = db.get_latest_timestamp(&symbol.symbol).await.ok().flatten();
+
+        let start_time = match latest_ts {
+            Some(ts) => {
+                info!("[{}/{}] {} - Has data until {}, continuing from there",
+                    i + 1, total_symbols, symbol.symbol, ts);
+                ts + 60000
+            }
+            None => {
+                // No existing data, use the later of symbol start or cutoff
+                let actual_start = symbol.start_timestamp.max(backend::DATA_CUTOFF_TIMESTAMP);
+                if symbol.start_timestamp < backend::DATA_CUTOFF_TIMESTAMP {
+                    info!("[{}/{}] {} - Original start is before 2024-01-01, starting from cutoff instead",
+                        i + 1, total_symbols, symbol.symbol);
+                }
+                actual_start
+            }
+        };
+
+        // Create channels
+        let (tx, rx) = mpsc::channel::<CandleData>(500000);
+        let (db_tx, db_rx) = mpsc::channel::<CandleData>(500000);
+        
+        // Progress tracking
+        let candle_count = Arc::new(AtomicU64::new(0));
+        let candle_count_clone = candle_count.clone();
+        let symbol_name = symbol.symbol.clone();
+        let idx = i + 1;
+        
+        // Progress forwarder
+        let progress_handle = tokio::spawn(async move {
+            let mut rx = rx;
+            let mut last_report = 0u64;
+            
+            while let Some(candle) = rx.recv().await {
+                let count = candle_count_clone.fetch_add(1, Ordering::Relaxed) + 1;
+                
+                if count - last_report >= 500000 {
+                    info!("[{}/{}] {} - Progress: {} candles", idx, total_symbols, symbol_name, count);
+                    last_report = count;
+                }
+                
+                let _ = db_tx.send(candle).await;
+            }
+        });
+
+        // DB consumer (batch insert)
+        let db_clone = db.clone();
+        let consumer_handle = tokio::spawn(async move {
+            db_clone.start_consumer(db_rx).await;
+        });
+
+        // STEP 1: Download historical data from archive (with proxies)
+        if start_time < archive_end && !download_clients.is_empty() {
+            info!("[{}/{}] {} - Downloading from archive with {} proxies...", 
+                i + 1, total_symbols, symbol.symbol, download_clients.len());
+            
+            match HistoricalDownloader::download_symbol_with_clients(
+                &symbol.symbol,
+                start_time,
+                &download_clients,
+                tx.clone(),
+            ).await {
+                Ok(count) => info!("[{}/{}] {} - Archive download: {} candles", 
+                    i + 1, total_symbols, symbol.symbol, count),
+                Err(e) => error!("[{}/{}] {} - Archive download failed: {}", 
+                    i + 1, total_symbols, symbol.symbol, e),
+            }
+        }
+
+        // STEP 2: Sync recent data via API
+        let api_start = archive_end.max(start_time);
+        let now = std::time::SystemTime::now()
+            .duration_since(std::time::UNIX_EPOCH)
+            .unwrap()
+            .as_millis() as i64;
+        
+        if now - api_start > 60000 && !api_clients.is_empty() {
+            info!("[{}/{}] {} - Syncing recent data via API...", i + 1, total_symbols, symbol.symbol);
+            
+            match BinanceCollector::sync_from_scratch(
+                symbol.symbol.clone(),
+                api_start,
+                api_clients.clone(),
+                tx.clone(),
+            ).await {
+                Ok(count) => info!("[{}/{}] {} - API sync: {} candles", 
+                    i + 1, total_symbols, symbol.symbol, count),
+                Err(e) => error!("[{}/{}] {} - API sync failed: {}", 
+                    i + 1, total_symbols, symbol.symbol, e),
+            }
+        }
+
+        // Close channel and wait
+        drop(tx);
+        let _ = progress_handle.await;
+        let _ = consumer_handle.await;
+
+        let final_count = candle_count.load(Ordering::Relaxed);
+        info!("[{}/{}] {} - Complete: {} total candles", 
+            i + 1, total_symbols, symbol.symbol, final_count);
+    }
+
+    info!("========================================");
+    info!("All symbols synced!");
+}