Upload files.

backend/tests/connection_test.rs

@@ -0,0 +1,86 @@
+use backend::{BinanceCollector, CandleData};
+use tokio::sync::mpsc;
+use tokio::time::{timeout, Duration};
+use dotenv::dotenv;
+use log::{info,error,debug};
+
+#[tokio::test]
+async fn test_backfill_and_stream() {
+
+    dotenv().ok();
+    env_logger::init();
+
+    let collector = BinanceCollector::new(vec!["btcusdt".to_string(), "ethusdt".to_string()]);
+    let (tx, mut rx) = mpsc::channel::<CandleData>(1000);
+
+    // Calculate time range: last 10 minutes
+    let now = std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap()
+        .as_millis() as i64;
+    let start_time = now - (10 * 60 * 1000); // 10 minutes ago
+
+    // First, backfill recent data for btcusdt
+    info!("Starting backfill...");
+    let backfill_count = collector.backfill("btcusdt", start_time, now, &tx).await.unwrap();
+    info!("Backfill completed: {} candles", backfill_count);
+
+    // Then start WebSocket stream
+    let collector_handle = tokio::spawn(async move {
+        collector.start_stream(tx).await
+    });
+
+    let mut received_count = 0;
+    
+    // Drain backfill data first
+    while let Ok(Some(candle)) = timeout(Duration::from_millis(100), rx.recv()).await {
+        info!(
+            "[BACKFILL] {} ts={} c={:.2} net_vol={:.4}",
+            candle.symbol, candle.timestamp, candle.close, candle.net_volume
+        );
+        received_count += 1;
+    }
+    info!("Received {} backfill candles from channel", received_count);
+
+    // Wait for WebSocket data (up to 90 seconds for at least 1 closed candle)
+    let mut ws_count = 0;
+    let ws_result = timeout(Duration::from_secs(90), async {
+        while let Some(candle) = rx.recv().await {
+            info!(
+                "[WEBSOCKET] {} ts={} c={:.2} net_vol={:.4}",
+                candle.symbol, candle.timestamp, candle.close, candle.net_volume
+            );
+            ws_count += 1;
+            if ws_count >= 1 {
+                return true;
+            }
+        }
+        false
+    }).await;
+
+    collector_handle.abort();
+
+    info!("\nFinal: Backfill={}, WebSocket={}", backfill_count, ws_count);
+    
+    assert!(backfill_count >= 5, "Expected at least 5 backfill candles, got {}", backfill_count);
+    
+    match ws_result {
+        Ok(true) => info!("Test completed successfully!"),
+        Ok(false) => info!("WebSocket closed before receiving data"),
+        Err(_) => info!("WebSocket timed out (normal if test runs mid-minute)"),
+    }
+}
+
+#[tokio::test]
+
+async fn test_get_symbol() {
+
+    dotenv().ok();
+    env_logger::init();
+
+    let collector = BinanceCollector::new(vec!["btcusdt".to_string(), "ethusdt".to_string()]);
+
+    let result = BinanceCollector::get_symbol().await.unwrap();
+
+    info!("Result = {:?}",result);
+}

backend/tests/database_test.rs

@@ -0,0 +1,128 @@
+use backend::{DatabaseHandler, CandleData};
+use tokio::sync::mpsc;
+use dotenv::dotenv;
+use log::info;
+use std::env;
+
+fn get_database_url() -> String {
+
+    dotenv().ok();
+    env::var("DATABASE_URL").unwrap_or_else(|_| {
+        "postgres://quant:2Nr!Ya&oVvY5pp@172.18.0.10:5432/crypto_database".to_string()
+    })
+}
+
+#[tokio::test]
+async fn test_database_connection() {
+
+    dotenv().ok();
+    env_logger::init();
+    
+    let db_url = get_database_url();
+    info!("Connecting to database...");
+    
+    let db = DatabaseHandler::new(&db_url).await;
+    assert!(db.is_ok(), "Failed to connect to database: {:?}", db.err());
+    
+    info!("Database connection successful!");
+}
+
+#[tokio::test]
+async fn test_get_active_symbols() {
+
+    dotenv().ok();
+    env_logger::init();
+
+    let db_url = get_database_url();
+    let db = DatabaseHandler::new(&db_url).await.expect("Failed to connect");
+    
+    let symbols = db.get_active_symbols().await;
+    info!("Active symbols: {:?}", symbols);
+    
+    assert!(symbols.is_ok(), "Failed to get active symbols: {:?}", symbols.err());
+}
+
+#[tokio::test]
+async fn test_insert_and_query() {
+
+    dotenv().ok();
+    env_logger::init();
+
+    let db_url = get_database_url();
+    let db = DatabaseHandler::new(&db_url).await.expect("Failed to connect");
+    
+    // Create test candle
+    let test_candle = CandleData {
+        symbol: "TESTUSDT".to_string(),
+        timestamp: 1700000000000, // Fixed test timestamp
+        open: 100.0,
+        high: 105.0,
+        low: 99.0,
+        close: 102.0,
+        volume: 1000.0,
+        taker_buy_volume: 600.0,
+        net_volume: 200.0,
+        is_closed: true,
+    };
+    
+    // Insert
+    let result = db.insert_candle(&test_candle).await;
+    assert!(result.is_ok(), "Failed to insert candle: {:?}", result.err());
+    info!("Inserted test candle");
+    
+    // Query latest timestamp
+    let latest = db.get_latest_timestamp("TESTUSDT").await;
+    assert!(latest.is_ok(), "Failed to get latest timestamp: {:?}", latest.err());
+    
+    let ts = latest.unwrap();
+    assert!(ts.is_some(), "No timestamp found for TESTUSDT");
+    assert_eq!(ts.unwrap(), 1700000000000, "Timestamp mismatch");
+    
+    info!("Latest timestamp for TESTUSDT: {:?}", ts);
+}
+
+#[tokio::test]
+async fn test_batch_consumer() {
+
+    dotenv().ok();
+    env_logger::init();
+
+    let db_url = get_database_url();
+    let db = DatabaseHandler::new(&db_url).await.expect("Failed to connect");
+    
+    let (tx, rx) = mpsc::channel::<CandleData>(100);
+    
+    // Spawn consumer
+    let db_handle = tokio::spawn(async move {
+        db.start_consumer(rx).await;
+    });
+    
+    // Send test candles
+    let base_ts = 1700000100000i64;
+    for i in 0..10 {
+        let candle = CandleData {
+            symbol: "BATCHTEST".to_string(),
+            timestamp: base_ts + (i * 60000),
+            open: 100.0 + i as f64,
+            high: 105.0 + i as f64,
+            low: 99.0 + i as f64,
+            close: 102.0 + i as f64,
+            volume: 1000.0,
+            taker_buy_volume: 600.0,
+            net_volume: 200.0,
+            is_closed: true,
+        };
+        tx.send(candle).await.unwrap();
+    }
+    
+    // Close channel to trigger flush
+    drop(tx);
+    
+    // Wait for consumer to finish
+    let _ = tokio::time::timeout(
+        tokio::time::Duration::from_secs(10),
+        db_handle
+    ).await;
+    
+    info!("Batch insert test completed");
+}

backend/tests/scheduler_test.rs

@@ -0,0 +1,322 @@
+use backend::{
+    Scheduler, DatabaseHandler, SchedulerCommand,
+    create_command_channel,
+};
+use std::sync::Arc;
+use tokio::sync::oneshot;
+use tokio::time::{timeout, Duration};
+use dotenv::dotenv;
+use log::info;
+use std::env;
+
+fn get_database_url() -> String {
+    dotenv().ok();
+    env::var("DATABASE_URL").unwrap_or_else(|_| {
+        "postgres://quant:2Nr!Ya&oVvY5pp@172.18.0.2:5432/crypto_database".to_string()
+    })
+}
+
+#[tokio::test]
+async fn test_scheduler_startup_and_shutdown() {
+
+    dotenv().ok();
+    env_logger::try_init().ok();
+    
+    let db_url = get_database_url();
+    let db = Arc::new(DatabaseHandler::new(&db_url).await.expect("Failed to connect to database"));
+    
+    let (command_tx, command_rx) = create_command_channel();
+    let mut scheduler = Scheduler::new(db.clone(), command_rx, None);
+    
+    // Start scheduler in background
+    let scheduler_handle = tokio::spawn(async move {
+        scheduler.run().await;
+    });
+    
+    // Give it time to start
+    tokio::time::sleep(Duration::from_secs(2)).await;
+    
+    // Send shutdown command
+    command_tx.send(SchedulerCommand::Shutdown).await.expect("Failed to send shutdown");
+    
+    // Wait for scheduler to stop
+    let result = timeout(Duration::from_secs(10), scheduler_handle).await;
+    assert!(result.is_ok(), "Scheduler did not shutdown in time");
+    
+    info!("Scheduler startup and shutdown test passed");
+}
+
+#[tokio::test]
+async fn test_get_status() {
+
+    dotenv().ok();
+    env_logger::try_init().ok();
+    
+    let db_url = get_database_url();
+    let db = Arc::new(DatabaseHandler::new(&db_url).await.expect("Failed to connect to database"));
+    
+    let (command_tx, command_rx) = create_command_channel();
+    let mut scheduler = Scheduler::new(db.clone(), command_rx, None);
+    
+    // Start scheduler
+    let scheduler_handle = tokio::spawn(async move {
+        scheduler.run().await;
+    });
+    
+    // Give it time to start
+    tokio::time::sleep(Duration::from_secs(2)).await;
+    
+    // Request status
+    let (reply_tx, reply_rx) = oneshot::channel();
+    command_tx.send(SchedulerCommand::GetStatus { reply: reply_tx }).await.expect("Failed to send GetStatus");
+    
+    let status = timeout(Duration::from_secs(5), reply_rx).await
+        .expect("Timeout waiting for status")
+        .expect("Failed to receive status");
+    
+    info!("Scheduler status: {:?}", status);
+    assert!(status.is_running || status.active_symbols.is_empty(), "Scheduler should be running or have no symbols");
+    
+    // Shutdown
+    command_tx.send(SchedulerCommand::Shutdown).await.expect("Failed to send shutdown");
+    let _ = timeout(Duration::from_secs(10), scheduler_handle).await;
+    
+    info!("GetStatus test passed");
+}
+
+#[tokio::test]
+async fn test_add_and_remove_symbol() {
+
+    dotenv().ok();
+    env_logger::try_init().ok();
+    
+    let db_url = get_database_url();
+    let db = Arc::new(DatabaseHandler::new(&db_url).await.expect("Failed to connect to database"));
+    
+    let test_symbol = "testscheduler";
+    
+    // Clean up first - remove test symbol if exists
+    let _ = db.remove_symbol(test_symbol).await;
+    
+    let (command_tx, command_rx) = create_command_channel();
+    let mut scheduler = Scheduler::new(db.clone(), command_rx, None);
+    
+    // Start scheduler
+    let scheduler_handle = tokio::spawn(async move {
+        scheduler.run().await;
+    });
+    
+    // Give it time to start
+    tokio::time::sleep(Duration::from_secs(2)).await;
+    
+    // Add symbol (with recent backfill_from to avoid long backfill)
+    let now = std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap()
+        .as_millis() as i64;
+    let recent_time = now - (5 * 60 * 1000); // 5 minutes ago
+    
+    info!("Adding test symbol: {}", test_symbol);
+    command_tx.send(SchedulerCommand::AddSymbol { 
+        symbol: test_symbol.to_string(),
+        backfill_from: Some(recent_time),
+    }).await.expect("Failed to send AddSymbol");
+    
+    // Wait for processing
+    tokio::time::sleep(Duration::from_secs(5)).await;
+    
+    // Check status
+    let (reply_tx, reply_rx) = oneshot::channel();
+    command_tx.send(SchedulerCommand::GetStatus { reply: reply_tx }).await.expect("Failed to send GetStatus");
+    
+    let status = timeout(Duration::from_secs(5), reply_rx).await
+        .expect("Timeout")
+        .expect("Failed to receive");
+    
+    info!("Status after add: {:?}", status);
+    
+    // Verify symbol was added
+    let is_tracked = db.is_symbol_tracked(test_symbol).await.expect("Failed to check tracking");
+    assert!(is_tracked, "Symbol should be tracked after AddSymbol");
+    
+    // Remove symbol
+    info!("Removing test symbol: {}", test_symbol);
+    command_tx.send(SchedulerCommand::RemoveSymbol { 
+        symbol: test_symbol.to_string(),
+    }).await.expect("Failed to send RemoveSymbol");
+    
+    // Wait for processing
+    tokio::time::sleep(Duration::from_secs(3)).await;
+    
+    // Verify symbol was removed
+    let is_tracked = db.is_symbol_tracked(test_symbol).await.expect("Failed to check tracking");
+    assert!(!is_tracked, "Symbol should not be tracked after RemoveSymbol");
+    
+    // Shutdown
+    command_tx.send(SchedulerCommand::Shutdown).await.expect("Failed to send shutdown");
+    let _ = timeout(Duration::from_secs(10), scheduler_handle).await;
+    
+    info!("Add and remove symbol test passed");
+}
+
+#[tokio::test]
+async fn test_restart_collector() {
+
+    dotenv().ok();
+    env_logger::try_init().ok();
+    
+    let db_url = get_database_url();
+    let db = Arc::new(DatabaseHandler::new(&db_url).await.expect("Failed to connect to database"));
+    
+    let (command_tx, command_rx) = create_command_channel();
+    let mut scheduler = Scheduler::new(db.clone(), command_rx, None);
+    
+    // Start scheduler
+    let scheduler_handle = tokio::spawn(async move {
+        scheduler.run().await;
+    });
+    
+    // Give it time to start
+    tokio::time::sleep(Duration::from_secs(2)).await;
+    
+    // Get initial status
+    let (reply_tx, reply_rx) = oneshot::channel();
+    command_tx.send(SchedulerCommand::GetStatus { reply: reply_tx }).await.unwrap();
+    let status_before = reply_rx.await.unwrap();
+    info!("Status before restart: {:?}", status_before);
+    
+    // Send restart command
+    info!("Sending RestartCollector command");
+    command_tx.send(SchedulerCommand::RestartCollector).await.expect("Failed to send RestartCollector");
+    
+    // Wait for restart
+    tokio::time::sleep(Duration::from_secs(5)).await;
+    
+    // Get status after restart
+    let (reply_tx, reply_rx) = oneshot::channel();
+    command_tx.send(SchedulerCommand::GetStatus { reply: reply_tx }).await.unwrap();
+    let status_after = reply_rx.await.unwrap();
+    info!("Status after restart: {:?}", status_after);
+    
+    // Shutdown
+    command_tx.send(SchedulerCommand::Shutdown).await.expect("Failed to send shutdown");
+    let _ = timeout(Duration::from_secs(10), scheduler_handle).await;
+    
+    info!("Restart collector test passed");
+}
+
+#[tokio::test]
+async fn test_multiple_commands() {
+
+    dotenv().ok();
+    env_logger::try_init().ok();
+    
+    let db_url = get_database_url();
+    let db = Arc::new(DatabaseHandler::new(&db_url).await.expect("Failed to connect to database"));
+    
+    let (command_tx, command_rx) = create_command_channel();
+    let mut scheduler = Scheduler::new(db.clone(), command_rx, None);
+    
+    // Start scheduler
+    let scheduler_handle = tokio::spawn(async move {
+        scheduler.run().await;
+    });
+    
+    tokio::time::sleep(Duration::from_secs(2)).await;
+    
+    // Send multiple status requests rapidly
+    for i in 0..5 {
+        let (reply_tx, reply_rx) = oneshot::channel();
+        command_tx.send(SchedulerCommand::GetStatus { reply: reply_tx }).await.unwrap();
+        let status = reply_rx.await.unwrap();
+        info!("Status request {}: {:?}", i, status);
+    }
+    
+    // Shutdown
+    command_tx.send(SchedulerCommand::Shutdown).await.expect("Failed to send shutdown");
+    let _ = timeout(Duration::from_secs(10), scheduler_handle).await;
+    
+    info!("Multiple commands test passed");
+}
+
+#[tokio::test]
+async fn test_real_btcusdt_collection() {
+
+    dotenv().ok();
+    env_logger::try_init().ok();
+    
+    let db_url = get_database_url();
+    let db = Arc::new(DatabaseHandler::new(&db_url).await.expect("Failed to connect to database"));
+    
+    let symbol = "btcusdt";
+    
+    // Clean up - remove from tracking first
+    let _ = db.remove_symbol(symbol).await;
+    
+    let (command_tx, command_rx) = create_command_channel();
+    let mut scheduler = Scheduler::new(db.clone(), command_rx, None);
+    
+    // Start scheduler
+    let scheduler_handle = tokio::spawn(async move {
+        scheduler.run().await;
+    });
+    
+    tokio::time::sleep(Duration::from_secs(2)).await;
+    
+    // Calculate start time: 5 minutes ago
+    let now = std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap()
+        .as_millis() as i64;
+    let five_min_ago = now - (5 * 60 * 1000);
+    
+    info!("Adding BTCUSDT, backfill from {} (5 min ago)", five_min_ago);
+    
+    // Add BTCUSDT with backfill from 5 minutes ago
+    command_tx.send(SchedulerCommand::AddSymbol { 
+        symbol: symbol.to_string(),
+        backfill_from: Some(five_min_ago),
+    }).await.expect("Failed to send AddSymbol");
+    
+    // Wait for backfill to complete
+    info!("Waiting for backfill...");
+    tokio::time::sleep(Duration::from_secs(10)).await;
+    
+    // Check latest timestamp in database
+    let latest_ts = db.get_latest_timestamp(symbol).await.expect("Failed to get timestamp");
+    info!("Latest BTCUSDT timestamp in DB: {:?}", latest_ts);
+    
+    assert!(latest_ts.is_some(), "Should have BTCUSDT data in database");
+    
+    // Wait for 2 more candles (about 2 minutes + buffer)
+    info!("Waiting for 2 live candles (~2.5 minutes)...");
+    tokio::time::sleep(Duration::from_secs(150)).await;
+    
+    // Check new latest timestamp
+    let new_latest_ts = db.get_latest_timestamp(symbol).await.expect("Failed to get timestamp");
+    info!("New latest BTCUSDT timestamp: {:?}", new_latest_ts);
+    
+    assert!(new_latest_ts.unwrap() > latest_ts.unwrap(), "Should have received new candles");
+    
+    // Get status
+    let (reply_tx, reply_rx) = oneshot::channel();
+    command_tx.send(SchedulerCommand::GetStatus { reply: reply_tx }).await.unwrap();
+    let status = reply_rx.await.unwrap();
+    info!("Final status: {:?}", status);
+    
+    assert!(status.active_symbols.contains(&symbol.to_string()), "BTCUSDT should be in active symbols");
+    
+    // Clean up - remove symbol
+    command_tx.send(SchedulerCommand::RemoveSymbol { 
+        symbol: symbol.to_string(),
+    }).await.expect("Failed to remove symbol");
+    
+    tokio::time::sleep(Duration::from_secs(2)).await;
+    
+    // Shutdown
+    command_tx.send(SchedulerCommand::Shutdown).await.expect("Failed to send shutdown");
+    let _ = timeout(Duration::from_secs(10), scheduler_handle).await;
+    
+    info!("Real BTCUSDT collection test passed!");
+    info!("Check database: SELECT * FROM klines_1m WHERE symbol = 'BTCUSDT' ORDER BY timestamp DESC LIMIT 10;");
+}

backend/tests/sync_full_history_test.rs

@@ -0,0 +1,65 @@
+use backend::{BinanceCollector, CandleData};
+use tokio::sync::mpsc;
+use dotenv::dotenv;
+use log::info;
+
+#[tokio::test]
+async fn test_sync_full_history_single_symbol() {
+    dotenv().ok();
+    env_logger::init();
+
+    // 1. Get all symbols
+    let symbol_vec = BinanceCollector::get_symbol().await.unwrap();
+    info!("Got {} symbols", symbol_vec.len());
+
+    // 2. Find ETHUSDT start time
+    let test_symbol = "ETHUSDT";
+    let test_start_time = symbol_vec
+        .iter()
+        .find(|item| item.symbol == test_symbol)
+        .map(|item| item.start_timestamp)
+        .expect("ETHUSDT not found");
+
+    info!("{} start_timestamp: {}", test_symbol, test_start_time);
+
+    // 3. Build clients (all clients work together on this symbol)
+    let clients = BinanceCollector::build_clients().await;
+    info!("Built {} clients", clients.len());
+    assert!(!clients.is_empty(), "Need at least 1 client");
+
+    // 4. Create channel
+    let (tx, mut rx) = mpsc::channel::<CandleData>(100000);
+
+    // 5. Sync from scratch (all clients work in parallel on different time segments)
+    info!("Starting full history sync for {} with {} clients...", test_symbol, clients.len());
+    
+    let sync_handle = tokio::spawn(async move {
+        BinanceCollector::sync_from_scratch(
+            test_symbol.to_string(),
+            test_start_time,
+            clients,  // Pass all clients
+            tx,
+        ).await
+    });
+
+    // 6. Consume and count
+    let mut count = 0u64;
+    let mut last_ts = 0i64;
+    
+    while let Some(candle) = rx.recv().await {
+        count += 1;
+        last_ts = candle.timestamp;
+        
+        if count % 100000 == 0 {
+            info!("Progress: {} candles, last_ts: {}", count, last_ts);
+        }
+    }
+
+    let result = sync_handle.await.unwrap();
+    
+    info!("Sync result: {:?}", result);
+    info!("Total received: {} candles", count);
+    info!("Last timestamp: {}", last_ts);
+
+    assert!(count > 0, "Should have synced some candles");
+}

backend/tests/sync_test.rs

@@ -0,0 +1,68 @@
+use backend::{BinanceCollector, CandleData};
+use tokio::sync::mpsc;
+use dotenv::dotenv;
+use log::info;
+
+#[tokio::test]
+async fn test_build_clients() {
+    dotenv().ok();
+    env_logger::init();
+
+    info!("Testing build_clients...");
+    
+    let clients = BinanceCollector::build_clients().await;
+    
+    info!("Built {} working clients", clients.len());
+    assert!(clients.len() > 0, "Should have at least 1 working client");
+}
+
+#[tokio::test]
+async fn test_sync_single_symbol() {
+    dotenv().ok();
+    let _ = env_logger::try_init();
+
+    info!("Testing sync_from_scratch for single symbol...");
+
+    // Build clients (all work together)
+    let clients = BinanceCollector::build_clients().await;
+    assert!(!clients.is_empty(), "Need at least 1 client");
+    info!("Built {} clients", clients.len());
+
+    // Create channel
+    let (tx, mut rx) = mpsc::channel::<CandleData>(10000);
+
+    // Sync last 5 minutes of BTCUSDT
+    let now = std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap()
+        .as_millis() as i64;
+    let start_time = now - (5 * 60 * 1000); // 5 minutes ago
+
+    info!("Syncing BTCUSDT from {} to {}", start_time, now);
+
+    let count = BinanceCollector::sync_from_scratch(
+        "BTCUSDT".to_string(),
+        start_time,
+        clients,  // All clients work together
+        tx,
+    ).await.unwrap();
+
+    info!("Sync returned {} candles", count);
+
+    // Drain channel
+    let mut received = 0;
+    while let Ok(candle) = rx.try_recv() {
+        info!(
+            "Candle: {} ts={} o={:.2} h={:.2} l={:.2} c={:.2} nv={:.4}",
+            candle.symbol, candle.timestamp, 
+            candle.open, candle.high, candle.low, candle.close,
+            candle.net_volume
+        );
+        received += 1;
+    }
+
+    info!("Received {} candles from channel", received);
+    assert!(count >= 3, "Expected at least 3 candles for 5 min, got {}", count);
+}
+
+