fix: recover Hyperliquid WS from idle-disconnect and bound TradeSnap requests

The monitor could hang indefinitely (last event 2026-06-01) after an idle
server disconnect. Root cause: InfoClient::new disables SDK reconnect, so on
disconnect the reader task emitted a single Message::NoData and exited without
dropping the channel sender; the old code swallowed NoData into the catch-all
arm and looped back to recv(), which then blocked forever.

- hyperliquid.rs: create the client with InfoClient::with_reconnect so the SDK
  auto-reconnects (~1s) and resubscribes UserEvents on the same channel. Handle
  Message::NoData explicitly for observability instead of swallowing it.
- Drop the 180s recv() timeout: the SDK does not forward Pong frames to the
  subscription channel, so a quiet market (no fills) is indistinguishable from a
  dead connection and any finite timeout caused spurious reconnects, each
  opening a blind window where fills (not backfilled on resubscribe) are missed.
- notion.rs: give the TradeSnap HTTP client a 15s timeout. Events are consumed
  serially, so a hung screenshot request would otherwise stall all trade syncing.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
0xcathiefish
2026-06-03 17:03:01 +00:00
parent 6d7b049416
commit be89765d80
2 changed files with 17 additions and 2 deletions
+13 -1
View File
@@ -47,7 +47,7 @@ impl HyperliquidMonitor {
loop { loop {
info!("Connecting to Hyperliquid InfoClient..."); info!("Connecting to Hyperliquid InfoClient...");
let mut info_client = match InfoClient::new(None, Some(base_url)).await { let mut info_client = match InfoClient::with_reconnect(None, Some(base_url)).await {
Ok(client) => client, Ok(client) => client,
Err(e) => { Err(e) => {
error!( error!(
@@ -174,6 +174,18 @@ impl HyperliquidMonitor {
Some(Message::Pong) => { Some(Message::Pong) => {
debug!("Received Pong from Hyperliquid WS"); debug!("Received Pong from Hyperliquid WS");
} }
Some(Message::NoData) => {
// Server idle-disconnect or dropped connection. Because the
// client is created with InfoClient::with_reconnect, the SDK
// auto-reconnects (~1s) and resubscribes UserEvents on the same
// channel, so we keep reading instead of tearing down here.
// NOTE: a short application-level recv() timeout was intentionally
// NOT added: the SDK does not forward Pong frames to the
// subscription channel, so a quiet market (no fills) is
// indistinguishable from a dead connection and any finite timeout
// would cause spurious reconnects and missed-fill blind windows.
warn!("Hyperliquid WS disconnected; SDK is auto-reconnecting...");
}
Some(other) => { Some(other) => {
debug!(msg = ?other, "Received other message from Hyperliquid WS"); debug!(msg = ?other, "Received other message from Hyperliquid WS");
} }
+4 -1
View File
@@ -231,7 +231,10 @@ impl NotionWriter {
format!("BINANCE:{}USDC.P", coin.to_uppercase()) format!("BINANCE:{}USDC.P", coin.to_uppercase())
}; };
let tradesnap_url = url.trim_end_matches('/'); let tradesnap_url = url.trim_end_matches('/');
let http_client = reqwest::Client::new(); let http_client = reqwest::Client::builder()
.timeout(std::time::Duration::from_secs(15))
.build()
.unwrap_or_else(|_| reqwest::Client::new());
let mut children = Vec::new(); let mut children = Vec::new();
let mut timeframes = Vec::new(); let mut timeframes = Vec::new();