trtr

commit 7042fe83c9192f8e84b888be7704c4f80c0ea874
parent 1cafc196da4d8f376cc86e95018574a960e2bff6
Author: ea <ea@ea.contact>
Date:   Tue, 26 May 2026 15:13:31 +0000

Expect gz comressed data

Diffstat:
M
Cargo.toml
 | 
1
+
M
src/data.rs
 | 
11
++++++++++-
M
src/main.rs
 | 
2
+-

3 files changed, 12 insertions(+), 2 deletions(-)
diff --git a/Cargo.toml b/Cargo.toml
@@ -13,3 +13,4 @@ csv = "1.3"
 uuid = { version = "1", features = ["v4"] }
 rand = "0.9"
 dashmap = "6"
+flate2 = "1"
diff --git a/src/data.rs b/src/data.rs
@@ -1,3 +1,7 @@
+use std::fs::File;
+use std::io::BufReader;
+
+use flate2::read::GzDecoder;
 use serde::Serialize;
 
 #[derive(Clone, Serialize)]
@@ -15,7 +19,12 @@ pub struct AppData {
 }
 
 pub fn load_csv(path: &str) -> Result<Vec<Candle>, Box<dyn std::error::Error + Send + Sync>> {
-    let mut rdr = csv::Reader::from_path(path)?;
+    let file = File::open(path)?;
+    let mut rdr = if path.ends_with(".gz") {
+        csv::Reader::from_reader(Box::new(BufReader::new(GzDecoder::new(file))) as Box<dyn std::io::Read>)
+    } else {
+        csv::Reader::from_reader(Box::new(BufReader::new(file)) as Box<dyn std::io::Read>)
+    };
     let mut candles = Vec::with_capacity(8_000_000);
 
     for result in rdr.records() {
diff --git a/src/main.rs b/src/main.rs
@@ -14,7 +14,7 @@ async fn main() {
     let t = Instant::now();
 
     let candles = tokio::task::spawn_blocking(|| {
-        data::load_csv("data/btc/btcusd_1-min_data.csv").expect("Failed to load CSV")
+        data::load_csv("data/btc/btcusd_1-min_data.csv.gz").expect("Failed to load CSV")
     })
     .await
     .unwrap();