diff --git a/Cargo.lock b/Cargo.lock index 978f9ef..a460f25 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1832,6 +1832,7 @@ name = "mtgc-core" version = "0.1.0" dependencies = [ "anyhow", + "flate2", "nucleo", "rusqlite", "serde", diff --git a/Cargo.toml b/Cargo.toml index efd22f2..7bb0f2e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,6 +13,11 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" nucleo = "0.5" ureq = { version = "2", features = ["json"] } +# Scryfall serves the bulk downloads as `.jsonl.gz` with `Content-Type: +# application/gzip` (not `Content-Encoding: gzip`), so ureq won't transparently +# decompress them — we do it ourselves. Pure-Rust backend, so it cross-compiles +# to Android without another C dependency. +flate2 = { version = "1", default-features = false, features = ["rust_backend"] } anyhow = "1" thiserror = "1" diff --git a/crates/mtgc-core/Cargo.toml b/crates/mtgc-core/Cargo.toml index 88f1fc0..2262adc 100644 --- a/crates/mtgc-core/Cargo.toml +++ b/crates/mtgc-core/Cargo.toml @@ -20,6 +20,7 @@ serde = { workspace = true } serde_json = { workspace = true } nucleo = { workspace = true } ureq = { workspace = true } +flate2 = { workspace = true } anyhow = { workspace = true } thiserror = { workspace = true } diff --git a/crates/mtgc-core/src/ingest.rs b/crates/mtgc-core/src/ingest.rs index 478a0d9..fa5e095 100644 --- a/crates/mtgc-core/src/ingest.rs +++ b/crates/mtgc-core/src/ingest.rs @@ -4,17 +4,18 @@ //! `oracle_cards` + the normalized type/subtype/keyword tables + the name FTS; //! the *Default Cards* bulk (one entry per printing) fills `printings`. //! -//! Both bulk files are large JSON arrays, so we stream them element-by-element -//! rather than loading the whole array into memory. +//! Both bulk files are large gzipped JSONL (one card object per line), so we +//! decompress and parse them record-by-record rather than holding a file in +//! memory. -use std::io::{BufReader, Read}; -use std::marker::PhantomData; +use std::io::{BufRead, BufReader, Read}; use std::path::Path; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; +use flate2::read::GzDecoder; use rusqlite::{params, Connection}; -use serde::de::{DeserializeOwned, SeqAccess, Visitor}; +use serde::de::DeserializeOwned; use serde::Deserialize; use crate::db::open_reference_for_build; @@ -56,10 +57,10 @@ impl CancelToken { } } -/// A cancellation-triggered early exit from inside `stream_json_array`'s -/// per-item closure round-trips through `serde_json::Error::custom`, which -/// stringifies (and appends position info to) whatever error it's given. Undo -/// that mangling here so callers can match on a clean [`CoreError::Cancelled`]. +/// Report a clean [`CoreError::Cancelled`] whenever the token is set, whatever +/// error actually surfaced. Cancelling mid-download typically aborts the reader +/// first, so the raw error is an unhelpful I/O or parse failure describing the +/// symptom rather than the cause. fn override_if_cancelled(e: CoreError, cancel: &CancelToken) -> CoreError { if cancel.is_cancelled() { CoreError::Cancelled @@ -202,6 +203,8 @@ fn build_into( stages: stages.clone(), }); let reader = http_reader(&bulk.oracle_uri)?; + // Progress wraps the *compressed* stream, since `compressed_size` is what + // the bulk index reports; decompression happens after the byte count. let reader = progress_reader( reader, bulk.oracle_size, @@ -210,6 +213,7 @@ fn build_into( &stages, progress, ); + let reader = GzDecoder::new(reader); let oracle_cards = load_oracle_cards(&conn, reader, cancel).map_err(|e| override_if_cancelled(e, cancel))?; progress(IngestProgress { @@ -227,6 +231,7 @@ fn build_into( &stages, progress, ); + let reader = GzDecoder::new(reader); let printings = load_printings(&conn, reader, cancel).map_err(|e| override_if_cancelled(e, cancel))?; if cancel.is_cancelled() { @@ -321,11 +326,15 @@ struct BulkIndex { struct BulkEntry { #[serde(rename = "type")] kind: String, - download_uri: String, - /// Uncompressed byte size of `download_uri`'s contents, per the Scryfall - /// bulk-data API — used for download progress since the actual download - /// response is chunked (no `Content-Length` header to rely on). - size: u64, + /// Scryfall serves gzipped JSONL (one card object per line). This replaced + /// the older `download_uri`/`size` pair, which pointed at an uncompressed + /// JSON array — a db built before that change fails here with + /// "missing field `download_uri`". + jsonl_download_uri: String, + /// *Compressed* byte size, used for download progress since the response + /// is chunked (no `Content-Length` to rely on). Progress is therefore + /// measured on the compressed stream, before decompression. + compressed_size: u64, } struct BulkUris { @@ -367,10 +376,10 @@ fn fetch_bulk_uris() -> Result { let oracle = find("oracle_cards")?; let default = find("default_cards")?; Ok(BulkUris { - oracle_uri: oracle.download_uri.clone(), - oracle_size: oracle.size, - default_uri: default.download_uri.clone(), - default_size: default.size, + oracle_uri: oracle.jsonl_download_uri.clone(), + oracle_size: oracle.compressed_size, + default_uri: default.jsonl_download_uri.clone(), + default_size: default.compressed_size, }) } @@ -379,51 +388,29 @@ fn http_reader(url: &str) -> Result { Ok(BufReader::with_capacity(1 << 20, resp.into_reader())) } -// ---- Streaming array parse ---------------------------------------------- +// ---- Streaming JSONL parse ---------------------------------------------- -/// Deserialize a top-level JSON array element-by-element, invoking `f` per item -/// without holding the whole array in memory. -fn stream_json_array(reader: R, f: F) -> Result<()> +/// Deserialize newline-delimited JSON record-by-record, invoking `f` per record +/// without holding the whole file in memory. +/// +/// Scryfall's bulk downloads are JSONL — one complete card object per line — +/// so a line reader replaces what used to be a streaming JSON-array visitor. +/// An error from `f` propagates directly (it is no longer laundered through +/// `serde::de::Error::custom`), which keeps cancellation reporting exact. +fn stream_jsonl(reader: R, mut f: F) -> Result<()> where R: Read, T: DeserializeOwned, F: FnMut(T) -> Result<()>, { - struct SeqVisitor { - f: F, - _marker: PhantomData T>, - } - - impl<'de, T, F> Visitor<'de> for SeqVisitor - where - T: DeserializeOwned, - F: FnMut(T) -> Result<()>, - { - type Value = (); - - fn expecting(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result { - fmt.write_str("a JSON array of objects") - } - - fn visit_seq(mut self, mut seq: A) -> std::result::Result<(), A::Error> - where - A: SeqAccess<'de>, - { - while let Some(item) = seq.next_element::()? { - (self.f)(item).map_err(serde::de::Error::custom)?; - } - Ok(()) + for line in BufReader::with_capacity(1 << 20, reader).lines() { + let line = line?; + // Trailing newline at EOF, and any blank separator lines. + if line.trim().is_empty() { + continue; } + f(serde_json::from_str(&line)?)?; } - - let mut de = serde_json::Deserializer::from_reader(reader); - serde::de::Deserializer::deserialize_seq( - &mut de, - SeqVisitor { - f, - _marker: PhantomData, - }, - )?; Ok(()) } @@ -503,7 +490,7 @@ fn load_oracle_cards(conn: &Connection, reader: impl Read, cancel: &CancelToken) let mut ins_kw = conn.prepare("INSERT INTO card_keywords(oracle_id,keyword) VALUES (?1,?2)")?; - stream_json_array::<_, ScryOracle, _>(reader, |c| { + stream_jsonl::<_, ScryOracle, _>(reader, |c| { if cancel.is_cancelled() { return Err(CoreError::Cancelled); } @@ -732,7 +719,7 @@ fn load_printings(conn: &Connection, reader: impl Read, cancel: &CancelToken) -> (scryfall_id,oracle_id,set_code,collector_number,finishes,rarity,image_uri,back_image_uri,price_usd) VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)", )?; - stream_json_array::<_, ScryPrinting, _>(reader, |p| { + stream_jsonl::<_, ScryPrinting, _>(reader, |p| { if cancel.is_cancelled() { return Err(CoreError::Cancelled); } @@ -790,6 +777,19 @@ fn load_printings(conn: &Connection, reader: impl Read, cancel: &CancelToken) -> mod tests { use super::*; + /// Fixtures below are written as pretty-printed JSON arrays because that + /// stays readable in-source; the real bulk files are JSONL. Convert here so + /// the tests still drive the production JSONL parser rather than a + /// test-only code path. + fn jsonl(array_json: &str) -> String { + serde_json::from_str::>(array_json) + .expect("fixture is a JSON array") + .iter() + .map(|v| v.to_string()) + .collect::>() + .join("\n") + } + #[test] fn parses_type_lines() { let (t, s) = parse_type_line("Legendary Creature — Elf Druid"); @@ -807,14 +807,16 @@ mod tests { } #[test] - fn streams_array_of_objects() { + fn streams_jsonl_records() { #[derive(serde::Deserialize)] struct Item { n: i32, } - let json = br#"[{"n":1},{"n":2},{"n":3}]"#; + // Blank lines and a trailing newline are tolerated, as real bulk files + // end with one. + let jsonl = b"{\"n\":1}\n{\"n\":2}\n\n{\"n\":3}\n"; let mut sum = 0; - stream_json_array::<_, Item, _>(&json[..], |it| { + stream_jsonl::<_, Item, _>(&jsonl[..], |it| { sum += it.n; Ok(()) }) @@ -822,6 +824,21 @@ mod tests { assert_eq!(sum, 6); } + #[test] + fn stream_jsonl_propagates_the_closure_error_unmangled() { + #[derive(serde::Deserialize)] + struct Item { + #[allow(dead_code)] // only the deserialization has to succeed here + n: i32, + } + // Cancellation reporting depends on this: the old array-visitor path + // laundered closure errors through `serde::de::Error::custom`, turning + // them into strings. + let jsonl = b"{\"n\":1}\n{\"n\":2}\n"; + let err = stream_jsonl::<_, Item, _>(&jsonl[..], |_| Err(CoreError::Cancelled)).unwrap_err(); + assert!(matches!(err, CoreError::Cancelled), "got {err:?}"); + } + #[test] fn excludes_art_series_cards() { let oracle = r#"[ @@ -836,7 +853,7 @@ mod tests { let dest = dir.path().join("reference.db"); let conn = open_reference_for_build(&dest).unwrap(); - let oc = load_oracle_cards(&conn, oracle.as_bytes(), &CancelToken::new()).unwrap(); + let oc = load_oracle_cards(&conn, jsonl(oracle).as_bytes(), &CancelToken::new()).unwrap(); assert_eq!(oc, 1); let count: i64 = conn @@ -866,7 +883,7 @@ mod tests { let cancel = CancelToken::new(); cancel.cancel(); - let err = load_oracle_cards(&conn, oracle.as_bytes(), &cancel) + let err = load_oracle_cards(&conn, jsonl(oracle).as_bytes(), &cancel) .map_err(|e| override_if_cancelled(e, &cancel)) .unwrap_err(); assert!(matches!(err, CoreError::Cancelled)); @@ -908,8 +925,8 @@ mod tests { let dest = dir.path().join("reference.db"); let conn = open_reference_for_build(&dest).unwrap(); - let oc = load_oracle_cards(&conn, oracle.as_bytes(), &CancelToken::new()).unwrap(); - let pc = load_printings(&conn, printings.as_bytes(), &CancelToken::new()).unwrap(); + let oc = load_oracle_cards(&conn, jsonl(oracle).as_bytes(), &CancelToken::new()).unwrap(); + let pc = load_printings(&conn, jsonl(printings).as_bytes(), &CancelToken::new()).unwrap(); assert_eq!(oc, 2); assert_eq!(pc, 1); @@ -956,7 +973,7 @@ mod tests { let dest = dir.path().join("reference.db"); let conn = open_reference_for_build(&dest).unwrap(); - load_oracle_cards(&conn, oracle.as_bytes(), &CancelToken::new()).unwrap(); + load_oracle_cards(&conn, jsonl(oracle).as_bytes(), &CancelToken::new()).unwrap(); let (type_line, colors, oracle_text): (String, i64, String) = conn .query_row( @@ -987,7 +1004,7 @@ mod tests { let dest = dir.path().join("reference.db"); let conn = open_reference_for_build(&dest).unwrap(); - load_oracle_cards(&conn, oracle.as_bytes(), &CancelToken::new()).unwrap(); + load_oracle_cards(&conn, jsonl(oracle).as_bytes(), &CancelToken::new()).unwrap(); let fetch = |id: &str| -> (i64, i64) { conn.query_row( @@ -1025,7 +1042,7 @@ mod tests { let dest = dir.path().join("reference.db"); let conn = open_reference_for_build(&dest).unwrap(); - load_oracle_cards(&conn, oracle.as_bytes(), &CancelToken::new()).unwrap(); + load_oracle_cards(&conn, jsonl(oracle).as_bytes(), &CancelToken::new()).unwrap(); let fetch = |id: &str| -> Legalities { Legalities( @@ -1065,7 +1082,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let dest = dir.path().join("reference.db"); let conn = open_reference_for_build(&dest).unwrap(); - load_oracle_cards(&conn, oracle.as_bytes(), &CancelToken::new()).unwrap(); + load_oracle_cards(&conn, jsonl(oracle).as_bytes(), &CancelToken::new()).unwrap(); let fetch = |id: &str| -> (i64, Option) { conn.query_row( @@ -1108,7 +1125,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let dest = dir.path().join("reference.db"); let conn = open_reference_for_build(&dest).unwrap(); - load_oracle_cards(&conn, oracle.as_bytes(), &CancelToken::new()).unwrap(); + load_oracle_cards(&conn, jsonl(oracle).as_bytes(), &CancelToken::new()).unwrap(); let horizontal = |id: &str| -> bool { conn.query_row( @@ -1153,7 +1170,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let dest = dir.path().join("reference.db"); let conn = open_reference_for_build(&dest).unwrap(); - load_oracle_cards(&conn, oracle.as_bytes(), &CancelToken::new()).unwrap(); + load_oracle_cards(&conn, jsonl(oracle).as_bytes(), &CancelToken::new()).unwrap(); let images = |id: &str| -> (String, String, String) { conn.query_row( @@ -1206,8 +1223,8 @@ mod tests { let dest = dir.path().join("reference.db"); let conn = open_reference_for_build(&dest).unwrap(); - load_printings(&conn, dfc.as_bytes(), &CancelToken::new()).unwrap(); - load_printings(&conn, split.as_bytes(), &CancelToken::new()).unwrap(); + load_printings(&conn, jsonl(dfc).as_bytes(), &CancelToken::new()).unwrap(); + load_printings(&conn, jsonl(split).as_bytes(), &CancelToken::new()).unwrap(); let (front, back): (String, String) = conn .query_row( @@ -1245,7 +1262,7 @@ mod tests { let dest = dir.path().join("reference.db"); let conn = open_reference_for_build(&dest).unwrap(); - load_printings(&conn, printings.as_bytes(), &CancelToken::new()).unwrap(); + load_printings(&conn, jsonl(printings).as_bytes(), &CancelToken::new()).unwrap(); let fetch = |id: &str| -> f64 { conn.query_row( diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index 82f81f9..b676780 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -77,6 +77,8 @@ pub async fn update_card_data(app: AppHandle, db: Db<'_>) -> Res { db.lock().unwrap().ingest_cancel = Some(cancel.clone()); let app_for_progress = app.clone(); + crate::mlog!("update_card_data: starting ingest into {}", reference_path.display()); + let result = tauri::async_runtime::spawn_blocking({ let cancel = cancel.clone(); move || { @@ -89,10 +91,15 @@ pub async fn update_card_data(app: AppHandle, db: Db<'_>) -> Res { .map_err(|e| e.to_string())?; db.lock().unwrap().ingest_cancel = None; - result.map_err(|e| e.to_string())?; + let stats = crate::logged("update_card_data", result.map_err(|e| e.to_string()))?; + crate::mlog!( + "update_card_data: ok ({} oracle cards, {} printings)", + stats.oracle_cards, + stats.printings + ); let mut s = db.lock().unwrap(); - s.reopen()?; + crate::logged("update_card_data reopen", s.reopen())?; Ok(status_of(&s)) } @@ -113,9 +120,15 @@ pub fn import_reference_db(db: Db, path: String) -> Res { return Err("That file is already the active card database.".into()); } + crate::mlog!("import_reference_db: from {}", src.display()); + // Validate before disturbing the working db, so picking the wrong file is a // harmless error rather than a broken install. - mtgc_core::db::inspect_reference_db(&src).map_err(|e| e.to_string())?; + let cards = crate::logged( + "import_reference_db validate", + mtgc_core::db::inspect_reference_db(&src).map_err(|e| e.to_string()), + )?; + crate::mlog!("import_reference_db: source holds {cards} oracle cards"); // Release the handles on the file we're about to replace. s.close(); diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index eeaef73..aa04eb4 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -7,6 +7,30 @@ use tauri::Manager; use state::AppState; +/// Write a line to the platform log. +/// +/// On Android this reaches `adb logcat` under the tag `RustStdoutStderr`: tao's +/// Android glue `dup2`s stdout/stderr into a pipe and forwards each line to the +/// system log, so a plain `eprintln!` is all that's needed and no logging +/// dependency is warranted. On desktop it's just stderr. +/// +/// This matters because Tauri command failures are *returned* to the WebView as +/// values, not raised — so without logging them explicitly they never appear in +/// logcat at all, and the UI only ever shows a truncated version. +#[macro_export] +macro_rules! mlog { + ($($arg:tt)*) => { eprintln!("[mtgc] {}", format_args!($($arg)*)) }; +} + +/// Log a command failure (with its full, untruncated message) and pass the +/// result through unchanged. +pub(crate) fn logged(what: &str, result: Result) -> Result { + if let Err(e) = &result { + crate::mlog!("{what} failed: {e}"); + } + result +} + #[cfg_attr(mobile, tauri::mobile_entry_point)] pub fn run() { tauri::Builder::default() @@ -17,6 +41,7 @@ pub fn run() { .app_data_dir() .expect("resolve app data dir"); std::fs::create_dir_all(&data_dir).ok(); + mlog!("data dir: {}", data_dir.display()); app.manage(Mutex::new(AppState::new(data_dir))); Ok(()) })