Fix card-data download for Scryfall's new gzipped-JSONL bulk API

"Download card data" had been failing on every platform with "missing field
`download_uri`". Scryfall changed the bulk-data API: entries now carry
`jsonl_download_uri`/`compressed_size` instead of `download_uri`/`size`, and
the payload is gzipped JSONL (one card object per line) rather than an
uncompressed JSON array. An existing reference.db keeps working, which is why
this went unnoticed.

- Rename the BulkEntry fields and decompress with flate2 (the response is
  Content-Type: application/gzip with no Content-Encoding, so ureq won't
  transparently inflate it). rust_backend keeps it pure Rust, so nothing new
  has to cross-compile for Android.
- Replace the streaming JSON-array visitor with a line-based JSONL parser.
  Closure errors now propagate directly instead of being laundered through
  serde::de::Error::custom, so cancellation reports a clean Cancelled;
  override_if_cancelled stays as a safety net and its doc comment is corrected.
- Progress wraps the compressed stream, since compressed_size is what the index
  reports.
- Test fixtures stay readable pretty-printed JSON arrays via a jsonl() helper
  that converts them, so tests still exercise the production parser.

Downloads are far smaller now: 101MB compressed (24 + 77) versus ~740MB, which
matters a lot on a phone.

Also log command failures to the platform log. Tauri command errors are
returned to the WebView as values, so they never reached logcat and the UI only
showed them truncated -- this bug was invisible until then. mlog! is a plain
eprintln!, which tao's Android glue forwards to logcat under RustStdoutStderr,
so no logging dependency is needed. View with: adb logcat -s RustStdoutStderr

Verified against the live API: 36,242 oracle cards, 116,409 printings, trigram
FTS working, and the aarch64 APK still builds.

Cargo.lock

@@ -1832,6 +1832,7 @@ name = "mtgc-core"
 version = "0.1.0"
 dependencies = [
  "anyhow",
+ "flate2",
  "nucleo",
  "rusqlite",
  "serde",

Cargo.toml

@@ -13,6 +13,11 @@ serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 nucleo = "0.5"
 ureq = { version = "2", features = ["json"] }
+# Scryfall serves the bulk downloads as `.jsonl.gz` with `Content-Type:
+# application/gzip` (not `Content-Encoding: gzip`), so ureq won't transparently
+# decompress them — we do it ourselves. Pure-Rust backend, so it cross-compiles
+# to Android without another C dependency.
+flate2 = { version = "1", default-features = false, features = ["rust_backend"] }
 anyhow = "1"
 thiserror = "1"
 

crates/mtgc-core/Cargo.toml

@@ -20,6 +20,7 @@ serde = { workspace = true }
 serde_json = { workspace = true }
 nucleo = { workspace = true }
 ureq = { workspace = true }
+flate2 = { workspace = true }
 anyhow = { workspace = true }
 thiserror = { workspace = true }
 

crates/mtgc-core/src/ingest.rs

@@ -4,17 +4,18 @@
 //! `oracle_cards` + the normalized type/subtype/keyword tables + the name FTS;
 //! the *Default Cards* bulk (one entry per printing) fills `printings`.
 //!
-//! Both bulk files are large JSON arrays, so we stream them element-by-element
+//! Both bulk files are large gzipped JSONL (one card object per line), so we
-//! rather than loading the whole array into memory.
+//! decompress and parse them record-by-record rather than holding a file in
+//! memory.
 
-use std::io::{BufReader, Read};
+use std::io::{BufRead, BufReader, Read};
-use std::marker::PhantomData;
 use std::path::Path;
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Arc;
 
+use flate2::read::GzDecoder;
 use rusqlite::{params, Connection};
-use serde::de::{DeserializeOwned, SeqAccess, Visitor};
+use serde::de::DeserializeOwned;
 use serde::Deserialize;
 
 use crate::db::open_reference_for_build;
@@ -56,10 +57,10 @@ impl CancelToken {
     }
 }
 
-/// A cancellation-triggered early exit from inside `stream_json_array`'s
+/// Report a clean [`CoreError::Cancelled`] whenever the token is set, whatever
-/// per-item closure round-trips through `serde_json::Error::custom`, which
+/// error actually surfaced. Cancelling mid-download typically aborts the reader
-/// stringifies (and appends position info to) whatever error it's given. Undo
+/// first, so the raw error is an unhelpful I/O or parse failure describing the
-/// that mangling here so callers can match on a clean [`CoreError::Cancelled`].
+/// symptom rather than the cause.
 fn override_if_cancelled(e: CoreError, cancel: &CancelToken) -> CoreError {
     if cancel.is_cancelled() {
         CoreError::Cancelled
@@ -202,6 +203,8 @@ fn build_into(
         stages: stages.clone(),
     });
     let reader = http_reader(&bulk.oracle_uri)?;
+    // Progress wraps the *compressed* stream, since `compressed_size` is what
+    // the bulk index reports; decompression happens after the byte count.
     let reader = progress_reader(
         reader,
         bulk.oracle_size,
@@ -210,6 +213,7 @@ fn build_into(
         &stages,
         progress,
     );
+    let reader = GzDecoder::new(reader);
     let oracle_cards = load_oracle_cards(&conn, reader, cancel).map_err(|e| override_if_cancelled(e, cancel))?;
 
     progress(IngestProgress {
@@ -227,6 +231,7 @@ fn build_into(
         &stages,
         progress,
     );
+    let reader = GzDecoder::new(reader);
     let printings = load_printings(&conn, reader, cancel).map_err(|e| override_if_cancelled(e, cancel))?;
 
     if cancel.is_cancelled() {
@@ -321,11 +326,15 @@ struct BulkIndex {
 struct BulkEntry {
     #[serde(rename = "type")]
     kind: String,
-    download_uri: String,
+    /// Scryfall serves gzipped JSONL (one card object per line). This replaced
-    /// Uncompressed byte size of `download_uri`'s contents, per the Scryfall
+    /// the older `download_uri`/`size` pair, which pointed at an uncompressed
-    /// bulk-data API — used for download progress since the actual download
+    /// JSON array — a db built before that change fails here with
-    /// response is chunked (no `Content-Length` header to rely on).
+    /// "missing field `download_uri`".
-    size: u64,
+    jsonl_download_uri: String,
+    /// *Compressed* byte size, used for download progress since the response
+    /// is chunked (no `Content-Length` to rely on). Progress is therefore
+    /// measured on the compressed stream, before decompression.
+    compressed_size: u64,
 }
 
 struct BulkUris {
@@ -367,10 +376,10 @@ fn fetch_bulk_uris() -> Result<BulkUris> {
     let oracle = find("oracle_cards")?;
     let default = find("default_cards")?;
     Ok(BulkUris {
-        oracle_uri: oracle.download_uri.clone(),
+        oracle_uri: oracle.jsonl_download_uri.clone(),
-        oracle_size: oracle.size,
+        oracle_size: oracle.compressed_size,
-        default_uri: default.download_uri.clone(),
+        default_uri: default.jsonl_download_uri.clone(),
-        default_size: default.size,
+        default_size: default.compressed_size,
     })
 }
 
@@ -379,51 +388,29 @@ fn http_reader(url: &str) -> Result<impl Read> {
     Ok(BufReader::with_capacity(1 << 20, resp.into_reader()))
 }
 
-// ---- Streaming array parse ----------------------------------------------
+// ---- Streaming JSONL parse ----------------------------------------------
 
-/// Deserialize a top-level JSON array element-by-element, invoking `f` per item
+/// Deserialize newline-delimited JSON record-by-record, invoking `f` per record
-/// without holding the whole array in memory.
+/// without holding the whole file in memory.
-fn stream_json_array<R, T, F>(reader: R, f: F) -> Result<()>
+///
+/// Scryfall's bulk downloads are JSONL — one complete card object per line —
+/// so a line reader replaces what used to be a streaming JSON-array visitor.
+/// An error from `f` propagates directly (it is no longer laundered through
+/// `serde::de::Error::custom`), which keeps cancellation reporting exact.
+fn stream_jsonl<R, T, F>(reader: R, mut f: F) -> Result<()>
 where
     R: Read,
     T: DeserializeOwned,
     F: FnMut(T) -> Result<()>,
 {
-    struct SeqVisitor<T, F> {
+    for line in BufReader::with_capacity(1 << 20, reader).lines() {
-        f: F,
+        let line = line?;
-        _marker: PhantomData<fn() -> T>,
+        // Trailing newline at EOF, and any blank separator lines.
-    }
+        if line.trim().is_empty() {
-
+            continue;
-    impl<'de, T, F> Visitor<'de> for SeqVisitor<T, F>
-    where
-        T: DeserializeOwned,
-        F: FnMut(T) -> Result<()>,
-    {
-        type Value = ();
-
-        fn expecting(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
-            fmt.write_str("a JSON array of objects")
-        }
-
-        fn visit_seq<A>(mut self, mut seq: A) -> std::result::Result<(), A::Error>
-        where
-            A: SeqAccess<'de>,
-        {
-            while let Some(item) = seq.next_element::<T>()? {
-                (self.f)(item).map_err(serde::de::Error::custom)?;
-            }
-            Ok(())
         }
+        f(serde_json::from_str(&line)?)?;
     }
-
-    let mut de = serde_json::Deserializer::from_reader(reader);
-    serde::de::Deserializer::deserialize_seq(
-        &mut de,
-        SeqVisitor {
-            f,
-            _marker: PhantomData,
-        },
-    )?;
     Ok(())
 }
 
@@ -503,7 +490,7 @@ fn load_oracle_cards(conn: &Connection, reader: impl Read, cancel: &CancelToken)
         let mut ins_kw =
             conn.prepare("INSERT INTO card_keywords(oracle_id,keyword) VALUES (?1,?2)")?;
 
-        stream_json_array::<_, ScryOracle, _>(reader, |c| {
+        stream_jsonl::<_, ScryOracle, _>(reader, |c| {
             if cancel.is_cancelled() {
                 return Err(CoreError::Cancelled);
             }
@@ -732,7 +719,7 @@ fn load_printings(conn: &Connection, reader: impl Read, cancel: &CancelToken) ->
                (scryfall_id,oracle_id,set_code,collector_number,finishes,rarity,image_uri,back_image_uri,price_usd)
              VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)",
         )?;
-        stream_json_array::<_, ScryPrinting, _>(reader, |p| {
+        stream_jsonl::<_, ScryPrinting, _>(reader, |p| {
             if cancel.is_cancelled() {
                 return Err(CoreError::Cancelled);
             }
@@ -790,6 +777,19 @@ fn load_printings(conn: &Connection, reader: impl Read, cancel: &CancelToken) ->
 mod tests {
     use super::*;
 
+    /// Fixtures below are written as pretty-printed JSON arrays because that
+    /// stays readable in-source; the real bulk files are JSONL. Convert here so
+    /// the tests still drive the production JSONL parser rather than a
+    /// test-only code path.
+    fn jsonl(array_json: &str) -> String {
+        serde_json::from_str::<Vec<serde_json::Value>>(array_json)
+            .expect("fixture is a JSON array")
+            .iter()
+            .map(|v| v.to_string())
+            .collect::<Vec<_>>()
+            .join("\n")
+    }
+
     #[test]
     fn parses_type_lines() {
         let (t, s) = parse_type_line("Legendary Creature — Elf Druid");
@@ -807,14 +807,16 @@ mod tests {
     }
 
     #[test]
-    fn streams_array_of_objects() {
+    fn streams_jsonl_records() {
         #[derive(serde::Deserialize)]
         struct Item {
             n: i32,
         }
-        let json = br#"[{"n":1},{"n":2},{"n":3}]"#;
+        // Blank lines and a trailing newline are tolerated, as real bulk files
+        // end with one.
+        let jsonl = b"{\"n\":1}\n{\"n\":2}\n\n{\"n\":3}\n";
         let mut sum = 0;
-        stream_json_array::<_, Item, _>(&json[..], |it| {
+        stream_jsonl::<_, Item, _>(&jsonl[..], |it| {
             sum += it.n;
             Ok(())
         })
@@ -822,6 +824,21 @@ mod tests {
         assert_eq!(sum, 6);
     }
 
+    #[test]
+    fn stream_jsonl_propagates_the_closure_error_unmangled() {
+        #[derive(serde::Deserialize)]
+        struct Item {
+            #[allow(dead_code)] // only the deserialization has to succeed here
+            n: i32,
+        }
+        // Cancellation reporting depends on this: the old array-visitor path
+        // laundered closure errors through `serde::de::Error::custom`, turning
+        // them into strings.
+        let jsonl = b"{\"n\":1}\n{\"n\":2}\n";
+        let err = stream_jsonl::<_, Item, _>(&jsonl[..], |_| Err(CoreError::Cancelled)).unwrap_err();
+        assert!(matches!(err, CoreError::Cancelled), "got {err:?}");
+    }
+
     #[test]
     fn excludes_art_series_cards() {
         let oracle = r#"[
@@ -836,7 +853,7 @@ mod tests {
         let dest = dir.path().join("reference.db");
         let conn = open_reference_for_build(&dest).unwrap();
 
-        let oc = load_oracle_cards(&conn, oracle.as_bytes(), &CancelToken::new()).unwrap();
+        let oc = load_oracle_cards(&conn, jsonl(oracle).as_bytes(), &CancelToken::new()).unwrap();
         assert_eq!(oc, 1);
 
         let count: i64 = conn
@@ -866,7 +883,7 @@ mod tests {
 
         let cancel = CancelToken::new();
         cancel.cancel();
-        let err = load_oracle_cards(&conn, oracle.as_bytes(), &cancel)
+        let err = load_oracle_cards(&conn, jsonl(oracle).as_bytes(), &cancel)
             .map_err(|e| override_if_cancelled(e, &cancel))
             .unwrap_err();
         assert!(matches!(err, CoreError::Cancelled));
@@ -908,8 +925,8 @@ mod tests {
         let dest = dir.path().join("reference.db");
         let conn = open_reference_for_build(&dest).unwrap();
 
-        let oc = load_oracle_cards(&conn, oracle.as_bytes(), &CancelToken::new()).unwrap();
+        let oc = load_oracle_cards(&conn, jsonl(oracle).as_bytes(), &CancelToken::new()).unwrap();
-        let pc = load_printings(&conn, printings.as_bytes(), &CancelToken::new()).unwrap();
+        let pc = load_printings(&conn, jsonl(printings).as_bytes(), &CancelToken::new()).unwrap();
         assert_eq!(oc, 2);
         assert_eq!(pc, 1);
 
@@ -956,7 +973,7 @@ mod tests {
         let dest = dir.path().join("reference.db");
         let conn = open_reference_for_build(&dest).unwrap();
 
-        load_oracle_cards(&conn, oracle.as_bytes(), &CancelToken::new()).unwrap();
+        load_oracle_cards(&conn, jsonl(oracle).as_bytes(), &CancelToken::new()).unwrap();
 
         let (type_line, colors, oracle_text): (String, i64, String) = conn
             .query_row(
@@ -987,7 +1004,7 @@ mod tests {
         let dest = dir.path().join("reference.db");
         let conn = open_reference_for_build(&dest).unwrap();
 
-        load_oracle_cards(&conn, oracle.as_bytes(), &CancelToken::new()).unwrap();
+        load_oracle_cards(&conn, jsonl(oracle).as_bytes(), &CancelToken::new()).unwrap();
 
         let fetch = |id: &str| -> (i64, i64) {
             conn.query_row(
@@ -1025,7 +1042,7 @@ mod tests {
         let dest = dir.path().join("reference.db");
         let conn = open_reference_for_build(&dest).unwrap();
 
-        load_oracle_cards(&conn, oracle.as_bytes(), &CancelToken::new()).unwrap();
+        load_oracle_cards(&conn, jsonl(oracle).as_bytes(), &CancelToken::new()).unwrap();
 
         let fetch = |id: &str| -> Legalities {
             Legalities(
@@ -1065,7 +1082,7 @@ mod tests {
         let dir = tempfile::tempdir().unwrap();
         let dest = dir.path().join("reference.db");
         let conn = open_reference_for_build(&dest).unwrap();
-        load_oracle_cards(&conn, oracle.as_bytes(), &CancelToken::new()).unwrap();
+        load_oracle_cards(&conn, jsonl(oracle).as_bytes(), &CancelToken::new()).unwrap();
 
         let fetch = |id: &str| -> (i64, Option<i64>) {
             conn.query_row(
@@ -1108,7 +1125,7 @@ mod tests {
         let dir = tempfile::tempdir().unwrap();
         let dest = dir.path().join("reference.db");
         let conn = open_reference_for_build(&dest).unwrap();
-        load_oracle_cards(&conn, oracle.as_bytes(), &CancelToken::new()).unwrap();
+        load_oracle_cards(&conn, jsonl(oracle).as_bytes(), &CancelToken::new()).unwrap();
 
         let horizontal = |id: &str| -> bool {
             conn.query_row(
@@ -1153,7 +1170,7 @@ mod tests {
         let dir = tempfile::tempdir().unwrap();
         let dest = dir.path().join("reference.db");
         let conn = open_reference_for_build(&dest).unwrap();
-        load_oracle_cards(&conn, oracle.as_bytes(), &CancelToken::new()).unwrap();
+        load_oracle_cards(&conn, jsonl(oracle).as_bytes(), &CancelToken::new()).unwrap();
 
         let images = |id: &str| -> (String, String, String) {
             conn.query_row(
@@ -1206,8 +1223,8 @@ mod tests {
         let dest = dir.path().join("reference.db");
         let conn = open_reference_for_build(&dest).unwrap();
 
-        load_printings(&conn, dfc.as_bytes(), &CancelToken::new()).unwrap();
+        load_printings(&conn, jsonl(dfc).as_bytes(), &CancelToken::new()).unwrap();
-        load_printings(&conn, split.as_bytes(), &CancelToken::new()).unwrap();
+        load_printings(&conn, jsonl(split).as_bytes(), &CancelToken::new()).unwrap();
 
         let (front, back): (String, String) = conn
             .query_row(
@@ -1245,7 +1262,7 @@ mod tests {
         let dest = dir.path().join("reference.db");
         let conn = open_reference_for_build(&dest).unwrap();
 
-        load_printings(&conn, printings.as_bytes(), &CancelToken::new()).unwrap();
+        load_printings(&conn, jsonl(printings).as_bytes(), &CancelToken::new()).unwrap();
 
         let fetch = |id: &str| -> f64 {
             conn.query_row(

src-tauri/src/commands.rs

@@ -77,6 +77,8 @@ pub async fn update_card_data(app: AppHandle, db: Db<'_>) -> Res<Status> {
     db.lock().unwrap().ingest_cancel = Some(cancel.clone());
     let app_for_progress = app.clone();
 
+    crate::mlog!("update_card_data: starting ingest into {}", reference_path.display());
+
     let result = tauri::async_runtime::spawn_blocking({
         let cancel = cancel.clone();
         move || {
@@ -89,10 +91,15 @@ pub async fn update_card_data(app: AppHandle, db: Db<'_>) -> Res<Status> {
     .map_err(|e| e.to_string())?;
 
     db.lock().unwrap().ingest_cancel = None;
-    result.map_err(|e| e.to_string())?;
+    let stats = crate::logged("update_card_data", result.map_err(|e| e.to_string()))?;
+    crate::mlog!(
+        "update_card_data: ok ({} oracle cards, {} printings)",
+        stats.oracle_cards,
+        stats.printings
+    );
 
     let mut s = db.lock().unwrap();
-    s.reopen()?;
+    crate::logged("update_card_data reopen", s.reopen())?;
     Ok(status_of(&s))
 }
 
@@ -113,9 +120,15 @@ pub fn import_reference_db(db: Db, path: String) -> Res<Status> {
         return Err("That file is already the active card database.".into());
     }
 
+    crate::mlog!("import_reference_db: from {}", src.display());
+
     // Validate before disturbing the working db, so picking the wrong file is a
     // harmless error rather than a broken install.
-    mtgc_core::db::inspect_reference_db(&src).map_err(|e| e.to_string())?;
+    let cards = crate::logged(
+        "import_reference_db validate",
+        mtgc_core::db::inspect_reference_db(&src).map_err(|e| e.to_string()),
+    )?;
+    crate::mlog!("import_reference_db: source holds {cards} oracle cards");
 
     // Release the handles on the file we're about to replace.
     s.close();

src-tauri/src/lib.rs

@@ -7,6 +7,30 @@ use tauri::Manager;
 
 use state::AppState;
 
+/// Write a line to the platform log.
+///
+/// On Android this reaches `adb logcat` under the tag `RustStdoutStderr`: tao's
+/// Android glue `dup2`s stdout/stderr into a pipe and forwards each line to the
+/// system log, so a plain `eprintln!` is all that's needed and no logging
+/// dependency is warranted. On desktop it's just stderr.
+///
+/// This matters because Tauri command failures are *returned* to the WebView as
+/// values, not raised — so without logging them explicitly they never appear in
+/// logcat at all, and the UI only ever shows a truncated version.
+#[macro_export]
+macro_rules! mlog {
+    ($($arg:tt)*) => { eprintln!("[mtgc] {}", format_args!($($arg)*)) };
+}
+
+/// Log a command failure (with its full, untruncated message) and pass the
+/// result through unchanged.
+pub(crate) fn logged<T>(what: &str, result: Result<T, String>) -> Result<T, String> {
+    if let Err(e) = &result {
+        crate::mlog!("{what} failed: {e}");
+    }
+    result
+}
+
 #[cfg_attr(mobile, tauri::mobile_entry_point)]
 pub fn run() {
     tauri::Builder::default()
@@ -17,6 +41,7 @@ pub fn run() {
                 .app_data_dir()
                 .expect("resolve app data dir");
             std::fs::create_dir_all(&data_dir).ok();
+            mlog!("data dir: {}", data_dir.display());
             app.manage(Mutex::new(AppState::new(data_dir)));
             Ok(())
         })