mtgc/crates/mtgc-core/src/ingest.rs

//! Build `reference.db` from Scryfall bulk data.
//!
//! Two passes: the *Oracle Cards* bulk (one entry per `oracle_id`) fills
//! `oracle_cards` + the normalized type/subtype/keyword tables + the name FTS;
//! the *Default Cards* bulk (one entry per printing) fills `printings`.
//!
//! Both bulk files are large gzipped JSONL (one card object per line), so we
//! decompress and parse them record-by-record rather than holding a file in
//! memory.

use std::io::{BufRead, BufReader, Read};
use std::path::Path;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;

use flate2::read::GzDecoder;
use rusqlite::{params, Connection};
use serde::de::DeserializeOwned;
use serde::Deserialize;

use crate::db::open_reference_for_build;
use crate::error::{CoreError, Result};
use crate::model::{ColorSet, Legalities};

const BULK_INDEX_URL: &str = "https://api.scryfall.com/bulk-data";

const STAGE_INDEX: &str = "index";
const STAGE_ORACLE: &str = "oracle";
const STAGE_PRINTINGS: &str = "printings";
const STAGE_OPTIMIZE: &str = "optimize";

/// Row counts from a build, for reporting/verification.
#[derive(Debug, Default, Clone, Copy)]
pub struct IngestStats {
    pub oracle_cards: usize,
    pub printings: usize,
}

/// A flag a caller can flip from another thread to abort an in-progress
/// ingest at the next checkpoint (checked once per parsed record — thousands
/// of times a second, so cancellation is near-instant).
#[derive(Clone, Default)]
pub struct CancelToken(Arc<AtomicBool>);

impl CancelToken {
    pub fn new() -> Self {
        Self(Arc::new(AtomicBool::new(false)))
    }

    pub fn cancel(&self) {
        self.0.store(true, Ordering::Relaxed);
    }

    pub fn is_cancelled(&self) -> bool {
        self.0.load(Ordering::Relaxed)
    }
}

/// Report a clean [`CoreError::Cancelled`] whenever the token is set, whatever
/// error actually surfaced. Cancelling mid-download typically aborts the reader
/// first, so the raw error is an unhelpful I/O or parse failure describing the
/// symptom rather than the cause.
fn override_if_cancelled(e: CoreError, cancel: &CancelToken) -> CoreError {
    if cancel.is_cancelled() {
        CoreError::Cancelled
    } else {
        e
    }
}

/// One segment of the overall ingest, with its share of the total work
/// (weights across all stages sum to 1.0). Lets the UI render a single
/// multi-segment progress bar instead of swapping bars per stage.
#[derive(Debug, Clone, serde::Serialize)]
pub struct IngestStageInfo {
    pub id: &'static str,
    pub label: &'static str,
    pub weight: f64,
}

/// A single progress update from a running ingest. `stage` is the id of the
/// currently active [`IngestStageInfo`]; `message` is a human-readable detail
/// for that stage; `overall_percent` is cumulative progress (0.0..=100.0)
/// across *all* stages, ready to drive a single combined bar; `stages` is the
/// (static, for a given run) segment layout for rendering.
#[derive(Debug, Clone, serde::Serialize)]
pub struct IngestProgress {
    pub stage: &'static str,
    pub message: String,
    pub overall_percent: f64,
    pub stages: Vec<IngestStageInfo>,
}

/// Fixed weight for the two bookkeeping stages (fetching the bulk index,
/// optimizing the FTS index); the remaining budget is split between the two
/// downloads by their known byte size, since both are network-bound.
const INDEX_WEIGHT: f64 = 0.02;
const OPTIMIZE_WEIGHT: f64 = 0.03;

fn stage_plan(oracle_size: u64, default_size: u64) -> Vec<IngestStageInfo> {
    let dl_total = (oracle_size + default_size).max(1) as f64;
    let dl_budget = 1.0 - INDEX_WEIGHT - OPTIMIZE_WEIGHT;
    let oracle_weight = dl_budget * (oracle_size as f64 / dl_total);
    let printings_weight = dl_budget - oracle_weight;
    vec![
        IngestStageInfo {
            id: STAGE_INDEX,
            label: "Index",
            weight: INDEX_WEIGHT,
        },
        IngestStageInfo {
            id: STAGE_ORACLE,
            label: "Card data",
            weight: oracle_weight,
        },
        IngestStageInfo {
            id: STAGE_PRINTINGS,
            label: "Printings",
            weight: printings_weight,
        },
        IngestStageInfo {
            id: STAGE_OPTIMIZE,
            label: "Indexing",
            weight: OPTIMIZE_WEIGHT,
        },
    ]
}

/// Cumulative percent through `stages` once `active` has completed
/// `local_fraction` (0.0..=1.0) of its own work.
fn overall_percent(stages: &[IngestStageInfo], active: &str, local_fraction: f64) -> f64 {
    let mut acc = 0.0;
    for s in stages {
        if s.id == active {
            return ((acc + s.weight * local_fraction) * 100.0).min(100.0);
        }
        acc += s.weight;
    }
    (acc * 100.0).min(100.0)
}

/// Download the latest Scryfall bulk data and (re)build `reference.db` at
/// `dest`. Builds into a temp file alongside `dest` and only replaces it on
/// full success, so a cancelled or failed update never destroys a working
/// reference db.
pub fn build_reference_db(
    dest: &Path,
    cancel: &CancelToken,
    mut progress: impl FnMut(IngestProgress),
) -> Result<IngestStats> {
    let tmp = dest.with_extension("db.tmp");
    let result = build_into(&tmp, cancel, &mut progress);
    match &result {
        Ok(_) => {
            // Rename straight over `dest`: POSIX rename replaces the
            // destination atomically, so there is never a moment with no
            // reference.db. (Deleting `dest` first would open a window where a
            // process kill — far likelier on Android, which reaps backgrounded
            // apps — leaves the user with no card data at all.)
            std::fs::rename(&tmp, dest)?;
        }
        Err(_) => {
            let _ = std::fs::remove_file(&tmp);
        }
    }
    result
}

fn build_into(
    tmp: &Path,
    cancel: &CancelToken,
    progress: &mut impl FnMut(IngestProgress),
) -> Result<IngestStats> {
    // Sizes aren't known until we've hit the bulk index, so the very first
    // event uses a placeholder 50/50 split; the real plan follows within the
    // same (fast) request/response round trip.
    let mut stages = stage_plan(1, 1);
    progress(IngestProgress {
        stage: STAGE_INDEX,
        message: "Fetching Scryfall bulk index…".into(),
        overall_percent: 0.0,
        stages: stages.clone(),
    });
    if cancel.is_cancelled() {
        return Err(CoreError::Cancelled);
    }
    let bulk = fetch_bulk_uris()?;
    stages = stage_plan(bulk.oracle_size, bulk.default_size);

    if tmp.exists() {
        std::fs::remove_file(tmp)?;
    }
    let conn = open_reference_for_build(tmp)?;
    // Bulk-load fast; this db is disposable/rebuildable so durability is moot.
    conn.pragma_update(None, "journal_mode", "MEMORY")?;
    conn.pragma_update(None, "synchronous", "OFF")?;

    progress(IngestProgress {
        stage: STAGE_ORACLE,
        message: "Downloading card data…".into(),
        overall_percent: overall_percent(&stages, STAGE_ORACLE, 0.0),
        stages: stages.clone(),
    });
    let reader = crate::http::get_reader(&bulk.oracle_uri)?;
    // Progress wraps the *compressed* stream, since `compressed_size` is what
    // the bulk index reports; decompression happens after the byte count.
    let reader = progress_reader(
        reader,
        bulk.oracle_size,
        STAGE_ORACLE,
        "Downloading card data",
        &stages,
        progress,
    );
    let reader = GzDecoder::new(reader);
    let oracle_cards = load_oracle_cards(&conn, reader, cancel).map_err(|e| override_if_cancelled(e, cancel))?;

    progress(IngestProgress {
        stage: STAGE_PRINTINGS,
        message: "Downloading printings…".into(),
        overall_percent: overall_percent(&stages, STAGE_PRINTINGS, 0.0),
        stages: stages.clone(),
    });
    let reader = crate::http::get_reader(&bulk.default_uri)?;
    let reader = progress_reader(
        reader,
        bulk.default_size,
        STAGE_PRINTINGS,
        "Downloading printings",
        &stages,
        progress,
    );
    let reader = GzDecoder::new(reader);
    let printings = load_printings(&conn, reader, cancel).map_err(|e| override_if_cancelled(e, cancel))?;

    if cancel.is_cancelled() {
        return Err(CoreError::Cancelled);
    }
    progress(IngestProgress {
        stage: STAGE_OPTIMIZE,
        message: "Optimizing search index…".into(),
        overall_percent: overall_percent(&stages, STAGE_OPTIMIZE, 0.5),
        stages: stages.clone(),
    });
    conn.execute_batch("INSERT INTO oracle_names_fts(oracle_names_fts) VALUES('optimize');")?;

    Ok(IngestStats {
        oracle_cards,
        printings,
    })
}

/// Human-readable byte size, e.g. `42.3 MB`.
fn human_mb(bytes: u64) -> String {
    format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0))
}

/// Wrap a reader so that, as it's consumed (concurrently with JSON parsing),
/// it periodically reports download progress against the overall multi-stage
/// bar. Throttled to avoid flooding the UI with an event per network read.
fn progress_reader<'a, R: Read>(
    inner: R,
    total: u64,
    stage: &'static str,
    label: &'static str,
    stages: &'a [IngestStageInfo],
    progress: &'a mut impl FnMut(IngestProgress),
) -> impl Read + 'a
where
    R: 'a,
{
    const EMIT_EVERY: u64 = 512 * 1024; // 512 KB
    ProgressReader {
        inner,
        read: 0,
        last_emit: 0,
        total,
        on_progress: move |read: u64, total: u64| {
            let message = format!("{label}… {} / {}", human_mb(read), human_mb(total));
            let fraction = if total > 0 {
                (read as f64 / total as f64).min(1.0)
            } else {
                1.0
            };
            progress(IngestProgress {
                stage,
                message,
                overall_percent: overall_percent(stages, stage, fraction),
                stages: stages.to_vec(),
            });
        },
        threshold: EMIT_EVERY,
    }
}

struct ProgressReader<R, F> {
    inner: R,
    read: u64,
    last_emit: u64,
    total: u64,
    threshold: u64,
    on_progress: F,
}

impl<R: Read, F: FnMut(u64, u64)> Read for ProgressReader<R, F> {
    fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
        let n = self.inner.read(buf)?;
        self.read += n as u64;
        if n == 0 || self.read - self.last_emit >= self.threshold {
            self.last_emit = self.read;
            (self.on_progress)(self.read, self.total);
        }
        Ok(n)
    }
}

// ---- Scryfall bulk index -------------------------------------------------

#[derive(Deserialize)]
struct BulkIndex {
    data: Vec<BulkEntry>,
}

#[derive(Deserialize)]
struct BulkEntry {
    #[serde(rename = "type")]
    kind: String,
    /// Scryfall serves gzipped JSONL (one card object per line). This replaced
    /// the older `download_uri`/`size` pair, which pointed at an uncompressed
    /// JSON array — a db built before that change fails here with
    /// "missing field `download_uri`".
    jsonl_download_uri: String,
    /// *Compressed* byte size, used for download progress since the response
    /// is chunked (no `Content-Length` to rely on). Progress is therefore
    /// measured on the compressed stream, before decompression.
    compressed_size: u64,
}

struct BulkUris {
    oracle_uri: String,
    oracle_size: u64,
    default_uri: String,
    default_size: u64,
}

/// The agent's timeouts matter especially here: cancellation is only polled per
/// *parsed record*, so if no bytes arrive, no records parse and `CancelToken`
/// is never consulted — without a read timeout the ingest wedges with no way
/// out. See [`crate::http::agent`].
fn fetch_bulk_uris() -> Result<BulkUris> {
    let index: BulkIndex = crate::http::get_json(BULK_INDEX_URL)?;

    let find = |kind: &str| {
        index
            .data
            .iter()
            .find(|e| e.kind == kind)
            .ok_or_else(|| CoreError::Other(format!("Scryfall bulk index missing '{kind}'")))
    };
    let oracle = find("oracle_cards")?;
    let default = find("default_cards")?;
    Ok(BulkUris {
        oracle_uri: oracle.jsonl_download_uri.clone(),
        oracle_size: oracle.compressed_size,
        default_uri: default.jsonl_download_uri.clone(),
        default_size: default.compressed_size,
    })
}

// ---- Streaming JSONL parse ----------------------------------------------

/// Deserialize newline-delimited JSON record-by-record, invoking `f` per record
/// without holding the whole file in memory.
///
/// Scryfall's bulk downloads are JSONL — one complete card object per line —
/// so a line reader replaces what used to be a streaming JSON-array visitor.
/// An error from `f` propagates directly (it is no longer laundered through
/// `serde::de::Error::custom`), which keeps cancellation reporting exact.
fn stream_jsonl<R, T, F>(reader: R, mut f: F) -> Result<()>
where
    R: Read,
    T: DeserializeOwned,
    F: FnMut(T) -> Result<()>,
{
    for line in BufReader::with_capacity(1 << 20, reader).lines() {
        let line = line?;
        // Trailing newline at EOF, and any blank separator lines.
        if line.trim().is_empty() {
            continue;
        }
        f(serde_json::from_str(&line)?)?;
    }
    Ok(())
}

// ---- Oracle pass ---------------------------------------------------------

#[derive(Deserialize)]
struct ScryOracle {
    /// This entry's own printing id — the "representative" printing Scryfall
    /// includes in the Oracle Cards bulk (its own doc: "the most up-to-date
    /// recognizable" version). Captured as `oracle_cards.default_scryfall_id`
    /// purely for reference; not currently read back.
    #[serde(default)]
    id: Option<String>,
    oracle_id: Option<String>,
    name: String,
    #[serde(default)]
    layout: String,
    #[serde(default)]
    cmc: f64,
    #[serde(default)]
    colors: Option<Vec<String>>,
    #[serde(default)]
    color_identity: Vec<String>,
    #[serde(default)]
    type_line: Option<String>,
    #[serde(default)]
    oracle_text: Option<String>,
    #[serde(default)]
    keywords: Vec<String>,
    #[serde(default)]
    image_uris: Option<ImageUris>,
    #[serde(default)]
    card_faces: Option<Vec<ScryFace>>,
    /// What colors of mana this card can add (lands, mana rocks, dorks). A
    /// top-level-only Scryfall field — not per-face, even on DFCs.
    #[serde(default)]
    produced_mana: Option<Vec<String>>,
    /// Format-slug -> status-string (e.g. "legal"/"not_legal"/"banned"/
    /// "restricted"). See [`crate::model::Legalities`].
    #[serde(default)]
    legalities: std::collections::HashMap<String, String>,
    /// Scryfall's EDHREC popularity rank (lower = more played); absent for
    /// some cards. Kept nullable — see `synergy::CardSuggestion::edhrec_rank`.
    #[serde(default)]
    edhrec_rank: Option<i64>,
    /// Printed power/toughness. Strings because Magic allows non-numeric
    /// values ("*", "1+*", "X"); those parse to None and stay unfilterable.
    #[serde(default)]
    power: Option<String>,
    #[serde(default)]
    toughness: Option<String>,
}

#[derive(Deserialize)]
struct ScryFace {
    #[serde(default)]
    type_line: Option<String>,
    #[serde(default)]
    power: Option<String>,
    #[serde(default)]
    toughness: Option<String>,
    #[serde(default)]
    colors: Option<Vec<String>>,
    #[serde(default)]
    oracle_text: Option<String>,
    #[serde(default)]
    image_uris: Option<ImageUris>,
}

/// Parse a printed power/toughness into a number, or `None` when it isn't one.
///
/// Magic prints values like `*`, `1+*` and `X` whose real value depends on game
/// state. Coercing those to 0 would quietly file every Tarmogoyf under "power
/// 0"; leaving them NULL drops them from range filters instead, which is the
/// honest answer to "power >= 5".
fn numeric_pt(raw: Option<&str>) -> Option<f64> {
    raw?.trim().parse::<f64>().ok()
}

fn load_oracle_cards(conn: &Connection, reader: impl Read, cancel: &CancelToken) -> Result<usize> {
    conn.execute_batch("BEGIN")?;
    let mut count = 0usize;
    {
        let mut ins_card = conn.prepare(
            "INSERT OR IGNORE INTO oracle_cards
               (oracle_id,name,name_lower,mana_value,colors,color_identity,type_line,oracle_text,
                produced_mana,produces_colorless,legalities,themes,edhrec_rank,horizontal,
                image_uri,back_image_uri,default_scryfall_id,power,toughness,vanilla)
             VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14,?15,?16,?17,?18,?19,?20)",
        )?;
        let mut ins_fts =
            conn.prepare("INSERT INTO oracle_names_fts(name,oracle_id) VALUES (?1,?2)")?;
        let mut ins_type =
            conn.prepare("INSERT INTO card_types(oracle_id,type) VALUES (?1,?2)")?;
        let mut ins_sub =
            conn.prepare("INSERT INTO card_subtypes(oracle_id,subtype) VALUES (?1,?2)")?;
        let mut ins_kw =
            conn.prepare("INSERT INTO card_keywords(oracle_id,keyword) VALUES (?1,?2)")?;

        stream_jsonl::<_, ScryOracle, _>(reader, |c| {
            if cancel.is_cancelled() {
                return Err(CoreError::Cancelled);
            }
            let Some(oracle_id) = c.oracle_id.clone() else {
                return Ok(()); // rare face-only records without a top-level oracle_id
            };
            // Art Series cards (Secret Lair) aren't real playable cards — exclude
            // them from search/library entirely.
            if c.layout == "art_series" {
                return Ok(());
            }

            // Colors: prefer top-level, else union of faces.
            let colors = match &c.colors {
                Some(cs) => ColorSet::from_letters(cs),
                None => {
                    let mut m = 0u8;
                    if let Some(faces) = &c.card_faces {
                        for f in faces {
                            if let Some(cs) = &f.colors {
                                m |= ColorSet::from_letters(cs).0;
                            }
                        }
                    }
                    ColorSet(m)
                }
            };
            let identity = ColorSet::from_letters(&c.color_identity);

            // Type line: top-level, else join faces.
            let type_line = c.type_line.clone().unwrap_or_else(|| {
                c.card_faces
                    .as_ref()
                    .map(|faces| {
                        faces
                            .iter()
                            .filter_map(|f| f.type_line.clone())
                            .collect::<Vec<_>>()
                            .join(" // ")
                    })
                    .unwrap_or_default()
            });

            // Oracle text: top-level, else join faces — double-faced cards (MDFC,
            // transform, etc.) never carry a top-level oracle_text in Scryfall's
            // data, only per-face text.
            let oracle_text = c.oracle_text.clone().unwrap_or_else(|| {
                c.card_faces
                    .as_ref()
                    .map(|faces| {
                        faces
                            .iter()
                            .filter_map(|f| f.oracle_text.clone())
                            .collect::<Vec<_>>()
                            .join("\n")
                    })
                    .unwrap_or_default()
            });

            // Mana production: colored letters map to the WUBRG bitmask (same as
            // colors/identity); "C" (colorless) is tracked separately since it
            // isn't a WUBRG bit. Top-level only — Scryfall doesn't put this on
            // individual card_faces.
            let produced = c.produced_mana.as_deref().unwrap_or(&[]);
            let produced_mana = ColorSet::from_letters(produced);
            let produces_colorless = produced.iter().any(|s| s.eq_ignore_ascii_case("C"));

            let legalities = Legalities::from_scryfall(&c.legalities);
            let themes = crate::synergy::compute_themes(&oracle_text);
            // Battles and Planes/Phenomena store their card image portrait-shaped
            // with the content itself rotated — viewers must rotate it 90° to
            // read it right-side up (verified against real Scryfall images).
            // Battles are *not* `layout: "battle"` in Scryfall's data — every
            // released Battle (the "Invasion of ..." Sieges) transforms into a
            // creature, so it's `layout: "transform"` with the Battle type only
            // on the front face. Detect it from the front face's type instead
            // of the layout.
            let front_type_line = type_line.split(" // ").next().unwrap_or("");
            let is_battle = front_type_line
                .split('—')
                .next()
                .unwrap_or("")
                .split_whitespace()
                .any(|w| w == "Battle");
            let horizontal = is_battle || c.layout == "planar";

            // Representative-printing images: top-level for single-faced
            // cards, else the front/back faces — same fallback shape as
            // `load_printings` below, since Oracle Cards entries mirror a
            // printing's image fields.
            let image_uri = c
                .image_uris
                .as_ref()
                .and_then(|i| i.normal.clone())
                .or_else(|| {
                    c.card_faces
                        .as_ref()
                        .and_then(|f| f.first())
                        .and_then(|f| f.image_uris.as_ref())
                        .and_then(|i| i.normal.clone())
                })
                .unwrap_or_default();
            let back_image_uri = c
                .card_faces
                .as_ref()
                .and_then(|f| f.get(1))
                .and_then(|f| f.image_uris.as_ref())
                .and_then(|i| i.normal.clone())
                .unwrap_or_default();
            let default_scryfall_id = c.id.clone().unwrap_or_default();
            // Fall back to the front face: a transforming creature carries its
            // P/T per-face rather than at the top level.
            let front = c.card_faces.as_ref().and_then(|f| f.first());
            let power = numeric_pt(
                c.power.as_deref().or_else(|| front.and_then(|f| f.power.as_deref())),
            );
            let toughness = numeric_pt(
                c.toughness.as_deref().or_else(|| front.and_then(|f| f.toughness.as_deref())),
            );

            // "Vanilla": a creature whose printed rules box is empty. Judged from
            // the front face's *own* text, never the joined `oracle_text` above —
            // an Adventure or Prepared card is a vanilla creature with a spell
            // stapled on, and the joined column always carries that spell's text.
            // The front face is the right one to ask: it's the half that resolves
            // as the creature, and a transforming card is cast from its front.
            let front_text = match front {
                Some(f) => f.oracle_text.clone().unwrap_or_default(),
                None => oracle_text.clone(),
            };
            let vanilla = front_type_line
                .split('—')
                .next()
                .unwrap_or("")
                .split_whitespace()
                .any(|w| w == "Creature")
                && front_text.trim().is_empty();

            let changed = ins_card.execute(params![
                oracle_id,
                c.name,
                c.name.to_lowercase(),
                c.cmc,
                colors.0 as i64,
                identity.0 as i64,
                type_line,
                oracle_text,
                produced_mana.0 as i64,
                produces_colorless as i64,
                legalities.0,
                themes,
                c.edhrec_rank,
                horizontal as i64,
                image_uri,
                back_image_uri,
                default_scryfall_id,
                power,
                toughness,
                vanilla as i64,
            ])?;
            // Only emit satellite rows the first time we see this oracle_id.
            if changed == 1 {
                ins_fts.execute(params![c.name, oracle_id])?;
                let (types, subtypes) = parse_type_line(&type_line);
                for t in types {
                    ins_type.execute(params![oracle_id, t])?;
                }
                for s in subtypes {
                    ins_sub.execute(params![oracle_id, s])?;
                }
                let mut kws = c.keywords.clone();
                kws.sort();
                kws.dedup();
                for k in kws {
                    ins_kw.execute(params![oracle_id, k])?;
                }
                count += 1;
            }
            Ok(())
        })?;
    }
    conn.execute_batch("COMMIT")?;
    Ok(count)
}

/// Split a Scryfall type line into (types+supertypes, subtypes). Handles split /
/// double-faced lines joined by " // ". The separator is the em dash '—'.
pub fn parse_type_line(type_line: &str) -> (Vec<String>, Vec<String>) {
    let mut types = Vec::new();
    let mut subtypes = Vec::new();
    for part in type_line.split("//") {
        let part = part.trim();
        let mut halves = part.splitn(2, '—');
        let left = halves.next().unwrap_or("").trim();
        let right = halves.next().unwrap_or("").trim();
        for t in left.split_whitespace() {
            types.push(t.to_string());
        }
        for s in right.split_whitespace() {
            subtypes.push(s.to_string());
        }
    }
    types.sort();
    types.dedup();
    subtypes.sort();
    subtypes.dedup();
    (types, subtypes)
}

// ---- Printings pass ------------------------------------------------------

#[derive(Deserialize)]
struct ScryPrinting {
    id: String,
    oracle_id: Option<String>,
    #[serde(default)]
    set: String,
    #[serde(default)]
    collector_number: String,
    #[serde(default)]
    finishes: Vec<String>,
    #[serde(default)]
    rarity: String,
    #[serde(default)]
    image_uris: Option<ImageUris>,
    #[serde(default)]
    card_faces: Option<Vec<PrintFace>>,
    #[serde(default)]
    prices: Option<ScryPrices>,
    /// Scryfall's per-printing markers. Carries `"universesbeyond"` on cards
    /// from another IP — the only field that reliably identifies them; there is
    /// no Universes Beyond `set_type`, and the security stamp is absent at low
    /// rarities.
    #[serde(default)]
    promo_types: Vec<String>,
    /// Where this printing can be played. Missing "paper" means it exists only
    /// on Arena/MTGO and can never be part of a physical collection.
    #[serde(default)]
    games: Vec<String>,
    /// Scryfall's set category; `"funny"` covers the Un-sets.
    #[serde(default)]
    set_type: String,
}

#[derive(Deserialize)]
struct ImageUris {
    #[serde(default)]
    normal: Option<String>,
}

#[derive(Deserialize)]
struct ScryPrices {
    #[serde(default)]
    usd: Option<String>,
}

#[derive(Deserialize)]
struct PrintFace {
    #[serde(default)]
    image_uris: Option<ImageUris>,
}

fn load_printings(conn: &Connection, reader: impl Read, cancel: &CancelToken) -> Result<usize> {
    conn.execute_batch("BEGIN")?;
    let mut count = 0usize;
    {
        let mut ins = conn.prepare(
            "INSERT OR IGNORE INTO printings
               (scryfall_id,oracle_id,set_code,collector_number,finishes,rarity,image_uri,back_image_uri,price_usd,universes_beyond,digital,funny)
             VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12)",
        )?;
        stream_jsonl::<_, ScryPrinting, _>(reader, |p| {
            if cancel.is_cancelled() {
                return Err(CoreError::Cancelled);
            }
            let Some(oracle_id) = p.oracle_id.clone() else {
                return Ok(());
            };
            let image = p
                .image_uris
                .as_ref()
                .and_then(|i| i.normal.clone())
                .or_else(|| {
                    p.card_faces
                        .as_ref()
                        .and_then(|f| f.first())
                        .and_then(|f| f.image_uris.as_ref())
                        .and_then(|i| i.normal.clone())
                })
                .unwrap_or_default();
            // Only transform/modal-DFC/reversible cards carry per-face images
            // with a distinct second face; split/adventure cards share one
            // combined top-level image and have no per-face `image_uris`.
            let back_image = p
                .card_faces
                .as_ref()
                .and_then(|f| f.get(1))
                .and_then(|f| f.image_uris.as_ref())
                .and_then(|i| i.normal.clone())
                .unwrap_or_default();
            let price_usd = p
                .prices
                .as_ref()
                .and_then(|pr| pr.usd.as_ref())
                .and_then(|s| s.parse::<f64>().ok())
                .unwrap_or(0.0);
            let universes_beyond = p.promo_types.iter().any(|t| t == "universesbeyond");
            // An empty `games` (some tokens/oddities) is treated as paper —
            // better to leave a card visible than to hide a real one.
            let digital = !p.games.is_empty() && !p.games.iter().any(|g| g == "paper");
            let funny = p.set_type == "funny";
            ins.execute(params![
                p.id,
                oracle_id,
                p.set,
                p.collector_number,
                p.finishes.join(","),
                p.rarity,
                image,
                back_image,
                price_usd,
                universes_beyond as i64,
                digital as i64,
                funny as i64,
            ])?;
            count += 1;
            Ok(())
        })?;
    }

    // Roll the per-printing flags up to the card. `MIN(...) = 1` means *every*
    // printing carries the flag, which is what distinguishes a card that only
    // exists as another IP's character (Frodo) from an ordinary card that
    // happens to have been reprinted into such a set (Lightning Bolt appears in
    // Marvel Commander). Using "any printing" instead would hide staples — and
    // the same reasoning applies to digital and funny printings, since paper
    // cards do get Arena versions and Un-set reprints.
    conn.execute_batch(
        "UPDATE oracle_cards SET
             universes_beyond = COALESCE((SELECT MIN(p.universes_beyond) FROM printings p
                                           WHERE p.oracle_id = oracle_cards.oracle_id), 0),
             digital_only     = COALESCE((SELECT MIN(p.digital) FROM printings p
                                           WHERE p.oracle_id = oracle_cards.oracle_id), 0),
             funny            = COALESCE((SELECT MIN(p.funny) FROM printings p
                                           WHERE p.oracle_id = oracle_cards.oracle_id), 0)
          WHERE EXISTS (SELECT 1 FROM printings p WHERE p.oracle_id = oracle_cards.oracle_id)",
    )?;
    conn.execute_batch("COMMIT")?;
    Ok(count)
}

#[cfg(test)]
mod tests {
    use super::*;

    /// Fixtures below are written as pretty-printed JSON arrays because that
    /// stays readable in-source; the real bulk files are JSONL. Convert here so
    /// the tests still drive the production JSONL parser rather than a
    /// test-only code path.
    fn jsonl(array_json: &str) -> String {
        serde_json::from_str::<Vec<serde_json::Value>>(array_json)
            .expect("fixture is a JSON array")
            .iter()
            .map(|v| v.to_string())
            .collect::<Vec<_>>()
            .join("\n")
    }

    #[test]
    fn parses_type_lines() {
        let (t, s) = parse_type_line("Legendary Creature — Elf Druid");
        assert_eq!(t, vec!["Creature", "Legendary"]);
        assert_eq!(s, vec!["Druid", "Elf"]);

        let (t, s) = parse_type_line("Instant");
        assert_eq!(t, vec!["Instant"]);
        assert!(s.is_empty());

        // Split card.
        let (t, s) = parse_type_line("Sorcery // Sorcery");
        assert_eq!(t, vec!["Sorcery"]);
        assert!(s.is_empty());
    }

    #[test]
    fn streams_jsonl_records() {
        #[derive(serde::Deserialize)]
        struct Item {
            n: i32,
        }
        // Blank lines and a trailing newline are tolerated, as real bulk files
        // end with one.
        let jsonl = b"{\"n\":1}\n{\"n\":2}\n\n{\"n\":3}\n";
        let mut sum = 0;
        stream_jsonl::<_, Item, _>(&jsonl[..], |it| {
            sum += it.n;
            Ok(())
        })
        .unwrap();
        assert_eq!(sum, 6);
    }

    #[test]
    fn stream_jsonl_propagates_the_closure_error_unmangled() {
        #[derive(serde::Deserialize)]
        struct Item {
            #[allow(dead_code)] // only the deserialization has to succeed here
            n: i32,
        }
        // Cancellation reporting depends on this: the old array-visitor path
        // laundered closure errors through `serde::de::Error::custom`, turning
        // them into strings.
        let jsonl = b"{\"n\":1}\n{\"n\":2}\n";
        let err = stream_jsonl::<_, Item, _>(&jsonl[..], |_| Err(CoreError::Cancelled)).unwrap_err();
        assert!(matches!(err, CoreError::Cancelled), "got {err:?}");
    }

    #[test]
    fn excludes_art_series_cards() {
        let oracle = r#"[
          {"oracle_id":"oid-1","name":"Llanowar Elves","cmc":1.0,
           "colors":["G"],"color_identity":["G"],
           "type_line":"Creature — Elf Druid","keywords":[]},
          {"oracle_id":"oid-2","name":"Llanowar Elves","layout":"art_series",
           "type_line":"Card","keywords":[]}
        ]"#;

        let dir = tempfile::tempdir().unwrap();
        let dest = dir.path().join("reference.db");
        let conn = open_reference_for_build(&dest).unwrap();

        let oc = load_oracle_cards(&conn, jsonl(oracle).as_bytes(), &CancelToken::new()).unwrap();
        assert_eq!(oc, 1);

        let count: i64 = conn
            .query_row(
                "SELECT count(*) FROM oracle_cards WHERE oracle_id='oid-2'",
                [],
                |r| r.get(0),
            )
            .unwrap();
        assert_eq!(count, 0);
    }

    #[test]
    fn cancel_token_aborts_load_with_cancelled_error() {
        let oracle = r#"[
          {"oracle_id":"oid-1","name":"Llanowar Elves","cmc":1.0,
           "colors":["G"],"color_identity":["G"],
           "type_line":"Creature — Elf Druid","keywords":[]},
          {"oracle_id":"oid-2","name":"Serra Angel","cmc":5.0,
           "colors":["W"],"color_identity":["W"],
           "type_line":"Creature — Angel","keywords":["Flying","Vigilance"]}
        ]"#;

        let dir = tempfile::tempdir().unwrap();
        let dest = dir.path().join("reference.db");
        let conn = open_reference_for_build(&dest).unwrap();

        let cancel = CancelToken::new();
        cancel.cancel();
        let err = load_oracle_cards(&conn, jsonl(oracle).as_bytes(), &cancel)
            .map_err(|e| override_if_cancelled(e, &cancel))
            .unwrap_err();
        assert!(matches!(err, CoreError::Cancelled));
    }

    #[test]
    fn stage_plan_weights_sum_to_one() {
        let stages = stage_plan(171_000_000, 530_000_000);
        let total: f64 = stages.iter().map(|s| s.weight).sum();
        assert!((total - 1.0).abs() < 1e-9);
        // The larger download gets the larger share of the two DL stages.
        let oracle_w = stages.iter().find(|s| s.id == STAGE_ORACLE).unwrap().weight;
        let printings_w = stages
            .iter()
            .find(|s| s.id == STAGE_PRINTINGS)
            .unwrap()
            .weight;
        assert!(printings_w > oracle_w);
    }

    #[test]
    fn only_cards_whose_every_printing_is_universes_beyond_are_flagged() {
        // The distinction that matters: Frodo exists only as Lord of the Rings,
        // while Lightning Bolt is an ordinary card that happens to have been
        // reprinted into a Universes Beyond set. Flagging on "any printing is
        // UB" would hide staples from anyone filtering them out.
        let oracle = r#"[
          {"oracle_id":"oid-frodo","name":"Frodo, Adventurous Hobbit","cmc":2.0,
           "colors":["W"],"color_identity":["W"],"type_line":"Legendary Creature — Halfling Scout","keywords":[]},
          {"oracle_id":"oid-bolt","name":"Lightning Bolt","cmc":1.0,
           "colors":["R"],"color_identity":["R"],"type_line":"Instant","keywords":[]},
          {"oracle_id":"oid-plain","name":"Serra Angel","cmc":5.0,
           "colors":["W"],"color_identity":["W"],"type_line":"Creature — Angel","keywords":[]}
        ]"#;
        let printings = r#"[
          {"id":"p-frodo-1","oracle_id":"oid-frodo","set":"ltr","collector_number":"1",
           "rarity":"rare","promo_types":["universesbeyond"]},
          {"id":"p-frodo-2","oracle_id":"oid-frodo","set":"ltc","collector_number":"2",
           "rarity":"rare","promo_types":["surgefoil","universesbeyond"]},
          {"id":"p-bolt-1","oracle_id":"oid-bolt","set":"lea","collector_number":"161",
           "rarity":"common"},
          {"id":"p-bolt-2","oracle_id":"oid-bolt","set":"msc","collector_number":"99",
           "rarity":"rare","promo_types":["universesbeyond"]},
          {"id":"p-plain-1","oracle_id":"oid-plain","set":"m19","collector_number":"34",
           "rarity":"uncommon"}
        ]"#;

        let dir = tempfile::tempdir().unwrap();
        let conn = open_reference_for_build(&dir.path().join("reference.db")).unwrap();
        load_oracle_cards(&conn, jsonl(oracle).as_bytes(), &CancelToken::new()).unwrap();
        load_printings(&conn, jsonl(printings).as_bytes(), &CancelToken::new()).unwrap();

        let flagged = |oid: &str| -> i64 {
            conn.query_row(
                "SELECT universes_beyond FROM oracle_cards WHERE oracle_id = ?1",
                [oid],
                |r| r.get(0),
            )
            .unwrap()
        };

        assert_eq!(flagged("oid-frodo"), 1, "every printing is UB");
        assert_eq!(
            flagged("oid-bolt"),
            0,
            "a normal card reprinted into a UB set must not be flagged"
        );
        assert_eq!(flagged("oid-plain"), 0, "no UB printings at all");
    }

    #[test]
    fn oracle_and_printing_load_from_fixture_json() {
        // Minimal Scryfall-shaped fixtures exercising the insert paths.
        let oracle = r#"[
          {"oracle_id":"oid-1","name":"Llanowar Elves","cmc":1.0,
           "colors":["G"],"color_identity":["G"],
           "type_line":"Creature — Elf Druid","keywords":[]},
          {"oracle_id":"oid-2","name":"Serra Angel","cmc":5.0,
           "colors":["W"],"color_identity":["W"],
           "type_line":"Creature — Angel","keywords":["Flying","Vigilance"]}
        ]"#;
        let printings = r#"[
          {"id":"p-1","oracle_id":"oid-1","set":"m19","collector_number":"310",
           "finishes":["nonfoil","foil"],"rarity":"common",
           "image_uris":{"normal":"http://img/1"}}
        ]"#;

        let dir = tempfile::tempdir().unwrap();
        let dest = dir.path().join("reference.db");
        let conn = open_reference_for_build(&dest).unwrap();

        let oc = load_oracle_cards(&conn, jsonl(oracle).as_bytes(), &CancelToken::new()).unwrap();
        let pc = load_printings(&conn, jsonl(printings).as_bytes(), &CancelToken::new()).unwrap();
        assert_eq!(oc, 2);
        assert_eq!(pc, 1);

        // Keyword table populated.
        let flyers: i64 = conn
            .query_row(
                "SELECT count(*) FROM card_keywords WHERE keyword='Flying'",
                [],
                |r| r.get(0),
            )
            .unwrap();
        assert_eq!(flyers, 1);

        // Bitmask colors stored.
        let g: i64 = conn
            .query_row(
                "SELECT colors FROM oracle_cards WHERE oracle_id='oid-1'",
                [],
                |r| r.get(0),
            )
            .unwrap();
        assert_eq!(g as u8, crate::model::G);
    }

    #[test]
    fn double_faced_cards_fall_back_to_card_faces_for_type_colors_and_text() {
        // Modal DFCs (and transform cards) never carry a top-level type_line,
        // colors, or oracle_text in Scryfall's oracle-cards bulk — only
        // per-face fields. Regression test for a bug where oracle_text had no
        // fallback (unlike type_line/colors), silently leaving every
        // double-faced card's oracle_text empty and unsearchable.
        let oracle = r#"[
          {"oracle_id":"oid-dfc","name":"Elesh Norn // The Argent Etchings",
           "cmc":5.0,"color_identity":["W"],"keywords":[],
           "card_faces":[
             {"type_line":"Legendary Creature — Phyrexian Praetor","colors":["W"],
              "oracle_text":"Whenever a source an opponent controls deals damage to you or a permanent you control, exile that source."},
             {"type_line":"Battle — Siege","colors":[],
              "oracle_text":"When you defeated The Argent Etchings, draw a card."}
           ]}
        ]"#;

        let dir = tempfile::tempdir().unwrap();
        let dest = dir.path().join("reference.db");
        let conn = open_reference_for_build(&dest).unwrap();

        load_oracle_cards(&conn, jsonl(oracle).as_bytes(), &CancelToken::new()).unwrap();

        let (type_line, colors, oracle_text): (String, i64, String) = conn
            .query_row(
                "SELECT type_line, colors, oracle_text FROM oracle_cards WHERE oracle_id='oid-dfc'",
                [],
                |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
            )
            .unwrap();

        assert_eq!(type_line, "Legendary Creature — Phyrexian Praetor // Battle — Siege");
        assert_eq!(colors as u8, crate::model::W);
        assert!(oracle_text.contains("an opponent controls deals damage"));
        assert!(oracle_text.contains("draw a card"));
    }

    #[test]
    fn vanilla_is_judged_from_the_creature_face_not_the_joined_text() {
        // The `oracle_text` column joins every face, so an Adventure card's
        // stored text always contains the spell half — the creature half's
        // emptiness is only visible in `card_faces`. Scute Swarm-style bodies
        // with a spell attached are exactly the case the filter must catch.
        let oracle = r#"[
          {"oracle_id":"oid-bears","name":"Grizzly Bears","cmc":2.0,"colors":["G"],
           "color_identity":["G"],"type_line":"Creature — Bear","keywords":[],
           "power":"2","toughness":"2"},
          {"oracle_id":"oid-adv","name":"Lovestruck Beast","cmc":3.0,"colors":["G"],
           "color_identity":["G"],"keywords":[],
           "type_line":"Creature — Beast // Sorcery — Adventure",
           "card_faces":[
             {"type_line":"Creature — Beast","colors":["G"],"power":"5","toughness":"5"},
             {"type_line":"Sorcery — Adventure","colors":["G"],
              "oracle_text":"Create a 1/1 white Human creature token."}
           ]},
          {"oracle_id":"oid-flyer","name":"Serra Angel","cmc":5.0,"colors":["W"],
           "color_identity":["W"],"type_line":"Creature — Angel","keywords":["Flying"],
           "oracle_text":"Flying, vigilance","power":"4","toughness":"4"},
          {"oracle_id":"oid-land","name":"Wastes","cmc":0.0,"colors":[],
           "color_identity":[],"type_line":"Land","keywords":[]}
        ]"#;

        let dir = tempfile::tempdir().unwrap();
        let dest = dir.path().join("reference.db");
        let conn = open_reference_for_build(&dest).unwrap();

        load_oracle_cards(&conn, jsonl(oracle).as_bytes(), &CancelToken::new()).unwrap();

        let vanilla = |id: &str| -> i64 {
            conn.query_row(
                "SELECT vanilla FROM oracle_cards WHERE oracle_id = ?1",
                params![id],
                |r| r.get(0),
            )
            .unwrap()
        };
        assert_eq!(vanilla("oid-bears"), 1, "a plain vanilla creature");
        assert_eq!(vanilla("oid-adv"), 1, "adventure whose creature half is vanilla");
        assert_eq!(vanilla("oid-flyer"), 0, "a creature with abilities");
        assert_eq!(vanilla("oid-land"), 0, "a textless card that isn't a creature");
    }

    #[test]
    fn captures_produced_mana_including_colorless() {
        let oracle = r#"[
          {"oracle_id":"oid-dual","name":"Overgrown Tomb","cmc":0.0,
           "color_identity":[],"keywords":[],"produced_mana":["B","G"]},
          {"oracle_id":"oid-wastes","name":"Wastes","cmc":0.0,
           "color_identity":[],"keywords":[],"produced_mana":["C"]},
          {"oracle_id":"oid-vanilla","name":"Grizzly Bears","cmc":2.0,
           "color_identity":["G"],"keywords":[]}
        ]"#;

        let dir = tempfile::tempdir().unwrap();
        let dest = dir.path().join("reference.db");
        let conn = open_reference_for_build(&dest).unwrap();

        load_oracle_cards(&conn, jsonl(oracle).as_bytes(), &CancelToken::new()).unwrap();

        let fetch = |id: &str| -> (i64, i64) {
            conn.query_row(
                "SELECT produced_mana, produces_colorless FROM oracle_cards WHERE oracle_id=?1",
                params![id],
                |r| Ok((r.get(0)?, r.get(1)?)),
            )
            .unwrap()
        };

        let (dual_mana, dual_colorless) = fetch("oid-dual");
        assert_eq!(dual_mana as u8, crate::model::B | crate::model::G);
        assert_eq!(dual_colorless, 0);

        let (wastes_mana, wastes_colorless) = fetch("oid-wastes");
        assert_eq!(wastes_mana, 0); // "C" isn't a WUBRG bit
        assert_eq!(wastes_colorless, 1);

        let (vanilla_mana, vanilla_colorless) = fetch("oid-vanilla");
        assert_eq!(vanilla_mana, 0);
        assert_eq!(vanilla_colorless, 0); // no produced_mana field at all
    }

    #[test]
    fn captures_legalities() {
        let oracle = r#"[
          {"oracle_id":"oid-banned","name":"Balance","cmc":1.0,"color_identity":["W"],"keywords":[],
           "legalities":{"standard":"not_legal","modern":"banned","legacy":"banned","vintage":"restricted","commander":"banned"}},
          {"oracle_id":"oid-legal","name":"Lightning Bolt","cmc":1.0,"color_identity":["R"],"keywords":[],
           "legalities":{"modern":"legal","legacy":"legal","vintage":"legal","pauper":"legal"}},
          {"oracle_id":"oid-nolegal","name":"Some Test Card","cmc":1.0,"color_identity":[],"keywords":[]}
        ]"#;

        let dir = tempfile::tempdir().unwrap();
        let dest = dir.path().join("reference.db");
        let conn = open_reference_for_build(&dest).unwrap();

        load_oracle_cards(&conn, jsonl(oracle).as_bytes(), &CancelToken::new()).unwrap();

        let fetch = |id: &str| -> Legalities {
            Legalities(
                conn.query_row(
                    "SELECT legalities FROM oracle_cards WHERE oracle_id=?1",
                    params![id],
                    |r| r.get(0),
                )
                .unwrap(),
            )
        };

        let balance = fetch("oid-banned");
        assert_eq!(balance.status_for("Vintage"), crate::model::LegalStatus::Restricted);
        assert_eq!(balance.status_for("Modern"), crate::model::LegalStatus::Banned);
        assert_eq!(balance.status_for("Standard"), crate::model::LegalStatus::NotLegal);

        let bolt = fetch("oid-legal");
        assert_eq!(bolt.status_for("Modern"), crate::model::LegalStatus::Legal);
        assert_eq!(bolt.status_for("Standard"), crate::model::LegalStatus::NotLegal); // absent key

        let vanilla = fetch("oid-nolegal");
        for (_, label) in crate::model::LEGALITY_FORMATS {
            assert_eq!(vanilla.status_for(label), crate::model::LegalStatus::NotLegal); // no legalities object at all
        }
    }

    #[test]
    fn captures_themes_and_edhrec_rank() {
        let oracle = r#"[
          {"oracle_id":"oid-sac","name":"Sac Outlet","cmc":2.0,"color_identity":["B"],"keywords":[],
           "oracle_text":"Whenever a creature you control dies, each opponent loses 1 life.","edhrec_rank":1200},
          {"oracle_id":"oid-vanilla","name":"Grizzly Bears","cmc":2.0,"color_identity":["G"],"keywords":[],
           "oracle_text":"You gain 2 life."}
        ]"#;

        let dir = tempfile::tempdir().unwrap();
        let dest = dir.path().join("reference.db");
        let conn = open_reference_for_build(&dest).unwrap();
        load_oracle_cards(&conn, jsonl(oracle).as_bytes(), &CancelToken::new()).unwrap();

        let fetch = |id: &str| -> (i64, Option<i64>) {
            conn.query_row(
                "SELECT themes, edhrec_rank FROM oracle_cards WHERE oracle_id=?1",
                params![id],
                |r| Ok((r.get(0)?, r.get(1)?)),
            )
            .unwrap()
        };

        let (sac_themes, sac_rank) = fetch("oid-sac");
        assert_ne!(sac_themes & 1, 0, "Sacrifice bit set from payoff phrasing");
        assert_eq!(sac_rank, Some(1200));

        let (van_themes, van_rank) = fetch("oid-vanilla");
        // "You gain 2 life." is an action, not the "whenever you gain life"
        // payoff, so no theme bit; and no edhrec_rank present → NULL.
        assert_eq!(van_themes, 0);
        assert_eq!(van_rank, None);
    }

    #[test]
    fn captures_horizontal_flag_for_battle_and_planar_layouts_only() {
        let oracle = r#"[
          {"oracle_id":"oid-battle","name":"Invasion of Zendikar // Awakened Skyclave",
           "layout":"transform","cmc":4.0,
           "card_faces":[
             {"type_line":"Battle — Siege"},
             {"type_line":"Creature — Elemental"}
           ],
           "color_identity":["G"],"keywords":[]},
          {"oracle_id":"oid-plane","name":"Academy at Tolaria West","layout":"planar","cmc":0.0,
           "type_line":"Plane — Dominaria","color_identity":[],"keywords":[]},
          {"oracle_id":"oid-scheme","name":"A Display of My Dark Power","layout":"scheme","cmc":0.0,
           "type_line":"Scheme","color_identity":[],"keywords":[]},
          {"oracle_id":"oid-normal","name":"Grizzly Bears","layout":"normal","cmc":2.0,
           "type_line":"Creature — Bear","color_identity":["G"],"keywords":[]}
        ]"#;

        let dir = tempfile::tempdir().unwrap();
        let dest = dir.path().join("reference.db");
        let conn = open_reference_for_build(&dest).unwrap();
        load_oracle_cards(&conn, jsonl(oracle).as_bytes(), &CancelToken::new()).unwrap();

        let horizontal = |id: &str| -> bool {
            conn.query_row(
                "SELECT horizontal FROM oracle_cards WHERE oracle_id=?1",
                params![id],
                |r| r.get::<_, i64>(0),
            )
            .unwrap()
                != 0
        };

        // Real Battle cards are `layout: "transform"` (they flip into a
        // creature once defeated) — only the front face carries the Battle
        // type, so detection has to key off that, not the layout string.
        assert!(horizontal("oid-battle"));
        assert!(horizontal("oid-plane"));
        // Schemes are oversized but not rotated — only Battle/planar cards are.
        assert!(!horizontal("oid-scheme"));
        assert!(!horizontal("oid-normal"));
    }

    #[test]
    fn captures_representative_printing_image_from_oracle_cards() {
        // Oracle Cards entries carry Scryfall's own pick of a recognizable
        // printing's images — top-level for single-faced cards, per-face for
        // DFCs (mirrors how `load_printings` reads a `printings` entry).
        let oracle = r#"[
          {"id":"scry-normal","oracle_id":"oid-normal","name":"Grizzly Bears",
           "layout":"normal","cmc":2.0,"type_line":"Creature — Bear",
           "color_identity":["G"],"keywords":[],
           "image_uris":{"normal":"http://img/normal"}},
          {"id":"scry-dfc","oracle_id":"oid-dfc","name":"Delver of Secrets // Insectile Aberration",
           "layout":"transform","cmc":1.0,"color_identity":["U"],"keywords":[],
           "card_faces":[
             {"type_line":"Creature — Human Wizard","image_uris":{"normal":"http://img/front"}},
             {"type_line":"Creature — Human Insect","image_uris":{"normal":"http://img/back"}}
           ]},
          {"oracle_id":"oid-noimg","name":"No Image Test","layout":"normal","cmc":0.0,
           "type_line":"Land","color_identity":[],"keywords":[]}
        ]"#;

        let dir = tempfile::tempdir().unwrap();
        let dest = dir.path().join("reference.db");
        let conn = open_reference_for_build(&dest).unwrap();
        load_oracle_cards(&conn, jsonl(oracle).as_bytes(), &CancelToken::new()).unwrap();

        let images = |id: &str| -> (String, String, String) {
            conn.query_row(
                "SELECT image_uri, back_image_uri, default_scryfall_id FROM oracle_cards WHERE oracle_id=?1",
                params![id],
                |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
            )
            .unwrap()
        };

        assert_eq!(
            images("oid-normal"),
            (
                "http://img/normal".to_string(),
                "".to_string(),
                "scry-normal".to_string()
            )
        );
        assert_eq!(
            images("oid-dfc"),
            (
                "http://img/front".to_string(),
                "http://img/back".to_string(),
                "scry-dfc".to_string()
            )
        );
        assert_eq!(images("oid-noimg"), ("".to_string(), "".to_string(), "".to_string()));
    }

    #[test]
    fn captures_back_face_image_for_double_faced_cards() {
        // Transform/MDFC cards: two `card_faces`, each with its own image.
        let dfc = r#"[
          {"id":"p-dfc","oracle_id":"oid-dfc","set":"znr","collector_number":"1",
           "rarity":"rare",
           "card_faces":[
             {"image_uris":{"normal":"http://img/front"}},
             {"image_uris":{"normal":"http://img/back"}}
           ]}
        ]"#;
        // Split card: two `card_faces` for oracle text, but no per-face images —
        // the whole card is one combined top-level image.
        let split = r#"[
          {"id":"p-split","oracle_id":"oid-split","set":"grn","collector_number":"2",
           "rarity":"common","image_uris":{"normal":"http://img/split-combined"},
           "card_faces":[{"image_uris":null},{"image_uris":null}]}
        ]"#;

        let dir = tempfile::tempdir().unwrap();
        let dest = dir.path().join("reference.db");
        let conn = open_reference_for_build(&dest).unwrap();

        load_printings(&conn, jsonl(dfc).as_bytes(), &CancelToken::new()).unwrap();
        load_printings(&conn, jsonl(split).as_bytes(), &CancelToken::new()).unwrap();

        let (front, back): (String, String) = conn
            .query_row(
                "SELECT image_uri, back_image_uri FROM printings WHERE scryfall_id='p-dfc'",
                [],
                |r| Ok((r.get(0)?, r.get(1)?)),
            )
            .unwrap();
        assert_eq!(front, "http://img/front");
        assert_eq!(back, "http://img/back");

        let (front, back): (String, String) = conn
            .query_row(
                "SELECT image_uri, back_image_uri FROM printings WHERE scryfall_id='p-split'",
                [],
                |r| Ok((r.get(0)?, r.get(1)?)),
            )
            .unwrap();
        assert_eq!(front, "http://img/split-combined");
        assert_eq!(back, "", "split cards have no distinct back face");
    }

    #[test]
    fn captures_usd_price_defaulting_to_zero_when_absent() {
        let printings = r#"[
          {"id":"p-priced","oracle_id":"oid-priced","set":"m19","collector_number":"1",
           "rarity":"common","prices":{"usd":"12.34","usd_foil":"20.00"}},
          {"id":"p-null","oracle_id":"oid-null","set":"m19","collector_number":"2",
           "rarity":"common","prices":{"usd":null}},
          {"id":"p-noprices","oracle_id":"oid-noprices","set":"m19","collector_number":"3",
           "rarity":"common"}
        ]"#;

        let dir = tempfile::tempdir().unwrap();
        let dest = dir.path().join("reference.db");
        let conn = open_reference_for_build(&dest).unwrap();

        load_printings(&conn, jsonl(printings).as_bytes(), &CancelToken::new()).unwrap();

        let fetch = |id: &str| -> f64 {
            conn.query_row(
                "SELECT price_usd FROM printings WHERE scryfall_id=?1",
                params![id],
                |r| r.get(0),
            )
            .unwrap()
        };
        assert_eq!(fetch("p-priced"), 12.34);
        assert_eq!(fetch("p-null"), 0.0);
        assert_eq!(fetch("p-noprices"), 0.0);
    }
}