garrettmills/mtgc
1
0
Fork 0
Code Issues 1 Pull Requests Releases Activity
Files
2b361c4a854bf71f3d6735b2f5898fc7769b3264
mtgc/crates/mtgc-core/src/ingest.rs
garrettmills 2b361c4a85 Add a Card Search tab and fix horizontal-card (Battle/Plane) preview orientation 
Card Search browses the whole downloaded card universe (not just owned
cards), reusing the Library tab's filter/group/sort/list/tiles UI; results
show ownership and offer quick "own" and "add to deck" actions.

Battles, Planes, and Phenomena store their Scryfall image portrait-shaped
with the content itself rotated, so hover/click previews now detect this
(a new `horizontal` flag threaded through ingest and every card query, with
graceful fallback for a reference.db built before the column existed) and
rotate the image back into place. Single-faced and non-DFC cases are
verified fixed; a double-faced Battle's front face still needs another pass.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-07-22 20:55:17 -05:00

1132 lines
40 KiB
Rust
Raw Blame History

//! Build `reference.db` from Scryfall bulk data.
//!
//! Two passes: the *Oracle Cards* bulk (one entry per `oracle_id`) fills
//! `oracle_cards` + the normalized type/subtype/keyword tables + the name FTS;
//! the *Default Cards* bulk (one entry per printing) fills `printings`.
//!
//! Both bulk files are large JSON arrays, so we stream them element-by-element
//! rather than loading the whole array into memory.
use std::io::{BufReader, Read};
use std::marker::PhantomData;
use std::path::Path;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use rusqlite::{params, Connection};
use serde::de::{DeserializeOwned, SeqAccess, Visitor};
use serde::Deserialize;
use crate::db::open_reference_for_build;
use crate::error::{CoreError, Result};
use crate::model::{ColorSet, Legalities};
const BULK_INDEX_URL: &str = "https://api.scryfall.com/bulk-data";
const USER_AGENT: &str = "mtgc/0.1 (personal library catalog)";
const STAGE_INDEX: &str = "index";
const STAGE_ORACLE: &str = "oracle";
const STAGE_PRINTINGS: &str = "printings";
const STAGE_OPTIMIZE: &str = "optimize";
/// Row counts from a build, for reporting/verification.
#[derive(Debug, Default, Clone, Copy)]
pub struct IngestStats {
pub oracle_cards: usize,
pub printings: usize,
}
/// A flag a caller can flip from another thread to abort an in-progress
/// ingest at the next checkpoint (checked once per parsed record — thousands
/// of times a second, so cancellation is near-instant).
#[derive(Clone, Default)]
pub struct CancelToken(Arc<AtomicBool>);
impl CancelToken {
pub fn new() -> Self {
Self(Arc::new(AtomicBool::new(false)))
}
pub fn cancel(&self) {
self.0.store(true, Ordering::Relaxed);
}
pub fn is_cancelled(&self) -> bool {
self.0.load(Ordering::Relaxed)
}
}
/// A cancellation-triggered early exit from inside `stream_json_array`'s
/// per-item closure round-trips through `serde_json::Error::custom`, which
/// stringifies (and appends position info to) whatever error it's given. Undo
/// that mangling here so callers can match on a clean [`CoreError::Cancelled`].
fn override_if_cancelled(e: CoreError, cancel: &CancelToken) -> CoreError {
if cancel.is_cancelled() {
CoreError::Cancelled
} else {
e
}
}
/// One segment of the overall ingest, with its share of the total work
/// (weights across all stages sum to 1.0). Lets the UI render a single
/// multi-segment progress bar instead of swapping bars per stage.
#[derive(Debug, Clone, serde::Serialize)]
pub struct IngestStageInfo {
pub id: &'static str,
pub label: &'static str,
pub weight: f64,
}
/// A single progress update from a running ingest. `stage` is the id of the
/// currently active [`IngestStageInfo`]; `message` is a human-readable detail
/// for that stage; `overall_percent` is cumulative progress (0.0..=100.0)
/// across *all* stages, ready to drive a single combined bar; `stages` is the
/// (static, for a given run) segment layout for rendering.
#[derive(Debug, Clone, serde::Serialize)]
pub struct IngestProgress {
pub stage: &'static str,
pub message: String,
pub overall_percent: f64,
pub stages: Vec<IngestStageInfo>,
}
/// Fixed weight for the two bookkeeping stages (fetching the bulk index,
/// optimizing the FTS index); the remaining budget is split between the two
/// downloads by their known byte size, since both are network-bound.
const INDEX_WEIGHT: f64 = 0.02;
const OPTIMIZE_WEIGHT: f64 = 0.03;
fn stage_plan(oracle_size: u64, default_size: u64) -> Vec<IngestStageInfo> {
let dl_total = (oracle_size + default_size).max(1) as f64;
let dl_budget = 1.0 - INDEX_WEIGHT - OPTIMIZE_WEIGHT;
let oracle_weight = dl_budget * (oracle_size as f64 / dl_total);
let printings_weight = dl_budget - oracle_weight;
vec![
IngestStageInfo {
id: STAGE_INDEX,
label: "Index",
weight: INDEX_WEIGHT,
},
IngestStageInfo {
id: STAGE_ORACLE,
label: "Card data",
weight: oracle_weight,
},
IngestStageInfo {
id: STAGE_PRINTINGS,
label: "Printings",
weight: printings_weight,
},
IngestStageInfo {
id: STAGE_OPTIMIZE,
label: "Indexing",
weight: OPTIMIZE_WEIGHT,
},
]
}
/// Cumulative percent through `stages` once `active` has completed
/// `local_fraction` (0.0..=1.0) of its own work.
fn overall_percent(stages: &[IngestStageInfo], active: &str, local_fraction: f64) -> f64 {
let mut acc = 0.0;
for s in stages {
if s.id == active {
return ((acc + s.weight * local_fraction) * 100.0).min(100.0);
}
acc += s.weight;
}
(acc * 100.0).min(100.0)
}
/// Download the latest Scryfall bulk data and (re)build `reference.db` at
/// `dest`. Builds into a temp file alongside `dest` and only replaces it on
/// full success, so a cancelled or failed update never destroys a working
/// reference db.
pub fn build_reference_db(
dest: &Path,
cancel: &CancelToken,
mut progress: impl FnMut(IngestProgress),
) -> Result<IngestStats> {
let tmp = dest.with_extension("db.tmp");
let result = build_into(&tmp, cancel, &mut progress);
match &result {
Ok(_) => {
let _ = std::fs::remove_file(dest);
std::fs::rename(&tmp, dest)?;
}
Err(_) => {
let _ = std::fs::remove_file(&tmp);
}
}
result
}
fn build_into(
tmp: &Path,
cancel: &CancelToken,
progress: &mut impl FnMut(IngestProgress),
) -> Result<IngestStats> {
// Sizes aren't known until we've hit the bulk index, so the very first
// event uses a placeholder 50/50 split; the real plan follows within the
// same (fast) request/response round trip.
let mut stages = stage_plan(1, 1);
progress(IngestProgress {
stage: STAGE_INDEX,
message: "Fetching Scryfall bulk index…".into(),
overall_percent: 0.0,
stages: stages.clone(),
});
if cancel.is_cancelled() {
return Err(CoreError::Cancelled);
}
let bulk = fetch_bulk_uris()?;
stages = stage_plan(bulk.oracle_size, bulk.default_size);
if tmp.exists() {
std::fs::remove_file(tmp)?;
}
let conn = open_reference_for_build(tmp)?;
// Bulk-load fast; this db is disposable/rebuildable so durability is moot.
conn.pragma_update(None, "journal_mode", "MEMORY")?;
conn.pragma_update(None, "synchronous", "OFF")?;
progress(IngestProgress {
stage: STAGE_ORACLE,
message: "Downloading card data…".into(),
overall_percent: overall_percent(&stages, STAGE_ORACLE, 0.0),
stages: stages.clone(),
});
let reader = http_reader(&bulk.oracle_uri)?;
let reader = progress_reader(
reader,
bulk.oracle_size,
STAGE_ORACLE,
"Downloading card data",
&stages,
progress,
);
let oracle_cards = load_oracle_cards(&conn, reader, cancel).map_err(|e| override_if_cancelled(e, cancel))?;
progress(IngestProgress {
stage: STAGE_PRINTINGS,
message: "Downloading printings…".into(),
overall_percent: overall_percent(&stages, STAGE_PRINTINGS, 0.0),
stages: stages.clone(),
});
let reader = http_reader(&bulk.default_uri)?;
let reader = progress_reader(
reader,
bulk.default_size,
STAGE_PRINTINGS,
"Downloading printings",
&stages,
progress,
);
let printings = load_printings(&conn, reader, cancel).map_err(|e| override_if_cancelled(e, cancel))?;
if cancel.is_cancelled() {
return Err(CoreError::Cancelled);
}
progress(IngestProgress {
stage: STAGE_OPTIMIZE,
message: "Optimizing search index…".into(),
overall_percent: overall_percent(&stages, STAGE_OPTIMIZE, 0.5),
stages: stages.clone(),
});
conn.execute_batch("INSERT INTO oracle_names_fts(oracle_names_fts) VALUES('optimize');")?;
Ok(IngestStats {
oracle_cards,
printings,
})
}
/// Human-readable byte size, e.g. `42.3 MB`.
fn human_mb(bytes: u64) -> String {
format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0))
}
/// Wrap a reader so that, as it's consumed (concurrently with JSON parsing),
/// it periodically reports download progress against the overall multi-stage
/// bar. Throttled to avoid flooding the UI with an event per network read.
fn progress_reader<'a, R: Read>(
inner: R,
total: u64,
stage: &'static str,
label: &'static str,
stages: &'a [IngestStageInfo],
progress: &'a mut impl FnMut(IngestProgress),
) -> impl Read + 'a
where
R: 'a,
{
const EMIT_EVERY: u64 = 512 * 1024; // 512 KB
ProgressReader {
inner,
read: 0,
last_emit: 0,
total,
on_progress: move |read: u64, total: u64| {
let message = format!("{label}… {} / {}", human_mb(read), human_mb(total));
let fraction = if total > 0 {
(read as f64 / total as f64).min(1.0)
} else {
1.0
};
progress(IngestProgress {
stage,
message,
overall_percent: overall_percent(stages, stage, fraction),
stages: stages.to_vec(),
});
},
threshold: EMIT_EVERY,
}
}
struct ProgressReader<R, F> {
inner: R,
read: u64,
last_emit: u64,
total: u64,
threshold: u64,
on_progress: F,
}
impl<R: Read, F: FnMut(u64, u64)> Read for ProgressReader<R, F> {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
let n = self.inner.read(buf)?;
self.read += n as u64;
if n == 0 || self.read - self.last_emit >= self.threshold {
self.last_emit = self.read;
(self.on_progress)(self.read, self.total);
}
Ok(n)
}
}
// ---- Scryfall bulk index -------------------------------------------------
#[derive(Deserialize)]
struct BulkIndex {
data: Vec<BulkEntry>,
}
#[derive(Deserialize)]
struct BulkEntry {
#[serde(rename = "type")]
kind: String,
download_uri: String,
/// Uncompressed byte size of `download_uri`'s contents, per the Scryfall
/// bulk-data API — used for download progress since the actual download
/// response is chunked (no `Content-Length` header to rely on).
size: u64,
}
struct BulkUris {
oracle_uri: String,
oracle_size: u64,
default_uri: String,
default_size: u64,
}
fn fetch_bulk_uris() -> Result<BulkUris> {
let index: BulkIndex = ureq::get(BULK_INDEX_URL)
.set("User-Agent", USER_AGENT)
.set("Accept", "application/json")
.call()?
.into_json()?;
let find = |kind: &str| {
index
.data
.iter()
.find(|e| e.kind == kind)
.ok_or_else(|| CoreError::Other(format!("Scryfall bulk index missing '{kind}'")))
};
let oracle = find("oracle_cards")?;
let default = find("default_cards")?;
Ok(BulkUris {
oracle_uri: oracle.download_uri.clone(),
oracle_size: oracle.size,
default_uri: default.download_uri.clone(),
default_size: default.size,
})
}
fn http_reader(url: &str) -> Result<impl Read> {
let resp = ureq::get(url).set("User-Agent", USER_AGENT).call()?;
Ok(BufReader::with_capacity(1 << 20, resp.into_reader()))
}
// ---- Streaming array parse ----------------------------------------------
/// Deserialize a top-level JSON array element-by-element, invoking `f` per item
/// without holding the whole array in memory.
fn stream_json_array<R, T, F>(reader: R, f: F) -> Result<()>
where
R: Read,
T: DeserializeOwned,
F: FnMut(T) -> Result<()>,
{
struct SeqVisitor<T, F> {
f: F,
_marker: PhantomData<fn() -> T>,
}
impl<'de, T, F> Visitor<'de> for SeqVisitor<T, F>
where
T: DeserializeOwned,
F: FnMut(T) -> Result<()>,
{
type Value = ();
fn expecting(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
fmt.write_str("a JSON array of objects")
}
fn visit_seq<A>(mut self, mut seq: A) -> std::result::Result<(), A::Error>
where
A: SeqAccess<'de>,
{
while let Some(item) = seq.next_element::<T>()? {
(self.f)(item).map_err(serde::de::Error::custom)?;
}
Ok(())
}
}
let mut de = serde_json::Deserializer::from_reader(reader);
serde::de::Deserializer::deserialize_seq(
&mut de,
SeqVisitor {
f,
_marker: PhantomData,
},
)?;
Ok(())
}
// ---- Oracle pass ---------------------------------------------------------
#[derive(Deserialize)]
struct ScryOracle {
oracle_id: Option<String>,
name: String,
#[serde(default)]
layout: String,
#[serde(default)]
cmc: f64,
#[serde(default)]
colors: Option<Vec<String>>,
#[serde(default)]
color_identity: Vec<String>,
#[serde(default)]
type_line: Option<String>,
#[serde(default)]
oracle_text: Option<String>,
#[serde(default)]
keywords: Vec<String>,
#[serde(default)]
card_faces: Option<Vec<ScryFace>>,
/// What colors of mana this card can add (lands, mana rocks, dorks). A
/// top-level-only Scryfall field — not per-face, even on DFCs.
#[serde(default)]
produced_mana: Option<Vec<String>>,
/// Format-slug -> status-string (e.g. "legal"/"not_legal"/"banned"/
/// "restricted"). See [`crate::model::Legalities`].
#[serde(default)]
legalities: std::collections::HashMap<String, String>,
/// Scryfall's EDHREC popularity rank (lower = more played); absent for
/// some cards. Kept nullable — see `synergy::CardSuggestion::edhrec_rank`.
#[serde(default)]
edhrec_rank: Option<i64>,
}
#[derive(Deserialize)]
struct ScryFace {
#[serde(default)]
type_line: Option<String>,
#[serde(default)]
colors: Option<Vec<String>>,
#[serde(default)]
oracle_text: Option<String>,
}
fn load_oracle_cards(conn: &Connection, reader: impl Read, cancel: &CancelToken) -> Result<usize> {
conn.execute_batch("BEGIN")?;
let mut count = 0usize;
{
let mut ins_card = conn.prepare(
"INSERT OR IGNORE INTO oracle_cards
(oracle_id,name,name_lower,mana_value,colors,color_identity,type_line,oracle_text,
produced_mana,produces_colorless,legalities,themes,edhrec_rank,horizontal)
VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14)",
)?;
let mut ins_fts =
conn.prepare("INSERT INTO oracle_names_fts(name,oracle_id) VALUES (?1,?2)")?;
let mut ins_type =
conn.prepare("INSERT INTO card_types(oracle_id,type) VALUES (?1,?2)")?;
let mut ins_sub =
conn.prepare("INSERT INTO card_subtypes(oracle_id,subtype) VALUES (?1,?2)")?;
let mut ins_kw =
conn.prepare("INSERT INTO card_keywords(oracle_id,keyword) VALUES (?1,?2)")?;
stream_json_array::<_, ScryOracle, _>(reader, |c| {
if cancel.is_cancelled() {
return Err(CoreError::Cancelled);
}
let Some(oracle_id) = c.oracle_id.clone() else {
return Ok(()); // rare face-only records without a top-level oracle_id
};
// Art Series cards (Secret Lair) aren't real playable cards — exclude
// them from search/library entirely.
if c.layout == "art_series" {
return Ok(());
}
// Colors: prefer top-level, else union of faces.
let colors = match &c.colors {
Some(cs) => ColorSet::from_letters(cs),
None => {
let mut m = 0u8;
if let Some(faces) = &c.card_faces {
for f in faces {
if let Some(cs) = &f.colors {
m |= ColorSet::from_letters(cs).0;
}
}
}
ColorSet(m)
}
};
let identity = ColorSet::from_letters(&c.color_identity);
// Type line: top-level, else join faces.
let type_line = c.type_line.clone().unwrap_or_else(|| {
c.card_faces
.as_ref()
.map(|faces| {
faces
.iter()
.filter_map(|f| f.type_line.clone())
.collect::<Vec<_>>()
.join(" // ")
})
.unwrap_or_default()
});
// Oracle text: top-level, else join faces — double-faced cards (MDFC,
// transform, etc.) never carry a top-level oracle_text in Scryfall's
// data, only per-face text.
let oracle_text = c.oracle_text.clone().unwrap_or_else(|| {
c.card_faces
.as_ref()
.map(|faces| {
faces
.iter()
.filter_map(|f| f.oracle_text.clone())
.collect::<Vec<_>>()
.join("\n")
})
.unwrap_or_default()
});
// Mana production: colored letters map to the WUBRG bitmask (same as
// colors/identity); "C" (colorless) is tracked separately since it
// isn't a WUBRG bit. Top-level only — Scryfall doesn't put this on
// individual card_faces.
let produced = c.produced_mana.as_deref().unwrap_or(&[]);
let produced_mana = ColorSet::from_letters(produced);
let produces_colorless = produced.iter().any(|s| s.eq_ignore_ascii_case("C"));
let legalities = Legalities::from_scryfall(&c.legalities);
let themes = crate::synergy::compute_themes(&oracle_text);
// Battles and Planes/Phenomena store their card image portrait-shaped
// with the content itself rotated — viewers must rotate it 90° to
// read it right-side up (verified against real Scryfall images).
let horizontal = matches!(c.layout.as_str(), "battle" | "planar");
let changed = ins_card.execute(params![
oracle_id,
c.name,
c.name.to_lowercase(),
c.cmc,
colors.0 as i64,
identity.0 as i64,
type_line,
oracle_text,
produced_mana.0 as i64,
produces_colorless as i64,
legalities.0,
themes,
c.edhrec_rank,
horizontal as i64,
])?;
// Only emit satellite rows the first time we see this oracle_id.
if changed == 1 {
ins_fts.execute(params![c.name, oracle_id])?;
let (types, subtypes) = parse_type_line(&type_line);
for t in types {
ins_type.execute(params![oracle_id, t])?;
}
for s in subtypes {
ins_sub.execute(params![oracle_id, s])?;
}
let mut kws = c.keywords.clone();
kws.sort();
kws.dedup();
for k in kws {
ins_kw.execute(params![oracle_id, k])?;
}
count += 1;
}
Ok(())
})?;
}
conn.execute_batch("COMMIT")?;
Ok(count)
}
/// Split a Scryfall type line into (types+supertypes, subtypes). Handles split /
/// double-faced lines joined by " // ". The separator is the em dash '—'.
pub fn parse_type_line(type_line: &str) -> (Vec<String>, Vec<String>) {
let mut types = Vec::new();
let mut subtypes = Vec::new();
for part in type_line.split("//") {
let part = part.trim();
let mut halves = part.splitn(2, '—');
let left = halves.next().unwrap_or("").trim();
let right = halves.next().unwrap_or("").trim();
for t in left.split_whitespace() {
types.push(t.to_string());
}
for s in right.split_whitespace() {
subtypes.push(s.to_string());
}
}
types.sort();
types.dedup();
subtypes.sort();
subtypes.dedup();
(types, subtypes)
}
// ---- Printings pass ------------------------------------------------------
#[derive(Deserialize)]
struct ScryPrinting {
id: String,
oracle_id: Option<String>,
#[serde(default)]
set: String,
#[serde(default)]
collector_number: String,
#[serde(default)]
finishes: Vec<String>,
#[serde(default)]
rarity: String,
#[serde(default)]
image_uris: Option<ImageUris>,
#[serde(default)]
card_faces: Option<Vec<PrintFace>>,
#[serde(default)]
prices: Option<ScryPrices>,
}
#[derive(Deserialize)]
struct ImageUris {
#[serde(default)]
normal: Option<String>,
}
#[derive(Deserialize)]
struct ScryPrices {
#[serde(default)]
usd: Option<String>,
}
#[derive(Deserialize)]
struct PrintFace {
#[serde(default)]
image_uris: Option<ImageUris>,
}
fn load_printings(conn: &Connection, reader: impl Read, cancel: &CancelToken) -> Result<usize> {
conn.execute_batch("BEGIN")?;
let mut count = 0usize;
{
let mut ins = conn.prepare(
"INSERT OR IGNORE INTO printings
(scryfall_id,oracle_id,set_code,collector_number,finishes,rarity,image_uri,back_image_uri,price_usd)
VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9)",
)?;
stream_json_array::<_, ScryPrinting, _>(reader, |p| {
if cancel.is_cancelled() {
return Err(CoreError::Cancelled);
}
let Some(oracle_id) = p.oracle_id.clone() else {
return Ok(());
};
let image = p
.image_uris
.as_ref()
.and_then(|i| i.normal.clone())
.or_else(|| {
p.card_faces
.as_ref()
.and_then(|f| f.first())
.and_then(|f| f.image_uris.as_ref())
.and_then(|i| i.normal.clone())
})
.unwrap_or_default();
// Only transform/modal-DFC/reversible cards carry per-face images
// with a distinct second face; split/adventure cards share one
// combined top-level image and have no per-face `image_uris`.
let back_image = p
.card_faces
.as_ref()
.and_then(|f| f.get(1))
.and_then(|f| f.image_uris.as_ref())
.and_then(|i| i.normal.clone())
.unwrap_or_default();
let price_usd = p
.prices
.as_ref()
.and_then(|pr| pr.usd.as_ref())
.and_then(|s| s.parse::<f64>().ok())
.unwrap_or(0.0);
ins.execute(params![
p.id,
oracle_id,
p.set,
p.collector_number,
p.finishes.join(","),
p.rarity,
image,
back_image,
price_usd,
])?;
count += 1;
Ok(())
})?;
}
conn.execute_batch("COMMIT")?;
Ok(count)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parses_type_lines() {
let (t, s) = parse_type_line("Legendary Creature — Elf Druid");
assert_eq!(t, vec!["Creature", "Legendary"]);
assert_eq!(s, vec!["Druid", "Elf"]);
let (t, s) = parse_type_line("Instant");
assert_eq!(t, vec!["Instant"]);
assert!(s.is_empty());
// Split card.
let (t, s) = parse_type_line("Sorcery // Sorcery");
assert_eq!(t, vec!["Sorcery"]);
assert!(s.is_empty());
}
#[test]
fn streams_array_of_objects() {
#[derive(serde::Deserialize)]
struct Item {
n: i32,
}
let json = br#"[{"n":1},{"n":2},{"n":3}]"#;
let mut sum = 0;
stream_json_array::<_, Item, _>(&json[..], |it| {
sum += it.n;
Ok(())
})
.unwrap();
assert_eq!(sum, 6);
}
#[test]
fn excludes_art_series_cards() {
let oracle = r#"[
{"oracle_id":"oid-1","name":"Llanowar Elves","cmc":1.0,
"colors":["G"],"color_identity":["G"],
"type_line":"Creature — Elf Druid","keywords":[]},
{"oracle_id":"oid-2","name":"Llanowar Elves","layout":"art_series",
"type_line":"Card","keywords":[]}
]"#;
let dir = tempfile::tempdir().unwrap();
let dest = dir.path().join("reference.db");
let conn = open_reference_for_build(&dest).unwrap();
let oc = load_oracle_cards(&conn, oracle.as_bytes(), &CancelToken::new()).unwrap();
assert_eq!(oc, 1);
let count: i64 = conn
.query_row(
"SELECT count(*) FROM oracle_cards WHERE oracle_id='oid-2'",
[],
|r| r.get(0),
)
.unwrap();
assert_eq!(count, 0);
}
#[test]
fn cancel_token_aborts_load_with_cancelled_error() {
let oracle = r#"[
{"oracle_id":"oid-1","name":"Llanowar Elves","cmc":1.0,
"colors":["G"],"color_identity":["G"],
"type_line":"Creature — Elf Druid","keywords":[]},
{"oracle_id":"oid-2","name":"Serra Angel","cmc":5.0,
"colors":["W"],"color_identity":["W"],
"type_line":"Creature — Angel","keywords":["Flying","Vigilance"]}
]"#;
let dir = tempfile::tempdir().unwrap();
let dest = dir.path().join("reference.db");
let conn = open_reference_for_build(&dest).unwrap();
let cancel = CancelToken::new();
cancel.cancel();
let err = load_oracle_cards(&conn, oracle.as_bytes(), &cancel)
.map_err(|e| override_if_cancelled(e, &cancel))
.unwrap_err();
assert!(matches!(err, CoreError::Cancelled));
}
#[test]
fn stage_plan_weights_sum_to_one() {
let stages = stage_plan(171_000_000, 530_000_000);
let total: f64 = stages.iter().map(|s| s.weight).sum();
assert!((total - 1.0).abs() < 1e-9);
// The larger download gets the larger share of the two DL stages.
let oracle_w = stages.iter().find(|s| s.id == STAGE_ORACLE).unwrap().weight;
let printings_w = stages
.iter()
.find(|s| s.id == STAGE_PRINTINGS)
.unwrap()
.weight;
assert!(printings_w > oracle_w);
}
#[test]
fn oracle_and_printing_load_from_fixture_json() {
// Minimal Scryfall-shaped fixtures exercising the insert paths.
let oracle = r#"[
{"oracle_id":"oid-1","name":"Llanowar Elves","cmc":1.0,
"colors":["G"],"color_identity":["G"],
"type_line":"Creature — Elf Druid","keywords":[]},
{"oracle_id":"oid-2","name":"Serra Angel","cmc":5.0,
"colors":["W"],"color_identity":["W"],
"type_line":"Creature — Angel","keywords":["Flying","Vigilance"]}
]"#;
let printings = r#"[
{"id":"p-1","oracle_id":"oid-1","set":"m19","collector_number":"310",
"finishes":["nonfoil","foil"],"rarity":"common",
"image_uris":{"normal":"http://img/1"}}
]"#;
let dir = tempfile::tempdir().unwrap();
let dest = dir.path().join("reference.db");
let conn = open_reference_for_build(&dest).unwrap();
let oc = load_oracle_cards(&conn, oracle.as_bytes(), &CancelToken::new()).unwrap();
let pc = load_printings(&conn, printings.as_bytes(), &CancelToken::new()).unwrap();
assert_eq!(oc, 2);
assert_eq!(pc, 1);
// Keyword table populated.
let flyers: i64 = conn
.query_row(
"SELECT count(*) FROM card_keywords WHERE keyword='Flying'",
[],
|r| r.get(0),
)
.unwrap();
assert_eq!(flyers, 1);
// Bitmask colors stored.
let g: i64 = conn
.query_row(
"SELECT colors FROM oracle_cards WHERE oracle_id='oid-1'",
[],
|r| r.get(0),
)
.unwrap();
assert_eq!(g as u8, crate::model::G);
}
#[test]
fn double_faced_cards_fall_back_to_card_faces_for_type_colors_and_text() {
// Modal DFCs (and transform cards) never carry a top-level type_line,
// colors, or oracle_text in Scryfall's oracle-cards bulk — only
// per-face fields. Regression test for a bug where oracle_text had no
// fallback (unlike type_line/colors), silently leaving every
// double-faced card's oracle_text empty and unsearchable.
let oracle = r#"[
{"oracle_id":"oid-dfc","name":"Elesh Norn // The Argent Etchings",
"cmc":5.0,"color_identity":["W"],"keywords":[],
"card_faces":[
{"type_line":"Legendary Creature — Phyrexian Praetor","colors":["W"],
"oracle_text":"Whenever a source an opponent controls deals damage to you or a permanent you control, exile that source."},
{"type_line":"Battle — Siege","colors":[],
"oracle_text":"When you defeated The Argent Etchings, draw a card."}
]}
]"#;
let dir = tempfile::tempdir().unwrap();
let dest = dir.path().join("reference.db");
let conn = open_reference_for_build(&dest).unwrap();
load_oracle_cards(&conn, oracle.as_bytes(), &CancelToken::new()).unwrap();
let (type_line, colors, oracle_text): (String, i64, String) = conn
.query_row(
"SELECT type_line, colors, oracle_text FROM oracle_cards WHERE oracle_id='oid-dfc'",
[],
|r| Ok((r.get(0)?, r.get(1)?, r.get(2)?)),
)
.unwrap();
assert_eq!(type_line, "Legendary Creature — Phyrexian Praetor // Battle — Siege");
assert_eq!(colors as u8, crate::model::W);
assert!(oracle_text.contains("an opponent controls deals damage"));
assert!(oracle_text.contains("draw a card"));
}
#[test]
fn captures_produced_mana_including_colorless() {
let oracle = r#"[
{"oracle_id":"oid-dual","name":"Overgrown Tomb","cmc":0.0,
"color_identity":[],"keywords":[],"produced_mana":["B","G"]},
{"oracle_id":"oid-wastes","name":"Wastes","cmc":0.0,
"color_identity":[],"keywords":[],"produced_mana":["C"]},
{"oracle_id":"oid-vanilla","name":"Grizzly Bears","cmc":2.0,
"color_identity":["G"],"keywords":[]}
]"#;
let dir = tempfile::tempdir().unwrap();
let dest = dir.path().join("reference.db");
let conn = open_reference_for_build(&dest).unwrap();
load_oracle_cards(&conn, oracle.as_bytes(), &CancelToken::new()).unwrap();
let fetch = |id: &str| -> (i64, i64) {
conn.query_row(
"SELECT produced_mana, produces_colorless FROM oracle_cards WHERE oracle_id=?1",
params![id],
|r| Ok((r.get(0)?, r.get(1)?)),
)
.unwrap()
};
let (dual_mana, dual_colorless) = fetch("oid-dual");
assert_eq!(dual_mana as u8, crate::model::B | crate::model::G);
assert_eq!(dual_colorless, 0);
let (wastes_mana, wastes_colorless) = fetch("oid-wastes");
assert_eq!(wastes_mana, 0); // "C" isn't a WUBRG bit
assert_eq!(wastes_colorless, 1);
let (vanilla_mana, vanilla_colorless) = fetch("oid-vanilla");
assert_eq!(vanilla_mana, 0);
assert_eq!(vanilla_colorless, 0); // no produced_mana field at all
}
#[test]
fn captures_legalities() {
let oracle = r#"[
{"oracle_id":"oid-banned","name":"Balance","cmc":1.0,"color_identity":["W"],"keywords":[],
"legalities":{"standard":"not_legal","modern":"banned","legacy":"banned","vintage":"restricted","commander":"banned"}},
{"oracle_id":"oid-legal","name":"Lightning Bolt","cmc":1.0,"color_identity":["R"],"keywords":[],
"legalities":{"modern":"legal","legacy":"legal","vintage":"legal","pauper":"legal"}},
{"oracle_id":"oid-nolegal","name":"Some Test Card","cmc":1.0,"color_identity":[],"keywords":[]}
]"#;
let dir = tempfile::tempdir().unwrap();
let dest = dir.path().join("reference.db");
let conn = open_reference_for_build(&dest).unwrap();
load_oracle_cards(&conn, oracle.as_bytes(), &CancelToken::new()).unwrap();
let fetch = |id: &str| -> Legalities {
Legalities(
conn.query_row(
"SELECT legalities FROM oracle_cards WHERE oracle_id=?1",
params![id],
|r| r.get(0),
)
.unwrap(),
)
};
let balance = fetch("oid-banned");
assert_eq!(balance.status_for("Vintage"), crate::model::LegalStatus::Restricted);
assert_eq!(balance.status_for("Modern"), crate::model::LegalStatus::Banned);
assert_eq!(balance.status_for("Standard"), crate::model::LegalStatus::NotLegal);
let bolt = fetch("oid-legal");
assert_eq!(bolt.status_for("Modern"), crate::model::LegalStatus::Legal);
assert_eq!(bolt.status_for("Standard"), crate::model::LegalStatus::NotLegal); // absent key
let vanilla = fetch("oid-nolegal");
for (_, label) in crate::model::LEGALITY_FORMATS {
assert_eq!(vanilla.status_for(label), crate::model::LegalStatus::NotLegal); // no legalities object at all
}
}
#[test]
fn captures_themes_and_edhrec_rank() {
let oracle = r#"[
{"oracle_id":"oid-sac","name":"Sac Outlet","cmc":2.0,"color_identity":["B"],"keywords":[],
"oracle_text":"Whenever a creature you control dies, each opponent loses 1 life.","edhrec_rank":1200},
{"oracle_id":"oid-vanilla","name":"Grizzly Bears","cmc":2.0,"color_identity":["G"],"keywords":[],
"oracle_text":"You gain 2 life."}
]"#;
let dir = tempfile::tempdir().unwrap();
let dest = dir.path().join("reference.db");
let conn = open_reference_for_build(&dest).unwrap();
load_oracle_cards(&conn, oracle.as_bytes(), &CancelToken::new()).unwrap();
let fetch = |id: &str| -> (i64, Option<i64>) {
conn.query_row(
"SELECT themes, edhrec_rank FROM oracle_cards WHERE oracle_id=?1",
params![id],
|r| Ok((r.get(0)?, r.get(1)?)),
)
.unwrap()
};
let (sac_themes, sac_rank) = fetch("oid-sac");
assert_ne!(sac_themes & 1, 0, "Sacrifice bit set from payoff phrasing");
assert_eq!(sac_rank, Some(1200));
let (van_themes, van_rank) = fetch("oid-vanilla");
// "You gain 2 life." is an action, not the "whenever you gain life"
// payoff, so no theme bit; and no edhrec_rank present → NULL.
assert_eq!(van_themes, 0);
assert_eq!(van_rank, None);
}
#[test]
fn captures_horizontal_flag_for_battle_and_planar_layouts_only() {
let oracle = r#"[
{"oracle_id":"oid-battle","name":"Invasion of Zendikar","layout":"battle","cmc":4.0,
"type_line":"Battle — Siege","color_identity":["G"],"keywords":[]},
{"oracle_id":"oid-plane","name":"Academy at Tolaria West","layout":"planar","cmc":0.0,
"type_line":"Plane — Dominaria","color_identity":[],"keywords":[]},
{"oracle_id":"oid-scheme","name":"A Display of My Dark Power","layout":"scheme","cmc":0.0,
"type_line":"Scheme","color_identity":[],"keywords":[]},
{"oracle_id":"oid-normal","name":"Grizzly Bears","layout":"normal","cmc":2.0,
"type_line":"Creature — Bear","color_identity":["G"],"keywords":[]}
]"#;
let dir = tempfile::tempdir().unwrap();
let dest = dir.path().join("reference.db");
let conn = open_reference_for_build(&dest).unwrap();
load_oracle_cards(&conn, oracle.as_bytes(), &CancelToken::new()).unwrap();
let horizontal = |id: &str| -> bool {
conn.query_row(
"SELECT horizontal FROM oracle_cards WHERE oracle_id=?1",
params![id],
|r| r.get::<_, i64>(0),
)
.unwrap()
!= 0
};
assert!(horizontal("oid-battle"));
assert!(horizontal("oid-plane"));
// Schemes are oversized but not rotated — only battle/planar layouts are.
assert!(!horizontal("oid-scheme"));
assert!(!horizontal("oid-normal"));
}
#[test]
fn captures_back_face_image_for_double_faced_cards() {
// Transform/MDFC cards: two `card_faces`, each with its own image.
let dfc = r#"[
{"id":"p-dfc","oracle_id":"oid-dfc","set":"znr","collector_number":"1",
"rarity":"rare",
"card_faces":[
{"image_uris":{"normal":"http://img/front"}},
{"image_uris":{"normal":"http://img/back"}}
]}
]"#;
// Split card: two `card_faces` for oracle text, but no per-face images —
// the whole card is one combined top-level image.
let split = r#"[
{"id":"p-split","oracle_id":"oid-split","set":"grn","collector_number":"2",
"rarity":"common","image_uris":{"normal":"http://img/split-combined"},
"card_faces":[{"image_uris":null},{"image_uris":null}]}
]"#;
let dir = tempfile::tempdir().unwrap();
let dest = dir.path().join("reference.db");
let conn = open_reference_for_build(&dest).unwrap();
load_printings(&conn, dfc.as_bytes(), &CancelToken::new()).unwrap();
load_printings(&conn, split.as_bytes(), &CancelToken::new()).unwrap();
let (front, back): (String, String) = conn
.query_row(
"SELECT image_uri, back_image_uri FROM printings WHERE scryfall_id='p-dfc'",
[],
|r| Ok((r.get(0)?, r.get(1)?)),
)
.unwrap();
assert_eq!(front, "http://img/front");
assert_eq!(back, "http://img/back");
let (front, back): (String, String) = conn
.query_row(
"SELECT image_uri, back_image_uri FROM printings WHERE scryfall_id='p-split'",
[],
|r| Ok((r.get(0)?, r.get(1)?)),
)
.unwrap();
assert_eq!(front, "http://img/split-combined");
assert_eq!(back, "", "split cards have no distinct back face");
}
#[test]
fn captures_usd_price_defaulting_to_zero_when_absent() {
let printings = r#"[
{"id":"p-priced","oracle_id":"oid-priced","set":"m19","collector_number":"1",
"rarity":"common","prices":{"usd":"12.34","usd_foil":"20.00"}},
{"id":"p-null","oracle_id":"oid-null","set":"m19","collector_number":"2",
"rarity":"common","prices":{"usd":null}},
{"id":"p-noprices","oracle_id":"oid-noprices","set":"m19","collector_number":"3",
"rarity":"common"}
]"#;
let dir = tempfile::tempdir().unwrap();
let dest = dir.path().join("reference.db");
let conn = open_reference_for_build(&dest).unwrap();
load_printings(&conn, printings.as_bytes(), &CancelToken::new()).unwrap();
let fetch = |id: &str| -> f64 {
conn.query_row(
"SELECT price_usd FROM printings WHERE scryfall_id=?1",
params![id],
|r| r.get(0),
)
.unwrap()
};
assert_eq!(fetch("p-priced"), 12.34);
assert_eq!(fetch("p-null"), 0.0);
assert_eq!(fetch("p-noprices"), 0.0);
}
}
Reference in New Issue View Git Blame Copy Permalink