garrettmills/mtgc
1
0
Fork 0
Code Issues 1 Pull Requests Releases Activity

Overhaul card-data ingest: filter art cards, segmented cancellable progress

Browse Source 
- Exclude Scryfall Art Series cards (layout "art_series") from ingest —
  they aren't playable cards and were polluting search results.
- Replace the coarse per-stage progress text with a byte-accurate,
  weighted multi-segment progress bar (Index / Card data / Printings /
  Indexing) that fills as one continuous bar instead of swapping per
  stage; segment weights are computed from the real download sizes
  reported by Scryfall's bulk-data index (the download responses
  themselves are chunked with no Content-Length to rely on).
- Add a Cancel button, wired through a new CancelToken checked once per
  parsed record so cancellation lands almost instantly.
- Fix a related correctness bug this surfaced: ingest used to delete the
  existing reference.db before writing its replacement, so a cancelled
  or crashed update left the app with no card database at all. It now
  builds into a temp file and only atomically replaces reference.db on
  full success.
- Split the topbar's single "Choose library…" action into separate
  "Open Library…" (existing file) and "New Library…" (save-dialog for a
  new file) buttons.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Garrett Mills
2026-07-07 20:22:31 -05:00
parent 0eaa41ec41
commit 342b8da797
9 changed files with 623 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
crates/mtgc-core/src/bin/ingest.rs
View File

@@ -5,6 +5,8 @@
use std::path::PathBuf;
use std::process::ExitCode;
use mtgc_core::ingest::CancelToken;
fn main() -> ExitCode {
let dest = std::env::args()
.nth(1)
@@ -12,7 +14,10 @@ fn main() -> ExitCode {
.unwrap_or_else(|| PathBuf::from("reference.db"));
eprintln!("Building reference db at {}", dest.display());
match mtgc_core::ingest::build_reference_db(&dest, |msg| eprintln!(" {msg}")) {
let cancel = CancelToken::new();
match mtgc_core::ingest::build_reference_db(&dest, &cancel, |p| {
eprintln!(" [{}] {:.1}% {}", p.stage, p.overall_percent, p.message)
}) {
Ok(stats) => {
eprintln!(
"Done: {} oracle cards, {} printings.",

3
crates/mtgc-core/src/error.rs
View File

@@ -16,6 +16,9 @@ pub enum CoreError {
#[error("json error: {0}")]
Json(#[from] serde_json::Error),
#[error("cancelled")]
Cancelled,
#[error("{0}")]
Other(String),
}

388
crates/mtgc-core/src/ingest.rs
View File

@@ -10,6 +10,8 @@
use std::io::{BufReader, Read};
use std::marker::PhantomData;
use std::path::Path;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;
use rusqlite::{params, Connection};
use serde::de::{DeserializeOwned, SeqAccess, Visitor};
@@ -22,6 +24,11 @@ use crate::model::ColorSet;
const BULK_INDEX_URL: &str = "https://api.scryfall.com/bulk-data";
const USER_AGENT: &str = "mtgc/0.1 (personal library catalog)";
const STAGE_INDEX: &str = "index";
const STAGE_ORACLE: &str = "oracle";
const STAGE_PRINTINGS: &str = "printings";
const STAGE_OPTIMIZE: &str = "optimize";
/// Row counts from a build, for reporting/verification.
#[derive(Debug, Default, Clone, Copy)]
pub struct IngestStats {
@@ -29,27 +36,204 @@ pub struct IngestStats {
pub printings: usize,
}
/// Download the latest Scryfall bulk data and (re)build `reference.db` at `dest`.
///
/// `progress` is called with human-readable status lines.
pub fn build_reference_db(dest: &Path, mut progress: impl FnMut(&str)) -> Result<IngestStats> {
let (oracle_uri, default_uri) = fetch_bulk_uris(&mut progress)?;
/// A flag a caller can flip from another thread to abort an in-progress
/// ingest at the next checkpoint (checked once per parsed record — thousands
/// of times a second, so cancellation is near-instant).
#[derive(Clone, Default)]
pub struct CancelToken(Arc<AtomicBool>);
if dest.exists() {
std::fs::remove_file(dest)?;
impl CancelToken {
pub fn new() -> Self {
Self(Arc::new(AtomicBool::new(false)))
}
let conn = open_reference_for_build(dest)?;
pub fn cancel(&self) {
self.0.store(true, Ordering::Relaxed);
}
pub fn is_cancelled(&self) -> bool {
self.0.load(Ordering::Relaxed)
}
}
/// A cancellation-triggered early exit from inside `stream_json_array`'s
/// per-item closure round-trips through `serde_json::Error::custom`, which
/// stringifies (and appends position info to) whatever error it's given. Undo
/// that mangling here so callers can match on a clean [`CoreError::Cancelled`].
fn override_if_cancelled(e: CoreError, cancel: &CancelToken) -> CoreError {
if cancel.is_cancelled() {
CoreError::Cancelled
} else {
e
}
}
/// One segment of the overall ingest, with its share of the total work
/// (weights across all stages sum to 1.0). Lets the UI render a single
/// multi-segment progress bar instead of swapping bars per stage.
#[derive(Debug, Clone, serde::Serialize)]
pub struct IngestStageInfo {
pub id: &'static str,
pub label: &'static str,
pub weight: f64,
}
/// A single progress update from a running ingest. `stage` is the id of the
/// currently active [`IngestStageInfo`]; `message` is a human-readable detail
/// for that stage; `overall_percent` is cumulative progress (0.0..=100.0)
/// across *all* stages, ready to drive a single combined bar; `stages` is the
/// (static, for a given run) segment layout for rendering.
#[derive(Debug, Clone, serde::Serialize)]
pub struct IngestProgress {
pub stage: &'static str,
pub message: String,
pub overall_percent: f64,
pub stages: Vec<IngestStageInfo>,
}
/// Fixed weight for the two bookkeeping stages (fetching the bulk index,
/// optimizing the FTS index); the remaining budget is split between the two
/// downloads by their known byte size, since both are network-bound.
const INDEX_WEIGHT: f64 = 0.02;
const OPTIMIZE_WEIGHT: f64 = 0.03;
fn stage_plan(oracle_size: u64, default_size: u64) -> Vec<IngestStageInfo> {
let dl_total = (oracle_size + default_size).max(1) as f64;
let dl_budget = 1.0 - INDEX_WEIGHT - OPTIMIZE_WEIGHT;
let oracle_weight = dl_budget * (oracle_size as f64 / dl_total);
let printings_weight = dl_budget - oracle_weight;
vec![
IngestStageInfo {
id: STAGE_INDEX,
label: "Index",
weight: INDEX_WEIGHT,
},
IngestStageInfo {
id: STAGE_ORACLE,
label: "Card data",
weight: oracle_weight,
},
IngestStageInfo {
id: STAGE_PRINTINGS,
label: "Printings",
weight: printings_weight,
},
IngestStageInfo {
id: STAGE_OPTIMIZE,
label: "Indexing",
weight: OPTIMIZE_WEIGHT,
},
]
}
/// Cumulative percent through `stages` once `active` has completed
/// `local_fraction` (0.0..=1.0) of its own work.
fn overall_percent(stages: &[IngestStageInfo], active: &str, local_fraction: f64) -> f64 {
let mut acc = 0.0;
for s in stages {
if s.id == active {
return ((acc + s.weight * local_fraction) * 100.0).min(100.0);
}
acc += s.weight;
}
(acc * 100.0).min(100.0)
}
/// Download the latest Scryfall bulk data and (re)build `reference.db` at
/// `dest`. Builds into a temp file alongside `dest` and only replaces it on
/// full success, so a cancelled or failed update never destroys a working
/// reference db.
pub fn build_reference_db(
dest: &Path,
cancel: &CancelToken,
mut progress: impl FnMut(IngestProgress),
) -> Result<IngestStats> {
let tmp = dest.with_extension("db.tmp");
let result = build_into(&tmp, cancel, &mut progress);
match &result {
Ok(_) => {
let _ = std::fs::remove_file(dest);
std::fs::rename(&tmp, dest)?;
}
Err(_) => {
let _ = std::fs::remove_file(&tmp);
}
}
result
}
fn build_into(
tmp: &Path,
cancel: &CancelToken,
progress: &mut impl FnMut(IngestProgress),
) -> Result<IngestStats> {
// Sizes aren't known until we've hit the bulk index, so the very first
// event uses a placeholder 50/50 split; the real plan follows within the
// same (fast) request/response round trip.
let mut stages = stage_plan(1, 1);
progress(IngestProgress {
stage: STAGE_INDEX,
message: "Fetching Scryfall bulk index…".into(),
overall_percent: 0.0,
stages: stages.clone(),
});
if cancel.is_cancelled() {
return Err(CoreError::Cancelled);
}
let bulk = fetch_bulk_uris()?;
stages = stage_plan(bulk.oracle_size, bulk.default_size);
if tmp.exists() {
std::fs::remove_file(tmp)?;
}
let conn = open_reference_for_build(tmp)?;
// Bulk-load fast; this db is disposable/rebuildable so durability is moot.
conn.pragma_update(None, "journal_mode", "MEMORY")?;
conn.pragma_update(None, "synchronous", "OFF")?;
progress("Loading oracle cards…");
let oracle_cards = load_oracle_cards(&conn, http_reader(&oracle_uri)?)?;
progress(IngestProgress {
stage: STAGE_ORACLE,
message: "Downloading card data…".into(),
overall_percent: overall_percent(&stages, STAGE_ORACLE, 0.0),
stages: stages.clone(),
});
let reader = http_reader(&bulk.oracle_uri)?;
let reader = progress_reader(
reader,
bulk.oracle_size,
STAGE_ORACLE,
"Downloading card data",
&stages,
progress,
);
let oracle_cards = load_oracle_cards(&conn, reader, cancel).map_err(|e| override_if_cancelled(e, cancel))?;
progress("Loading printings…");
let printings = load_printings(&conn, http_reader(&default_uri)?)?;
progress(IngestProgress {
stage: STAGE_PRINTINGS,
message: "Downloading printings…".into(),
overall_percent: overall_percent(&stages, STAGE_PRINTINGS, 0.0),
stages: stages.clone(),
});
let reader = http_reader(&bulk.default_uri)?;
let reader = progress_reader(
reader,
bulk.default_size,
STAGE_PRINTINGS,
"Downloading printings",
&stages,
progress,
);
let printings = load_printings(&conn, reader, cancel).map_err(|e| override_if_cancelled(e, cancel))?;
progress("Optimizing…");
if cancel.is_cancelled() {
return Err(CoreError::Cancelled);
}
progress(IngestProgress {
stage: STAGE_OPTIMIZE,
message: "Optimizing search index…".into(),
overall_percent: overall_percent(&stages, STAGE_OPTIMIZE, 0.5),
stages: stages.clone(),
});
conn.execute_batch("INSERT INTO oracle_names_fts(oracle_names_fts) VALUES('optimize');")?;
Ok(IngestStats {
@@ -58,6 +242,70 @@ pub fn build_reference_db(dest: &Path, mut progress: impl FnMut(&str)) -> Result
})
}
/// Human-readable byte size, e.g. `42.3 MB`.
fn human_mb(bytes: u64) -> String {
format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0))
}
/// Wrap a reader so that, as it's consumed (concurrently with JSON parsing),
/// it periodically reports download progress against the overall multi-stage
/// bar. Throttled to avoid flooding the UI with an event per network read.
fn progress_reader<'a, R: Read>(
inner: R,
total: u64,
stage: &'static str,
label: &'static str,
stages: &'a [IngestStageInfo],
progress: &'a mut impl FnMut(IngestProgress),
) -> impl Read + 'a
where
R: 'a,
{
const EMIT_EVERY: u64 = 512 * 1024; // 512 KB
ProgressReader {
inner,
read: 0,
last_emit: 0,
total,
on_progress: move |read: u64, total: u64| {
let message = format!("{label}… {} / {}", human_mb(read), human_mb(total));
let fraction = if total > 0 {
(read as f64 / total as f64).min(1.0)
} else {
1.0
};
progress(IngestProgress {
stage,
message,
overall_percent: overall_percent(stages, stage, fraction),
stages: stages.to_vec(),
});
},
threshold: EMIT_EVERY,
}
}
struct ProgressReader<R, F> {
inner: R,
read: u64,
last_emit: u64,
total: u64,
threshold: u64,
on_progress: F,
}
impl<R: Read, F: FnMut(u64, u64)> Read for ProgressReader<R, F> {
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
let n = self.inner.read(buf)?;
self.read += n as u64;
if n == 0 || self.read - self.last_emit >= self.threshold {
self.last_emit = self.read;
(self.on_progress)(self.read, self.total);
}
Ok(n)
}
}
// ---- Scryfall bulk index -------------------------------------------------
#[derive(Deserialize)]
@@ -70,10 +318,20 @@ struct BulkEntry {
#[serde(rename = "type")]
kind: String,
download_uri: String,
/// Uncompressed byte size of `download_uri`'s contents, per the Scryfall
/// bulk-data API — used for download progress since the actual download
/// response is chunked (no `Content-Length` header to rely on).
size: u64,
}
fn fetch_bulk_uris(progress: &mut impl FnMut(&str)) -> Result<(String, String)> {
progress("Fetching Scryfall bulk index…");
struct BulkUris {
oracle_uri: String,
oracle_size: u64,
default_uri: String,
default_size: u64,
}
fn fetch_bulk_uris() -> Result<BulkUris> {
let index: BulkIndex = ureq::get(BULK_INDEX_URL)
.set("User-Agent", USER_AGENT)
.set("Accept", "application/json")
@@ -85,16 +343,20 @@ fn fetch_bulk_uris(progress: &mut impl FnMut(&str)) -> Result<(String, String)>
.data
.iter()
.find(|e| e.kind == kind)
.map(|e| e.download_uri.clone())
.ok_or_else(|| CoreError::Other(format!("Scryfall bulk index missing '{kind}'")))
};
Ok((find("oracle_cards")?, find("default_cards")?))
let oracle = find("oracle_cards")?;
let default = find("default_cards")?;
Ok(BulkUris {
oracle_uri: oracle.download_uri.clone(),
oracle_size: oracle.size,
default_uri: default.download_uri.clone(),
default_size: default.size,
})
}
fn http_reader(url: &str) -> Result<impl Read> {
let resp = ureq::get(url)
.set("User-Agent", USER_AGENT)
.call()?;
let resp = ureq::get(url).set("User-Agent", USER_AGENT).call()?;
Ok(BufReader::with_capacity(1 << 20, resp.into_reader()))
}
@@ -153,6 +415,8 @@ struct ScryOracle {
oracle_id: Option<String>,
name: String,
#[serde(default)]
layout: String,
#[serde(default)]
cmc: f64,
#[serde(default)]
colors: Option<Vec<String>>,
@@ -176,7 +440,7 @@ struct ScryFace {
colors: Option<Vec<String>>,
}
fn load_oracle_cards(conn: &Connection, reader: impl Read) -> Result<usize> {
fn load_oracle_cards(conn: &Connection, reader: impl Read, cancel: &CancelToken) -> Result<usize> {
conn.execute_batch("BEGIN")?;
let mut count = 0usize;
{
@@ -195,9 +459,17 @@ fn load_oracle_cards(conn: &Connection, reader: impl Read) -> Result<usize> {
conn.prepare("INSERT INTO card_keywords(oracle_id,keyword) VALUES (?1,?2)")?;
stream_json_array::<_, ScryOracle, _>(reader, |c| {
if cancel.is_cancelled() {
return Err(CoreError::Cancelled);
}
let Some(oracle_id) = c.oracle_id.clone() else {
return Ok(()); // rare face-only records without a top-level oracle_id
};
// Art Series cards (Secret Lair) aren't real playable cards — exclude
// them from search/library entirely.
if c.layout == "art_series" {
return Ok(());
}
// Colors: prefer top-level, else union of faces.
let colors = match &c.colors {
@@ -321,7 +593,7 @@ struct PrintFace {
image_uris: Option<ImageUris>,
}
fn load_printings(conn: &Connection, reader: impl Read) -> Result<usize> {
fn load_printings(conn: &Connection, reader: impl Read, cancel: &CancelToken) -> Result<usize> {
conn.execute_batch("BEGIN")?;
let mut count = 0usize;
{
@@ -331,6 +603,9 @@ fn load_printings(conn: &Connection, reader: impl Read) -> Result<usize> {
VALUES (?1,?2,?3,?4,?5,?6,?7)",
)?;
stream_json_array::<_, ScryPrinting, _>(reader, |p| {
if cancel.is_cancelled() {
return Err(CoreError::Cancelled);
}
let Some(oracle_id) = p.oracle_id.clone() else {
return Ok(());
};
@@ -399,6 +674,71 @@ mod tests {
assert_eq!(sum, 6);
}
#[test]
fn excludes_art_series_cards() {
let oracle = r#"[
{"oracle_id":"oid-1","name":"Llanowar Elves","cmc":1.0,
"colors":["G"],"color_identity":["G"],
"type_line":"Creature — Elf Druid","keywords":[]},
{"oracle_id":"oid-2","name":"Llanowar Elves","layout":"art_series",
"type_line":"Card","keywords":[]}
]"#;
let dir = tempfile::tempdir().unwrap();
let dest = dir.path().join("reference.db");
let conn = open_reference_for_build(&dest).unwrap();
let oc = load_oracle_cards(&conn, oracle.as_bytes(), &CancelToken::new()).unwrap();
assert_eq!(oc, 1);
let count: i64 = conn
.query_row(
"SELECT count(*) FROM oracle_cards WHERE oracle_id='oid-2'",
[],
|r| r.get(0),
)
.unwrap();
assert_eq!(count, 0);
}
#[test]
fn cancel_token_aborts_load_with_cancelled_error() {
let oracle = r#"[
{"oracle_id":"oid-1","name":"Llanowar Elves","cmc":1.0,
"colors":["G"],"color_identity":["G"],
"type_line":"Creature — Elf Druid","keywords":[]},
{"oracle_id":"oid-2","name":"Serra Angel","cmc":5.0,
"colors":["W"],"color_identity":["W"],
"type_line":"Creature — Angel","keywords":["Flying","Vigilance"]}
]"#;
let dir = tempfile::tempdir().unwrap();
let dest = dir.path().join("reference.db");
let conn = open_reference_for_build(&dest).unwrap();
let cancel = CancelToken::new();
cancel.cancel();
let err = load_oracle_cards(&conn, oracle.as_bytes(), &cancel)
.map_err(|e| override_if_cancelled(e, &cancel))
.unwrap_err();
assert!(matches!(err, CoreError::Cancelled));
}
#[test]
fn stage_plan_weights_sum_to_one() {
let stages = stage_plan(171_000_000, 530_000_000);
let total: f64 = stages.iter().map(|s| s.weight).sum();
assert!((total - 1.0).abs() < 1e-9);
// The larger download gets the larger share of the two DL stages.
let oracle_w = stages.iter().find(|s| s.id == STAGE_ORACLE).unwrap().weight;
let printings_w = stages
.iter()
.find(|s| s.id == STAGE_PRINTINGS)
.unwrap()
.weight;
assert!(printings_w > oracle_w);
}
#[test]
fn oracle_and_printing_load_from_fixture_json() {
// Minimal Scryfall-shaped fixtures exercising the insert paths.
@@ -420,8 +760,8 @@ mod tests {
let dest = dir.path().join("reference.db");
let conn = open_reference_for_build(&dest).unwrap();
let oc = load_oracle_cards(&conn, oracle.as_bytes()).unwrap();
let pc = load_printings(&conn, printings.as_bytes()).unwrap();
let oc = load_oracle_cards(&conn, oracle.as_bytes(), &CancelToken::new()).unwrap();
let pc = load_printings(&conn, printings.as_bytes(), &CancelToken::new()).unwrap();
assert_eq!(oc, 2);
assert_eq!(pc, 1);