Overhaul card-data ingest: filter art cards, segmented cancellable progress
- Exclude Scryfall Art Series cards (layout "art_series") from ingest — they aren't playable cards and were polluting search results. - Replace the coarse per-stage progress text with a byte-accurate, weighted multi-segment progress bar (Index / Card data / Printings / Indexing) that fills as one continuous bar instead of swapping per stage; segment weights are computed from the real download sizes reported by Scryfall's bulk-data index (the download responses themselves are chunked with no Content-Length to rely on). - Add a Cancel button, wired through a new CancelToken checked once per parsed record so cancellation lands almost instantly. - Fix a related correctness bug this surfaced: ingest used to delete the existing reference.db before writing its replacement, so a cancelled or crashed update left the app with no card database at all. It now builds into a temp file and only atomically replaces reference.db on full success. - Split the topbar's single "Choose library…" action into separate "Open Library…" (existing file) and "New Library…" (save-dialog for a new file) buttons. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -5,6 +5,8 @@
|
||||
use std::path::PathBuf;
|
||||
use std::process::ExitCode;
|
||||
|
||||
use mtgc_core::ingest::CancelToken;
|
||||
|
||||
fn main() -> ExitCode {
|
||||
let dest = std::env::args()
|
||||
.nth(1)
|
||||
@@ -12,7 +14,10 @@ fn main() -> ExitCode {
|
||||
.unwrap_or_else(|| PathBuf::from("reference.db"));
|
||||
|
||||
eprintln!("Building reference db at {}", dest.display());
|
||||
match mtgc_core::ingest::build_reference_db(&dest, |msg| eprintln!(" {msg}")) {
|
||||
let cancel = CancelToken::new();
|
||||
match mtgc_core::ingest::build_reference_db(&dest, &cancel, |p| {
|
||||
eprintln!(" [{}] {:.1}% {}", p.stage, p.overall_percent, p.message)
|
||||
}) {
|
||||
Ok(stats) => {
|
||||
eprintln!(
|
||||
"Done: {} oracle cards, {} printings.",
|
||||
|
||||
@@ -16,6 +16,9 @@ pub enum CoreError {
|
||||
#[error("json error: {0}")]
|
||||
Json(#[from] serde_json::Error),
|
||||
|
||||
#[error("cancelled")]
|
||||
Cancelled,
|
||||
|
||||
#[error("{0}")]
|
||||
Other(String),
|
||||
}
|
||||
|
||||
@@ -10,6 +10,8 @@
|
||||
use std::io::{BufReader, Read};
|
||||
use std::marker::PhantomData;
|
||||
use std::path::Path;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::Arc;
|
||||
|
||||
use rusqlite::{params, Connection};
|
||||
use serde::de::{DeserializeOwned, SeqAccess, Visitor};
|
||||
@@ -22,6 +24,11 @@ use crate::model::ColorSet;
|
||||
const BULK_INDEX_URL: &str = "https://api.scryfall.com/bulk-data";
|
||||
const USER_AGENT: &str = "mtgc/0.1 (personal library catalog)";
|
||||
|
||||
const STAGE_INDEX: &str = "index";
|
||||
const STAGE_ORACLE: &str = "oracle";
|
||||
const STAGE_PRINTINGS: &str = "printings";
|
||||
const STAGE_OPTIMIZE: &str = "optimize";
|
||||
|
||||
/// Row counts from a build, for reporting/verification.
|
||||
#[derive(Debug, Default, Clone, Copy)]
|
||||
pub struct IngestStats {
|
||||
@@ -29,27 +36,204 @@ pub struct IngestStats {
|
||||
pub printings: usize,
|
||||
}
|
||||
|
||||
/// Download the latest Scryfall bulk data and (re)build `reference.db` at `dest`.
|
||||
///
|
||||
/// `progress` is called with human-readable status lines.
|
||||
pub fn build_reference_db(dest: &Path, mut progress: impl FnMut(&str)) -> Result<IngestStats> {
|
||||
let (oracle_uri, default_uri) = fetch_bulk_uris(&mut progress)?;
|
||||
/// A flag a caller can flip from another thread to abort an in-progress
|
||||
/// ingest at the next checkpoint (checked once per parsed record — thousands
|
||||
/// of times a second, so cancellation is near-instant).
|
||||
#[derive(Clone, Default)]
|
||||
pub struct CancelToken(Arc<AtomicBool>);
|
||||
|
||||
if dest.exists() {
|
||||
std::fs::remove_file(dest)?;
|
||||
impl CancelToken {
|
||||
pub fn new() -> Self {
|
||||
Self(Arc::new(AtomicBool::new(false)))
|
||||
}
|
||||
let conn = open_reference_for_build(dest)?;
|
||||
|
||||
pub fn cancel(&self) {
|
||||
self.0.store(true, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn is_cancelled(&self) -> bool {
|
||||
self.0.load(Ordering::Relaxed)
|
||||
}
|
||||
}
|
||||
|
||||
/// A cancellation-triggered early exit from inside `stream_json_array`'s
|
||||
/// per-item closure round-trips through `serde_json::Error::custom`, which
|
||||
/// stringifies (and appends position info to) whatever error it's given. Undo
|
||||
/// that mangling here so callers can match on a clean [`CoreError::Cancelled`].
|
||||
fn override_if_cancelled(e: CoreError, cancel: &CancelToken) -> CoreError {
|
||||
if cancel.is_cancelled() {
|
||||
CoreError::Cancelled
|
||||
} else {
|
||||
e
|
||||
}
|
||||
}
|
||||
|
||||
/// One segment of the overall ingest, with its share of the total work
|
||||
/// (weights across all stages sum to 1.0). Lets the UI render a single
|
||||
/// multi-segment progress bar instead of swapping bars per stage.
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
pub struct IngestStageInfo {
|
||||
pub id: &'static str,
|
||||
pub label: &'static str,
|
||||
pub weight: f64,
|
||||
}
|
||||
|
||||
/// A single progress update from a running ingest. `stage` is the id of the
|
||||
/// currently active [`IngestStageInfo`]; `message` is a human-readable detail
|
||||
/// for that stage; `overall_percent` is cumulative progress (0.0..=100.0)
|
||||
/// across *all* stages, ready to drive a single combined bar; `stages` is the
|
||||
/// (static, for a given run) segment layout for rendering.
|
||||
#[derive(Debug, Clone, serde::Serialize)]
|
||||
pub struct IngestProgress {
|
||||
pub stage: &'static str,
|
||||
pub message: String,
|
||||
pub overall_percent: f64,
|
||||
pub stages: Vec<IngestStageInfo>,
|
||||
}
|
||||
|
||||
/// Fixed weight for the two bookkeeping stages (fetching the bulk index,
|
||||
/// optimizing the FTS index); the remaining budget is split between the two
|
||||
/// downloads by their known byte size, since both are network-bound.
|
||||
const INDEX_WEIGHT: f64 = 0.02;
|
||||
const OPTIMIZE_WEIGHT: f64 = 0.03;
|
||||
|
||||
fn stage_plan(oracle_size: u64, default_size: u64) -> Vec<IngestStageInfo> {
|
||||
let dl_total = (oracle_size + default_size).max(1) as f64;
|
||||
let dl_budget = 1.0 - INDEX_WEIGHT - OPTIMIZE_WEIGHT;
|
||||
let oracle_weight = dl_budget * (oracle_size as f64 / dl_total);
|
||||
let printings_weight = dl_budget - oracle_weight;
|
||||
vec![
|
||||
IngestStageInfo {
|
||||
id: STAGE_INDEX,
|
||||
label: "Index",
|
||||
weight: INDEX_WEIGHT,
|
||||
},
|
||||
IngestStageInfo {
|
||||
id: STAGE_ORACLE,
|
||||
label: "Card data",
|
||||
weight: oracle_weight,
|
||||
},
|
||||
IngestStageInfo {
|
||||
id: STAGE_PRINTINGS,
|
||||
label: "Printings",
|
||||
weight: printings_weight,
|
||||
},
|
||||
IngestStageInfo {
|
||||
id: STAGE_OPTIMIZE,
|
||||
label: "Indexing",
|
||||
weight: OPTIMIZE_WEIGHT,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
/// Cumulative percent through `stages` once `active` has completed
|
||||
/// `local_fraction` (0.0..=1.0) of its own work.
|
||||
fn overall_percent(stages: &[IngestStageInfo], active: &str, local_fraction: f64) -> f64 {
|
||||
let mut acc = 0.0;
|
||||
for s in stages {
|
||||
if s.id == active {
|
||||
return ((acc + s.weight * local_fraction) * 100.0).min(100.0);
|
||||
}
|
||||
acc += s.weight;
|
||||
}
|
||||
(acc * 100.0).min(100.0)
|
||||
}
|
||||
|
||||
/// Download the latest Scryfall bulk data and (re)build `reference.db` at
|
||||
/// `dest`. Builds into a temp file alongside `dest` and only replaces it on
|
||||
/// full success, so a cancelled or failed update never destroys a working
|
||||
/// reference db.
|
||||
pub fn build_reference_db(
|
||||
dest: &Path,
|
||||
cancel: &CancelToken,
|
||||
mut progress: impl FnMut(IngestProgress),
|
||||
) -> Result<IngestStats> {
|
||||
let tmp = dest.with_extension("db.tmp");
|
||||
let result = build_into(&tmp, cancel, &mut progress);
|
||||
match &result {
|
||||
Ok(_) => {
|
||||
let _ = std::fs::remove_file(dest);
|
||||
std::fs::rename(&tmp, dest)?;
|
||||
}
|
||||
Err(_) => {
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn build_into(
|
||||
tmp: &Path,
|
||||
cancel: &CancelToken,
|
||||
progress: &mut impl FnMut(IngestProgress),
|
||||
) -> Result<IngestStats> {
|
||||
// Sizes aren't known until we've hit the bulk index, so the very first
|
||||
// event uses a placeholder 50/50 split; the real plan follows within the
|
||||
// same (fast) request/response round trip.
|
||||
let mut stages = stage_plan(1, 1);
|
||||
progress(IngestProgress {
|
||||
stage: STAGE_INDEX,
|
||||
message: "Fetching Scryfall bulk index…".into(),
|
||||
overall_percent: 0.0,
|
||||
stages: stages.clone(),
|
||||
});
|
||||
if cancel.is_cancelled() {
|
||||
return Err(CoreError::Cancelled);
|
||||
}
|
||||
let bulk = fetch_bulk_uris()?;
|
||||
stages = stage_plan(bulk.oracle_size, bulk.default_size);
|
||||
|
||||
if tmp.exists() {
|
||||
std::fs::remove_file(tmp)?;
|
||||
}
|
||||
let conn = open_reference_for_build(tmp)?;
|
||||
// Bulk-load fast; this db is disposable/rebuildable so durability is moot.
|
||||
conn.pragma_update(None, "journal_mode", "MEMORY")?;
|
||||
conn.pragma_update(None, "synchronous", "OFF")?;
|
||||
|
||||
progress("Loading oracle cards…");
|
||||
let oracle_cards = load_oracle_cards(&conn, http_reader(&oracle_uri)?)?;
|
||||
progress(IngestProgress {
|
||||
stage: STAGE_ORACLE,
|
||||
message: "Downloading card data…".into(),
|
||||
overall_percent: overall_percent(&stages, STAGE_ORACLE, 0.0),
|
||||
stages: stages.clone(),
|
||||
});
|
||||
let reader = http_reader(&bulk.oracle_uri)?;
|
||||
let reader = progress_reader(
|
||||
reader,
|
||||
bulk.oracle_size,
|
||||
STAGE_ORACLE,
|
||||
"Downloading card data",
|
||||
&stages,
|
||||
progress,
|
||||
);
|
||||
let oracle_cards = load_oracle_cards(&conn, reader, cancel).map_err(|e| override_if_cancelled(e, cancel))?;
|
||||
|
||||
progress("Loading printings…");
|
||||
let printings = load_printings(&conn, http_reader(&default_uri)?)?;
|
||||
progress(IngestProgress {
|
||||
stage: STAGE_PRINTINGS,
|
||||
message: "Downloading printings…".into(),
|
||||
overall_percent: overall_percent(&stages, STAGE_PRINTINGS, 0.0),
|
||||
stages: stages.clone(),
|
||||
});
|
||||
let reader = http_reader(&bulk.default_uri)?;
|
||||
let reader = progress_reader(
|
||||
reader,
|
||||
bulk.default_size,
|
||||
STAGE_PRINTINGS,
|
||||
"Downloading printings",
|
||||
&stages,
|
||||
progress,
|
||||
);
|
||||
let printings = load_printings(&conn, reader, cancel).map_err(|e| override_if_cancelled(e, cancel))?;
|
||||
|
||||
progress("Optimizing…");
|
||||
if cancel.is_cancelled() {
|
||||
return Err(CoreError::Cancelled);
|
||||
}
|
||||
progress(IngestProgress {
|
||||
stage: STAGE_OPTIMIZE,
|
||||
message: "Optimizing search index…".into(),
|
||||
overall_percent: overall_percent(&stages, STAGE_OPTIMIZE, 0.5),
|
||||
stages: stages.clone(),
|
||||
});
|
||||
conn.execute_batch("INSERT INTO oracle_names_fts(oracle_names_fts) VALUES('optimize');")?;
|
||||
|
||||
Ok(IngestStats {
|
||||
@@ -58,6 +242,70 @@ pub fn build_reference_db(dest: &Path, mut progress: impl FnMut(&str)) -> Result
|
||||
})
|
||||
}
|
||||
|
||||
/// Human-readable byte size, e.g. `42.3 MB`.
|
||||
fn human_mb(bytes: u64) -> String {
|
||||
format!("{:.1} MB", bytes as f64 / (1024.0 * 1024.0))
|
||||
}
|
||||
|
||||
/// Wrap a reader so that, as it's consumed (concurrently with JSON parsing),
|
||||
/// it periodically reports download progress against the overall multi-stage
|
||||
/// bar. Throttled to avoid flooding the UI with an event per network read.
|
||||
fn progress_reader<'a, R: Read>(
|
||||
inner: R,
|
||||
total: u64,
|
||||
stage: &'static str,
|
||||
label: &'static str,
|
||||
stages: &'a [IngestStageInfo],
|
||||
progress: &'a mut impl FnMut(IngestProgress),
|
||||
) -> impl Read + 'a
|
||||
where
|
||||
R: 'a,
|
||||
{
|
||||
const EMIT_EVERY: u64 = 512 * 1024; // 512 KB
|
||||
ProgressReader {
|
||||
inner,
|
||||
read: 0,
|
||||
last_emit: 0,
|
||||
total,
|
||||
on_progress: move |read: u64, total: u64| {
|
||||
let message = format!("{label}… {} / {}", human_mb(read), human_mb(total));
|
||||
let fraction = if total > 0 {
|
||||
(read as f64 / total as f64).min(1.0)
|
||||
} else {
|
||||
1.0
|
||||
};
|
||||
progress(IngestProgress {
|
||||
stage,
|
||||
message,
|
||||
overall_percent: overall_percent(stages, stage, fraction),
|
||||
stages: stages.to_vec(),
|
||||
});
|
||||
},
|
||||
threshold: EMIT_EVERY,
|
||||
}
|
||||
}
|
||||
|
||||
struct ProgressReader<R, F> {
|
||||
inner: R,
|
||||
read: u64,
|
||||
last_emit: u64,
|
||||
total: u64,
|
||||
threshold: u64,
|
||||
on_progress: F,
|
||||
}
|
||||
|
||||
impl<R: Read, F: FnMut(u64, u64)> Read for ProgressReader<R, F> {
|
||||
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
|
||||
let n = self.inner.read(buf)?;
|
||||
self.read += n as u64;
|
||||
if n == 0 || self.read - self.last_emit >= self.threshold {
|
||||
self.last_emit = self.read;
|
||||
(self.on_progress)(self.read, self.total);
|
||||
}
|
||||
Ok(n)
|
||||
}
|
||||
}
|
||||
|
||||
// ---- Scryfall bulk index -------------------------------------------------
|
||||
|
||||
#[derive(Deserialize)]
|
||||
@@ -70,10 +318,20 @@ struct BulkEntry {
|
||||
#[serde(rename = "type")]
|
||||
kind: String,
|
||||
download_uri: String,
|
||||
/// Uncompressed byte size of `download_uri`'s contents, per the Scryfall
|
||||
/// bulk-data API — used for download progress since the actual download
|
||||
/// response is chunked (no `Content-Length` header to rely on).
|
||||
size: u64,
|
||||
}
|
||||
|
||||
fn fetch_bulk_uris(progress: &mut impl FnMut(&str)) -> Result<(String, String)> {
|
||||
progress("Fetching Scryfall bulk index…");
|
||||
struct BulkUris {
|
||||
oracle_uri: String,
|
||||
oracle_size: u64,
|
||||
default_uri: String,
|
||||
default_size: u64,
|
||||
}
|
||||
|
||||
fn fetch_bulk_uris() -> Result<BulkUris> {
|
||||
let index: BulkIndex = ureq::get(BULK_INDEX_URL)
|
||||
.set("User-Agent", USER_AGENT)
|
||||
.set("Accept", "application/json")
|
||||
@@ -85,16 +343,20 @@ fn fetch_bulk_uris(progress: &mut impl FnMut(&str)) -> Result<(String, String)>
|
||||
.data
|
||||
.iter()
|
||||
.find(|e| e.kind == kind)
|
||||
.map(|e| e.download_uri.clone())
|
||||
.ok_or_else(|| CoreError::Other(format!("Scryfall bulk index missing '{kind}'")))
|
||||
};
|
||||
Ok((find("oracle_cards")?, find("default_cards")?))
|
||||
let oracle = find("oracle_cards")?;
|
||||
let default = find("default_cards")?;
|
||||
Ok(BulkUris {
|
||||
oracle_uri: oracle.download_uri.clone(),
|
||||
oracle_size: oracle.size,
|
||||
default_uri: default.download_uri.clone(),
|
||||
default_size: default.size,
|
||||
})
|
||||
}
|
||||
|
||||
fn http_reader(url: &str) -> Result<impl Read> {
|
||||
let resp = ureq::get(url)
|
||||
.set("User-Agent", USER_AGENT)
|
||||
.call()?;
|
||||
let resp = ureq::get(url).set("User-Agent", USER_AGENT).call()?;
|
||||
Ok(BufReader::with_capacity(1 << 20, resp.into_reader()))
|
||||
}
|
||||
|
||||
@@ -153,6 +415,8 @@ struct ScryOracle {
|
||||
oracle_id: Option<String>,
|
||||
name: String,
|
||||
#[serde(default)]
|
||||
layout: String,
|
||||
#[serde(default)]
|
||||
cmc: f64,
|
||||
#[serde(default)]
|
||||
colors: Option<Vec<String>>,
|
||||
@@ -176,7 +440,7 @@ struct ScryFace {
|
||||
colors: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
fn load_oracle_cards(conn: &Connection, reader: impl Read) -> Result<usize> {
|
||||
fn load_oracle_cards(conn: &Connection, reader: impl Read, cancel: &CancelToken) -> Result<usize> {
|
||||
conn.execute_batch("BEGIN")?;
|
||||
let mut count = 0usize;
|
||||
{
|
||||
@@ -195,9 +459,17 @@ fn load_oracle_cards(conn: &Connection, reader: impl Read) -> Result<usize> {
|
||||
conn.prepare("INSERT INTO card_keywords(oracle_id,keyword) VALUES (?1,?2)")?;
|
||||
|
||||
stream_json_array::<_, ScryOracle, _>(reader, |c| {
|
||||
if cancel.is_cancelled() {
|
||||
return Err(CoreError::Cancelled);
|
||||
}
|
||||
let Some(oracle_id) = c.oracle_id.clone() else {
|
||||
return Ok(()); // rare face-only records without a top-level oracle_id
|
||||
};
|
||||
// Art Series cards (Secret Lair) aren't real playable cards — exclude
|
||||
// them from search/library entirely.
|
||||
if c.layout == "art_series" {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Colors: prefer top-level, else union of faces.
|
||||
let colors = match &c.colors {
|
||||
@@ -321,7 +593,7 @@ struct PrintFace {
|
||||
image_uris: Option<ImageUris>,
|
||||
}
|
||||
|
||||
fn load_printings(conn: &Connection, reader: impl Read) -> Result<usize> {
|
||||
fn load_printings(conn: &Connection, reader: impl Read, cancel: &CancelToken) -> Result<usize> {
|
||||
conn.execute_batch("BEGIN")?;
|
||||
let mut count = 0usize;
|
||||
{
|
||||
@@ -331,6 +603,9 @@ fn load_printings(conn: &Connection, reader: impl Read) -> Result<usize> {
|
||||
VALUES (?1,?2,?3,?4,?5,?6,?7)",
|
||||
)?;
|
||||
stream_json_array::<_, ScryPrinting, _>(reader, |p| {
|
||||
if cancel.is_cancelled() {
|
||||
return Err(CoreError::Cancelled);
|
||||
}
|
||||
let Some(oracle_id) = p.oracle_id.clone() else {
|
||||
return Ok(());
|
||||
};
|
||||
@@ -399,6 +674,71 @@ mod tests {
|
||||
assert_eq!(sum, 6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn excludes_art_series_cards() {
|
||||
let oracle = r#"[
|
||||
{"oracle_id":"oid-1","name":"Llanowar Elves","cmc":1.0,
|
||||
"colors":["G"],"color_identity":["G"],
|
||||
"type_line":"Creature — Elf Druid","keywords":[]},
|
||||
{"oracle_id":"oid-2","name":"Llanowar Elves","layout":"art_series",
|
||||
"type_line":"Card","keywords":[]}
|
||||
]"#;
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let dest = dir.path().join("reference.db");
|
||||
let conn = open_reference_for_build(&dest).unwrap();
|
||||
|
||||
let oc = load_oracle_cards(&conn, oracle.as_bytes(), &CancelToken::new()).unwrap();
|
||||
assert_eq!(oc, 1);
|
||||
|
||||
let count: i64 = conn
|
||||
.query_row(
|
||||
"SELECT count(*) FROM oracle_cards WHERE oracle_id='oid-2'",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(count, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancel_token_aborts_load_with_cancelled_error() {
|
||||
let oracle = r#"[
|
||||
{"oracle_id":"oid-1","name":"Llanowar Elves","cmc":1.0,
|
||||
"colors":["G"],"color_identity":["G"],
|
||||
"type_line":"Creature — Elf Druid","keywords":[]},
|
||||
{"oracle_id":"oid-2","name":"Serra Angel","cmc":5.0,
|
||||
"colors":["W"],"color_identity":["W"],
|
||||
"type_line":"Creature — Angel","keywords":["Flying","Vigilance"]}
|
||||
]"#;
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let dest = dir.path().join("reference.db");
|
||||
let conn = open_reference_for_build(&dest).unwrap();
|
||||
|
||||
let cancel = CancelToken::new();
|
||||
cancel.cancel();
|
||||
let err = load_oracle_cards(&conn, oracle.as_bytes(), &cancel)
|
||||
.map_err(|e| override_if_cancelled(e, &cancel))
|
||||
.unwrap_err();
|
||||
assert!(matches!(err, CoreError::Cancelled));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stage_plan_weights_sum_to_one() {
|
||||
let stages = stage_plan(171_000_000, 530_000_000);
|
||||
let total: f64 = stages.iter().map(|s| s.weight).sum();
|
||||
assert!((total - 1.0).abs() < 1e-9);
|
||||
// The larger download gets the larger share of the two DL stages.
|
||||
let oracle_w = stages.iter().find(|s| s.id == STAGE_ORACLE).unwrap().weight;
|
||||
let printings_w = stages
|
||||
.iter()
|
||||
.find(|s| s.id == STAGE_PRINTINGS)
|
||||
.unwrap()
|
||||
.weight;
|
||||
assert!(printings_w > oracle_w);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oracle_and_printing_load_from_fixture_json() {
|
||||
// Minimal Scryfall-shaped fixtures exercising the insert paths.
|
||||
@@ -420,8 +760,8 @@ mod tests {
|
||||
let dest = dir.path().join("reference.db");
|
||||
let conn = open_reference_for_build(&dest).unwrap();
|
||||
|
||||
let oc = load_oracle_cards(&conn, oracle.as_bytes()).unwrap();
|
||||
let pc = load_printings(&conn, printings.as_bytes()).unwrap();
|
||||
let oc = load_oracle_cards(&conn, oracle.as_bytes(), &CancelToken::new()).unwrap();
|
||||
let pc = load_printings(&conn, printings.as_bytes(), &CancelToken::new()).unwrap();
|
||||
assert_eq!(oc, 2);
|
||||
assert_eq!(pc, 1);
|
||||
|
||||
|
||||
@@ -50,22 +50,39 @@ pub fn set_library_path(db: Db, path: String) -> Res<Status> {
|
||||
#[tauri::command]
|
||||
pub async fn update_card_data(app: AppHandle, db: Db<'_>) -> Res<Status> {
|
||||
let reference_path = db.lock().unwrap().reference_path.clone();
|
||||
let cancel = mtgc_core::ingest::CancelToken::new();
|
||||
db.lock().unwrap().ingest_cancel = Some(cancel.clone());
|
||||
let app_for_progress = app.clone();
|
||||
|
||||
tauri::async_runtime::spawn_blocking(move || {
|
||||
mtgc_core::ingest::build_reference_db(&reference_path, |msg| {
|
||||
let _ = app_for_progress.emit("ingest-progress", msg.to_string());
|
||||
})
|
||||
let result = tauri::async_runtime::spawn_blocking({
|
||||
let cancel = cancel.clone();
|
||||
move || {
|
||||
mtgc_core::ingest::build_reference_db(&reference_path, &cancel, |p| {
|
||||
let _ = app_for_progress.emit("ingest-progress", p);
|
||||
})
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|e| e.to_string())?
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
db.lock().unwrap().ingest_cancel = None;
|
||||
result.map_err(|e| e.to_string())?;
|
||||
|
||||
let mut s = db.lock().unwrap();
|
||||
s.reopen()?;
|
||||
Ok(status_of(&s))
|
||||
}
|
||||
|
||||
/// Cancel an in-progress `update_card_data`, if one is running. No-op otherwise.
|
||||
#[tauri::command]
|
||||
pub fn cancel_update_card_data(db: Db) -> Res<()> {
|
||||
let s = db.lock().unwrap();
|
||||
if let Some(c) = &s.ingest_cancel {
|
||||
c.cancel();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tauri::command]
|
||||
pub fn search_cards(db: Db, query: String, limit: usize) -> Res<Vec<CardRef>> {
|
||||
let s = db.lock().unwrap();
|
||||
|
||||
@@ -24,6 +24,7 @@ pub fn run() {
|
||||
commands::get_status,
|
||||
commands::set_library_path,
|
||||
commands::update_card_data,
|
||||
commands::cancel_update_card_data,
|
||||
commands::search_cards,
|
||||
commands::add_owned,
|
||||
commands::set_owned,
|
||||
|
||||
@@ -10,6 +10,9 @@ pub struct AppState {
|
||||
pub reference_path: PathBuf,
|
||||
pub catalog: Option<Catalog>,
|
||||
pub index: Option<NameIndex>,
|
||||
/// Set while `update_card_data` is running; `cancel_update_card_data`
|
||||
/// flips it to abort the in-progress ingest.
|
||||
pub ingest_cancel: Option<mtgc_core::ingest::CancelToken>,
|
||||
}
|
||||
|
||||
impl AppState {
|
||||
@@ -21,6 +24,7 @@ impl AppState {
|
||||
reference_path: data_dir.join("reference.db"),
|
||||
catalog: None,
|
||||
index: None,
|
||||
ingest_cancel: None,
|
||||
};
|
||||
// Best-effort open; failures just leave the catalog unloaded.
|
||||
let _ = s.reopen();
|
||||
|
||||
123
src/App.tsx
123
src/App.tsx
@@ -1,7 +1,7 @@
|
||||
import { useEffect, useState, useCallback, ReactNode } from "react";
|
||||
import { useEffect, useRef, useState, useCallback, ReactNode } from "react";
|
||||
import { listen } from "@tauri-apps/api/event";
|
||||
import { open } from "@tauri-apps/plugin-dialog";
|
||||
import { api, Status } from "./api";
|
||||
import { open, save } from "@tauri-apps/plugin-dialog";
|
||||
import { api, IngestProgress, IngestStageInfo, Status } from "./api";
|
||||
import { HoverPreviewProvider, usePersistedState } from "./components";
|
||||
import { CardEntry } from "./views/CardEntry";
|
||||
import { LibraryBrowser } from "./views/LibraryBrowser";
|
||||
@@ -51,11 +51,24 @@ const NAV: { id: Tab; label: string; icon: ReactNode }[] = [
|
||||
},
|
||||
];
|
||||
|
||||
/** Percent-fill (0..100) of one segment, given overall cumulative percent. */
|
||||
function segmentFill(stages: IngestStageInfo[], index: number, overallPercent: number): number {
|
||||
let start = 0;
|
||||
for (let i = 0; i < index; i++) start += stages[i].weight;
|
||||
const end = start + stages[index].weight;
|
||||
const [startPct, endPct] = [start * 100, end * 100];
|
||||
if (overallPercent <= startPct) return 0;
|
||||
if (overallPercent >= endPct) return 100;
|
||||
if (endPct === startPct) return 100;
|
||||
return ((overallPercent - startPct) / (endPct - startPct)) * 100;
|
||||
}
|
||||
|
||||
export default function App() {
|
||||
const [status, setStatus] = useState<Status | null>(null);
|
||||
const [tab, setTab] = usePersistedState<Tab>("active-tab", "add");
|
||||
const [progress, setProgress] = useState<string | null>(null);
|
||||
const [progress, setProgress] = useState<IngestProgress | null>(null);
|
||||
const [busy, setBusy] = useState(false);
|
||||
const lastProgress = useRef<IngestProgress | null>(null);
|
||||
|
||||
const refreshStatus = useCallback(async () => {
|
||||
setStatus(await api.getStatus());
|
||||
@@ -63,7 +76,10 @@ export default function App() {
|
||||
|
||||
useEffect(() => {
|
||||
refreshStatus();
|
||||
const un = listen<string>("ingest-progress", (e) => setProgress(e.payload));
|
||||
const un = listen<IngestProgress>("ingest-progress", (e) => {
|
||||
lastProgress.current = e.payload;
|
||||
setProgress(e.payload);
|
||||
});
|
||||
return () => {
|
||||
un.then((f) => f());
|
||||
};
|
||||
@@ -71,21 +87,57 @@ export default function App() {
|
||||
|
||||
async function updateCardData() {
|
||||
setBusy(true);
|
||||
setProgress("Starting…");
|
||||
try {
|
||||
const s = await api.updateCardData();
|
||||
setStatus(s);
|
||||
setProgress(`Loaded ${s.card_count.toLocaleString()} cards.`);
|
||||
setProgress({
|
||||
stage: "done",
|
||||
message: `Loaded ${s.card_count.toLocaleString()} cards.`,
|
||||
overall_percent: 100,
|
||||
stages: lastProgress.current?.stages ?? [],
|
||||
});
|
||||
setTimeout(() => setProgress((p) => (p?.stage === "done" ? null : p)), 4000);
|
||||
} catch (err) {
|
||||
setProgress(`Failed: ${String(err)}`);
|
||||
const msg = String(err);
|
||||
if (msg === "cancelled") {
|
||||
setProgress({
|
||||
stage: "cancelled",
|
||||
message: "Cancelled.",
|
||||
overall_percent: lastProgress.current?.overall_percent ?? 0,
|
||||
stages: lastProgress.current?.stages ?? [],
|
||||
});
|
||||
setTimeout(() => setProgress((p) => (p?.stage === "cancelled" ? null : p)), 3000);
|
||||
} else {
|
||||
setProgress({
|
||||
stage: "error",
|
||||
message: `Failed: ${msg}`,
|
||||
overall_percent: lastProgress.current?.overall_percent ?? 0,
|
||||
stages: lastProgress.current?.stages ?? [],
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function chooseLibrary() {
|
||||
async function cancelUpdate() {
|
||||
await api.cancelUpdateCardData();
|
||||
}
|
||||
|
||||
async function openLibrary() {
|
||||
const path = await open({
|
||||
title: "Choose or create library.db (put it in a synced folder)",
|
||||
title: "Open an existing library.db",
|
||||
filters: [{ name: "SQLite", extensions: ["db", "sqlite"] }],
|
||||
});
|
||||
if (typeof path === "string") {
|
||||
setStatus(await api.setLibraryPath(path));
|
||||
}
|
||||
}
|
||||
|
||||
async function newLibrary() {
|
||||
const path = await save({
|
||||
title: "Create a new library.db (put it in a synced folder)",
|
||||
defaultPath: "library.db",
|
||||
filters: [{ name: "SQLite", extensions: ["db", "sqlite"] }],
|
||||
});
|
||||
if (typeof path === "string") {
|
||||
@@ -131,15 +183,60 @@ export default function App() {
|
||||
</span>
|
||||
)}
|
||||
<span className="spacer" />
|
||||
{progress && <span className="progress">{progress}</span>}
|
||||
<button onClick={chooseLibrary} disabled={busy}>
|
||||
Choose library…
|
||||
<button onClick={openLibrary} disabled={busy}>
|
||||
Open Library…
|
||||
</button>
|
||||
<button onClick={newLibrary} disabled={busy}>
|
||||
New Library…
|
||||
</button>
|
||||
<button onClick={updateCardData} disabled={busy} className="primary">
|
||||
{ready ? "Update card data" : "Download card data"}
|
||||
</button>
|
||||
</header>
|
||||
|
||||
{progress && (
|
||||
<div className={`ingest-bar ingest-bar-${progress.stage}`}>
|
||||
<div className="ingest-bar-head">
|
||||
<span className="ingest-bar-message">{progress.message}</span>
|
||||
<span className="spacer" />
|
||||
{busy && (
|
||||
<button className="ingest-bar-cancel" onClick={cancelUpdate}>
|
||||
Cancel
|
||||
</button>
|
||||
)}
|
||||
{!busy &&
|
||||
(progress.stage === "done" ||
|
||||
progress.stage === "error" ||
|
||||
progress.stage === "cancelled") && (
|
||||
<button
|
||||
className="ingest-bar-dismiss"
|
||||
onClick={() => setProgress(null)}
|
||||
title="Dismiss"
|
||||
>
|
||||
✕
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
<div className="ingest-segments">
|
||||
{progress.stages.map((s, i) => (
|
||||
<div
|
||||
key={s.id}
|
||||
className={`ingest-segment ${s.id === progress.stage ? "active" : ""}`}
|
||||
style={{ flexGrow: s.weight, flexBasis: 0 }}
|
||||
>
|
||||
<div className="ingest-segment-track">
|
||||
<div
|
||||
className="ingest-segment-fill"
|
||||
style={{ width: `${segmentFill(progress.stages, i, progress.overall_percent)}%` }}
|
||||
/>
|
||||
</div>
|
||||
<span className="ingest-segment-label">{s.label}</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="content">
|
||||
{!ready ? (
|
||||
<div className="empty-state">
|
||||
|
||||
14
src/api.ts
14
src/api.ts
@@ -54,6 +54,19 @@ export interface Status {
|
||||
card_count: number;
|
||||
}
|
||||
|
||||
export interface IngestStageInfo {
|
||||
id: string;
|
||||
label: string;
|
||||
weight: number; // fraction of total work, 0..1, sums to 1 across all stages
|
||||
}
|
||||
|
||||
export interface IngestProgress {
|
||||
stage: "index" | "oracle" | "printings" | "optimize" | string;
|
||||
message: string;
|
||||
overall_percent: number; // 0..100 cumulative across all stages
|
||||
stages: IngestStageInfo[];
|
||||
}
|
||||
|
||||
// --- WUBRG bitmask helpers ------------------------------------------------
|
||||
|
||||
export const COLORS: { bit: number; code: string; label: string }[] = [
|
||||
@@ -76,6 +89,7 @@ export const api = {
|
||||
getStatus: () => invoke<Status>("get_status"),
|
||||
setLibraryPath: (path: string) => invoke<Status>("set_library_path", { path }),
|
||||
updateCardData: () => invoke<Status>("update_card_data"),
|
||||
cancelUpdateCardData: () => invoke<void>("cancel_update_card_data"),
|
||||
|
||||
searchCards: (query: string, limit = 20) =>
|
||||
invoke<CardRef[]>("search_cards", { query, limit }),
|
||||
|
||||
104
src/styles.css
104
src/styles.css
@@ -292,14 +292,108 @@ select:focus {
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
}
|
||||
.progress {
|
||||
color: var(--accent-hover);
|
||||
font-size: 12px;
|
||||
font-weight: 500;
|
||||
}
|
||||
.spacer {
|
||||
flex: 1;
|
||||
}
|
||||
|
||||
/* Ingest (card-data download/index) progress: one multi-segment bar spanning
|
||||
all stages, below the topbar. */
|
||||
.ingest-bar {
|
||||
padding: 8px 18px 10px;
|
||||
background: var(--bg-elev);
|
||||
border-bottom: 1px solid var(--border-soft);
|
||||
font-size: 12px;
|
||||
}
|
||||
.ingest-bar-head {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 12px;
|
||||
margin-bottom: 6px;
|
||||
}
|
||||
.ingest-bar-message {
|
||||
color: var(--muted);
|
||||
font-family: ui-monospace, "SF Mono", Menlo, monospace;
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
.ingest-bar-cancel {
|
||||
flex-shrink: 0;
|
||||
padding: 3px 10px;
|
||||
font-size: 11px;
|
||||
color: var(--muted);
|
||||
}
|
||||
.ingest-bar-cancel:hover {
|
||||
color: var(--danger);
|
||||
border-color: var(--danger);
|
||||
}
|
||||
.ingest-bar-dismiss {
|
||||
flex-shrink: 0;
|
||||
background: none;
|
||||
border: none;
|
||||
color: var(--faint);
|
||||
cursor: pointer;
|
||||
padding: 2px 6px;
|
||||
font-size: 13px;
|
||||
}
|
||||
.ingest-bar-dismiss:hover {
|
||||
color: var(--text);
|
||||
}
|
||||
|
||||
.ingest-segments {
|
||||
display: flex;
|
||||
gap: 4px;
|
||||
}
|
||||
.ingest-segment {
|
||||
min-width: 0;
|
||||
}
|
||||
.ingest-segment-track {
|
||||
height: 6px;
|
||||
border-radius: 999px;
|
||||
background: rgba(0, 0, 0, 0.3);
|
||||
overflow: hidden;
|
||||
}
|
||||
.ingest-segment-fill {
|
||||
height: 100%;
|
||||
border-radius: 999px;
|
||||
background: var(--border);
|
||||
transition: width 0.2s ease;
|
||||
}
|
||||
.ingest-segment.active .ingest-segment-fill {
|
||||
background: linear-gradient(90deg, var(--accent-hover), var(--accent));
|
||||
}
|
||||
.ingest-segment-label {
|
||||
display: block;
|
||||
margin-top: 3px;
|
||||
font-size: 10px;
|
||||
color: var(--faint);
|
||||
white-space: nowrap;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
.ingest-segment.active .ingest-segment-label {
|
||||
color: var(--accent-hover);
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.ingest-bar-done .ingest-segment-fill {
|
||||
background: var(--success);
|
||||
}
|
||||
.ingest-bar-done .ingest-bar-message {
|
||||
color: var(--success);
|
||||
}
|
||||
.ingest-bar-error .ingest-segment-fill {
|
||||
background: var(--danger);
|
||||
}
|
||||
.ingest-bar-error .ingest-bar-message {
|
||||
color: var(--danger);
|
||||
}
|
||||
.ingest-bar-cancelled .ingest-segment-fill {
|
||||
background: var(--warn);
|
||||
}
|
||||
.ingest-bar-cancelled .ingest-bar-message {
|
||||
color: var(--warn);
|
||||
}
|
||||
.content {
|
||||
flex: 1;
|
||||
overflow: auto;
|
||||
|
||||
Reference in New Issue
Block a user