Initial commit: MTG personal library catalog (Tauri v2 + React/TS)
Full working app with: - Rust core (mtgc-core): Scryfall ingester, nucleo fuzzy search, SQLite two-DB architecture (reference.db + library.db via ATTACH), criteria filters, deck ops - Tauri v2 shell exposing the core as typed IPC commands - React/TS frontend: card entry, library browser, deck editor - Features: fuzzy card search, color/CMC/type/commander-identity filters, tile/list view toggle, hover card previews, deck format + commander support, per-card location field, grouping by type/subtype/CMC/color, localStorage persistence for active tab/deck/grouping, Discord-inspired dark UI (#C8553D accent) Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
448
crates/mtgc-core/src/ingest.rs
Normal file
448
crates/mtgc-core/src/ingest.rs
Normal file
@@ -0,0 +1,448 @@
|
||||
//! Build `reference.db` from Scryfall bulk data.
|
||||
//!
|
||||
//! Two passes: the *Oracle Cards* bulk (one entry per `oracle_id`) fills
|
||||
//! `oracle_cards` + the normalized type/subtype/keyword tables + the name FTS;
|
||||
//! the *Default Cards* bulk (one entry per printing) fills `printings`.
|
||||
//!
|
||||
//! Both bulk files are large JSON arrays, so we stream them element-by-element
|
||||
//! rather than loading the whole array into memory.
|
||||
|
||||
use std::io::{BufReader, Read};
|
||||
use std::marker::PhantomData;
|
||||
use std::path::Path;
|
||||
|
||||
use rusqlite::{params, Connection};
|
||||
use serde::de::{DeserializeOwned, SeqAccess, Visitor};
|
||||
use serde::Deserialize;
|
||||
|
||||
use crate::db::open_reference_for_build;
|
||||
use crate::error::{CoreError, Result};
|
||||
use crate::model::ColorSet;
|
||||
|
||||
const BULK_INDEX_URL: &str = "https://api.scryfall.com/bulk-data";
|
||||
const USER_AGENT: &str = "mtgc/0.1 (personal library catalog)";
|
||||
|
||||
/// Row counts from a build, for reporting/verification.
|
||||
#[derive(Debug, Default, Clone, Copy)]
|
||||
pub struct IngestStats {
|
||||
pub oracle_cards: usize,
|
||||
pub printings: usize,
|
||||
}
|
||||
|
||||
/// Download the latest Scryfall bulk data and (re)build `reference.db` at `dest`.
|
||||
///
|
||||
/// `progress` is called with human-readable status lines.
|
||||
pub fn build_reference_db(dest: &Path, mut progress: impl FnMut(&str)) -> Result<IngestStats> {
|
||||
let (oracle_uri, default_uri) = fetch_bulk_uris(&mut progress)?;
|
||||
|
||||
if dest.exists() {
|
||||
std::fs::remove_file(dest)?;
|
||||
}
|
||||
let conn = open_reference_for_build(dest)?;
|
||||
// Bulk-load fast; this db is disposable/rebuildable so durability is moot.
|
||||
conn.pragma_update(None, "journal_mode", "MEMORY")?;
|
||||
conn.pragma_update(None, "synchronous", "OFF")?;
|
||||
|
||||
progress("Loading oracle cards…");
|
||||
let oracle_cards = load_oracle_cards(&conn, http_reader(&oracle_uri)?)?;
|
||||
|
||||
progress("Loading printings…");
|
||||
let printings = load_printings(&conn, http_reader(&default_uri)?)?;
|
||||
|
||||
progress("Optimizing…");
|
||||
conn.execute_batch("INSERT INTO oracle_names_fts(oracle_names_fts) VALUES('optimize');")?;
|
||||
|
||||
Ok(IngestStats {
|
||||
oracle_cards,
|
||||
printings,
|
||||
})
|
||||
}
|
||||
|
||||
// ---- Scryfall bulk index -------------------------------------------------
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct BulkIndex {
|
||||
data: Vec<BulkEntry>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct BulkEntry {
|
||||
#[serde(rename = "type")]
|
||||
kind: String,
|
||||
download_uri: String,
|
||||
}
|
||||
|
||||
fn fetch_bulk_uris(progress: &mut impl FnMut(&str)) -> Result<(String, String)> {
|
||||
progress("Fetching Scryfall bulk index…");
|
||||
let index: BulkIndex = ureq::get(BULK_INDEX_URL)
|
||||
.set("User-Agent", USER_AGENT)
|
||||
.set("Accept", "application/json")
|
||||
.call()?
|
||||
.into_json()?;
|
||||
|
||||
let find = |kind: &str| {
|
||||
index
|
||||
.data
|
||||
.iter()
|
||||
.find(|e| e.kind == kind)
|
||||
.map(|e| e.download_uri.clone())
|
||||
.ok_or_else(|| CoreError::Other(format!("Scryfall bulk index missing '{kind}'")))
|
||||
};
|
||||
Ok((find("oracle_cards")?, find("default_cards")?))
|
||||
}
|
||||
|
||||
fn http_reader(url: &str) -> Result<impl Read> {
|
||||
let resp = ureq::get(url)
|
||||
.set("User-Agent", USER_AGENT)
|
||||
.call()?;
|
||||
Ok(BufReader::with_capacity(1 << 20, resp.into_reader()))
|
||||
}
|
||||
|
||||
// ---- Streaming array parse ----------------------------------------------
|
||||
|
||||
/// Deserialize a top-level JSON array element-by-element, invoking `f` per item
|
||||
/// without holding the whole array in memory.
|
||||
fn stream_json_array<R, T, F>(reader: R, f: F) -> Result<()>
|
||||
where
|
||||
R: Read,
|
||||
T: DeserializeOwned,
|
||||
F: FnMut(T) -> Result<()>,
|
||||
{
|
||||
struct SeqVisitor<T, F> {
|
||||
f: F,
|
||||
_marker: PhantomData<fn() -> T>,
|
||||
}
|
||||
|
||||
impl<'de, T, F> Visitor<'de> for SeqVisitor<T, F>
|
||||
where
|
||||
T: DeserializeOwned,
|
||||
F: FnMut(T) -> Result<()>,
|
||||
{
|
||||
type Value = ();
|
||||
|
||||
fn expecting(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||||
fmt.write_str("a JSON array of objects")
|
||||
}
|
||||
|
||||
fn visit_seq<A>(mut self, mut seq: A) -> std::result::Result<(), A::Error>
|
||||
where
|
||||
A: SeqAccess<'de>,
|
||||
{
|
||||
while let Some(item) = seq.next_element::<T>()? {
|
||||
(self.f)(item).map_err(serde::de::Error::custom)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
let mut de = serde_json::Deserializer::from_reader(reader);
|
||||
serde::de::Deserializer::deserialize_seq(
|
||||
&mut de,
|
||||
SeqVisitor {
|
||||
f,
|
||||
_marker: PhantomData,
|
||||
},
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---- Oracle pass ---------------------------------------------------------
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ScryOracle {
|
||||
oracle_id: Option<String>,
|
||||
name: String,
|
||||
#[serde(default)]
|
||||
cmc: f64,
|
||||
#[serde(default)]
|
||||
colors: Option<Vec<String>>,
|
||||
#[serde(default)]
|
||||
color_identity: Vec<String>,
|
||||
#[serde(default)]
|
||||
type_line: Option<String>,
|
||||
#[serde(default)]
|
||||
oracle_text: Option<String>,
|
||||
#[serde(default)]
|
||||
keywords: Vec<String>,
|
||||
#[serde(default)]
|
||||
card_faces: Option<Vec<ScryFace>>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ScryFace {
|
||||
#[serde(default)]
|
||||
type_line: Option<String>,
|
||||
#[serde(default)]
|
||||
colors: Option<Vec<String>>,
|
||||
}
|
||||
|
||||
fn load_oracle_cards(conn: &Connection, reader: impl Read) -> Result<usize> {
|
||||
conn.execute_batch("BEGIN")?;
|
||||
let mut count = 0usize;
|
||||
{
|
||||
let mut ins_card = conn.prepare(
|
||||
"INSERT OR IGNORE INTO oracle_cards
|
||||
(oracle_id,name,name_lower,mana_value,colors,color_identity,type_line,oracle_text)
|
||||
VALUES (?1,?2,?3,?4,?5,?6,?7,?8)",
|
||||
)?;
|
||||
let mut ins_fts =
|
||||
conn.prepare("INSERT INTO oracle_names_fts(name,oracle_id) VALUES (?1,?2)")?;
|
||||
let mut ins_type =
|
||||
conn.prepare("INSERT INTO card_types(oracle_id,type) VALUES (?1,?2)")?;
|
||||
let mut ins_sub =
|
||||
conn.prepare("INSERT INTO card_subtypes(oracle_id,subtype) VALUES (?1,?2)")?;
|
||||
let mut ins_kw =
|
||||
conn.prepare("INSERT INTO card_keywords(oracle_id,keyword) VALUES (?1,?2)")?;
|
||||
|
||||
stream_json_array::<_, ScryOracle, _>(reader, |c| {
|
||||
let Some(oracle_id) = c.oracle_id.clone() else {
|
||||
return Ok(()); // rare face-only records without a top-level oracle_id
|
||||
};
|
||||
|
||||
// Colors: prefer top-level, else union of faces.
|
||||
let colors = match &c.colors {
|
||||
Some(cs) => ColorSet::from_letters(cs),
|
||||
None => {
|
||||
let mut m = 0u8;
|
||||
if let Some(faces) = &c.card_faces {
|
||||
for f in faces {
|
||||
if let Some(cs) = &f.colors {
|
||||
m |= ColorSet::from_letters(cs).0;
|
||||
}
|
||||
}
|
||||
}
|
||||
ColorSet(m)
|
||||
}
|
||||
};
|
||||
let identity = ColorSet::from_letters(&c.color_identity);
|
||||
|
||||
// Type line: top-level, else join faces.
|
||||
let type_line = c.type_line.clone().unwrap_or_else(|| {
|
||||
c.card_faces
|
||||
.as_ref()
|
||||
.map(|faces| {
|
||||
faces
|
||||
.iter()
|
||||
.filter_map(|f| f.type_line.clone())
|
||||
.collect::<Vec<_>>()
|
||||
.join(" // ")
|
||||
})
|
||||
.unwrap_or_default()
|
||||
});
|
||||
|
||||
let changed = ins_card.execute(params![
|
||||
oracle_id,
|
||||
c.name,
|
||||
c.name.to_lowercase(),
|
||||
c.cmc,
|
||||
colors.0 as i64,
|
||||
identity.0 as i64,
|
||||
type_line,
|
||||
c.oracle_text.clone().unwrap_or_default(),
|
||||
])?;
|
||||
// Only emit satellite rows the first time we see this oracle_id.
|
||||
if changed == 1 {
|
||||
ins_fts.execute(params![c.name, oracle_id])?;
|
||||
let (types, subtypes) = parse_type_line(&type_line);
|
||||
for t in types {
|
||||
ins_type.execute(params![oracle_id, t])?;
|
||||
}
|
||||
for s in subtypes {
|
||||
ins_sub.execute(params![oracle_id, s])?;
|
||||
}
|
||||
let mut kws = c.keywords.clone();
|
||||
kws.sort();
|
||||
kws.dedup();
|
||||
for k in kws {
|
||||
ins_kw.execute(params![oracle_id, k])?;
|
||||
}
|
||||
count += 1;
|
||||
}
|
||||
Ok(())
|
||||
})?;
|
||||
}
|
||||
conn.execute_batch("COMMIT")?;
|
||||
Ok(count)
|
||||
}
|
||||
|
||||
/// Split a Scryfall type line into (types+supertypes, subtypes). Handles split /
|
||||
/// double-faced lines joined by " // ". The separator is the em dash '—'.
|
||||
pub fn parse_type_line(type_line: &str) -> (Vec<String>, Vec<String>) {
|
||||
let mut types = Vec::new();
|
||||
let mut subtypes = Vec::new();
|
||||
for part in type_line.split("//") {
|
||||
let part = part.trim();
|
||||
let mut halves = part.splitn(2, '—');
|
||||
let left = halves.next().unwrap_or("").trim();
|
||||
let right = halves.next().unwrap_or("").trim();
|
||||
for t in left.split_whitespace() {
|
||||
types.push(t.to_string());
|
||||
}
|
||||
for s in right.split_whitespace() {
|
||||
subtypes.push(s.to_string());
|
||||
}
|
||||
}
|
||||
types.sort();
|
||||
types.dedup();
|
||||
subtypes.sort();
|
||||
subtypes.dedup();
|
||||
(types, subtypes)
|
||||
}
|
||||
|
||||
// ---- Printings pass ------------------------------------------------------
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ScryPrinting {
|
||||
id: String,
|
||||
oracle_id: Option<String>,
|
||||
#[serde(default)]
|
||||
set: String,
|
||||
#[serde(default)]
|
||||
collector_number: String,
|
||||
#[serde(default)]
|
||||
finishes: Vec<String>,
|
||||
#[serde(default)]
|
||||
rarity: String,
|
||||
#[serde(default)]
|
||||
image_uris: Option<ImageUris>,
|
||||
#[serde(default)]
|
||||
card_faces: Option<Vec<PrintFace>>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ImageUris {
|
||||
#[serde(default)]
|
||||
normal: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct PrintFace {
|
||||
#[serde(default)]
|
||||
image_uris: Option<ImageUris>,
|
||||
}
|
||||
|
||||
fn load_printings(conn: &Connection, reader: impl Read) -> Result<usize> {
|
||||
conn.execute_batch("BEGIN")?;
|
||||
let mut count = 0usize;
|
||||
{
|
||||
let mut ins = conn.prepare(
|
||||
"INSERT OR IGNORE INTO printings
|
||||
(scryfall_id,oracle_id,set_code,collector_number,finishes,rarity,image_uri)
|
||||
VALUES (?1,?2,?3,?4,?5,?6,?7)",
|
||||
)?;
|
||||
stream_json_array::<_, ScryPrinting, _>(reader, |p| {
|
||||
let Some(oracle_id) = p.oracle_id.clone() else {
|
||||
return Ok(());
|
||||
};
|
||||
let image = p
|
||||
.image_uris
|
||||
.as_ref()
|
||||
.and_then(|i| i.normal.clone())
|
||||
.or_else(|| {
|
||||
p.card_faces
|
||||
.as_ref()
|
||||
.and_then(|f| f.first())
|
||||
.and_then(|f| f.image_uris.as_ref())
|
||||
.and_then(|i| i.normal.clone())
|
||||
})
|
||||
.unwrap_or_default();
|
||||
ins.execute(params![
|
||||
p.id,
|
||||
oracle_id,
|
||||
p.set,
|
||||
p.collector_number,
|
||||
p.finishes.join(","),
|
||||
p.rarity,
|
||||
image,
|
||||
])?;
|
||||
count += 1;
|
||||
Ok(())
|
||||
})?;
|
||||
}
|
||||
conn.execute_batch("COMMIT")?;
|
||||
Ok(count)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn parses_type_lines() {
|
||||
let (t, s) = parse_type_line("Legendary Creature — Elf Druid");
|
||||
assert_eq!(t, vec!["Creature", "Legendary"]);
|
||||
assert_eq!(s, vec!["Druid", "Elf"]);
|
||||
|
||||
let (t, s) = parse_type_line("Instant");
|
||||
assert_eq!(t, vec!["Instant"]);
|
||||
assert!(s.is_empty());
|
||||
|
||||
// Split card.
|
||||
let (t, s) = parse_type_line("Sorcery // Sorcery");
|
||||
assert_eq!(t, vec!["Sorcery"]);
|
||||
assert!(s.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn streams_array_of_objects() {
|
||||
#[derive(serde::Deserialize)]
|
||||
struct Item {
|
||||
n: i32,
|
||||
}
|
||||
let json = br#"[{"n":1},{"n":2},{"n":3}]"#;
|
||||
let mut sum = 0;
|
||||
stream_json_array::<_, Item, _>(&json[..], |it| {
|
||||
sum += it.n;
|
||||
Ok(())
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(sum, 6);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oracle_and_printing_load_from_fixture_json() {
|
||||
// Minimal Scryfall-shaped fixtures exercising the insert paths.
|
||||
let oracle = r#"[
|
||||
{"oracle_id":"oid-1","name":"Llanowar Elves","cmc":1.0,
|
||||
"colors":["G"],"color_identity":["G"],
|
||||
"type_line":"Creature — Elf Druid","keywords":[]},
|
||||
{"oracle_id":"oid-2","name":"Serra Angel","cmc":5.0,
|
||||
"colors":["W"],"color_identity":["W"],
|
||||
"type_line":"Creature — Angel","keywords":["Flying","Vigilance"]}
|
||||
]"#;
|
||||
let printings = r#"[
|
||||
{"id":"p-1","oracle_id":"oid-1","set":"m19","collector_number":"310",
|
||||
"finishes":["nonfoil","foil"],"rarity":"common",
|
||||
"image_uris":{"normal":"http://img/1"}}
|
||||
]"#;
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let dest = dir.path().join("reference.db");
|
||||
let conn = open_reference_for_build(&dest).unwrap();
|
||||
|
||||
let oc = load_oracle_cards(&conn, oracle.as_bytes()).unwrap();
|
||||
let pc = load_printings(&conn, printings.as_bytes()).unwrap();
|
||||
assert_eq!(oc, 2);
|
||||
assert_eq!(pc, 1);
|
||||
|
||||
// Keyword table populated.
|
||||
let flyers: i64 = conn
|
||||
.query_row(
|
||||
"SELECT count(*) FROM card_keywords WHERE keyword='Flying'",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(flyers, 1);
|
||||
|
||||
// Bitmask colors stored.
|
||||
let g: i64 = conn
|
||||
.query_row(
|
||||
"SELECT colors FROM oracle_cards WHERE oracle_id='oid-1'",
|
||||
[],
|
||||
|r| r.get(0),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(g as u8, crate::model::G);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user