449 lines
14 KiB
Rust
449 lines
14 KiB
Rust
|
//! Build `reference.db` from Scryfall bulk data.
|
||
|
|
//!
|
||
|
|
//! Two passes: the *Oracle Cards* bulk (one entry per `oracle_id`) fills
|
||
|
|
//! `oracle_cards` + the normalized type/subtype/keyword tables + the name FTS;
|
||
|
|
//! the *Default Cards* bulk (one entry per printing) fills `printings`.
|
||
|
|
//!
|
||
|
|
//! Both bulk files are large JSON arrays, so we stream them element-by-element
|
||
|
|
//! rather than loading the whole array into memory.
|
||
|
|
|
||
|
|
use std::io::{BufReader, Read};
|
||
|
|
use std::marker::PhantomData;
|
||
|
|
use std::path::Path;
|
||
|
|
|
||
|
|
use rusqlite::{params, Connection};
|
||
|
|
use serde::de::{DeserializeOwned, SeqAccess, Visitor};
|
||
|
|
use serde::Deserialize;
|
||
|
|
|
||
|
|
use crate::db::open_reference_for_build;
|
||
|
|
use crate::error::{CoreError, Result};
|
||
|
|
use crate::model::ColorSet;
|
||
|
|
|
||
|
|
const BULK_INDEX_URL: &str = "https://api.scryfall.com/bulk-data";
|
||
|
|
const USER_AGENT: &str = "mtgc/0.1 (personal library catalog)";
|
||
|
|
|
||
|
|
/// Row counts from a build, for reporting/verification.
|
||
|
|
#[derive(Debug, Default, Clone, Copy)]
|
||
|
|
pub struct IngestStats {
|
||
|
|
pub oracle_cards: usize,
|
||
|
|
pub printings: usize,
|
||
|
|
}
|
||
|
|
|
||
|
|
/// Download the latest Scryfall bulk data and (re)build `reference.db` at `dest`.
|
||
|
|
///
|
||
|
|
/// `progress` is called with human-readable status lines.
|
||
|
|
pub fn build_reference_db(dest: &Path, mut progress: impl FnMut(&str)) -> Result<IngestStats> {
|
||
|
|
let (oracle_uri, default_uri) = fetch_bulk_uris(&mut progress)?;
|
||
|
|
|
||
|
|
if dest.exists() {
|
||
|
|
std::fs::remove_file(dest)?;
|
||
|
|
}
|
||
|
|
let conn = open_reference_for_build(dest)?;
|
||
|
|
// Bulk-load fast; this db is disposable/rebuildable so durability is moot.
|
||
|
|
conn.pragma_update(None, "journal_mode", "MEMORY")?;
|
||
|
|
conn.pragma_update(None, "synchronous", "OFF")?;
|
||
|
|
|
||
|
|
progress("Loading oracle cards…");
|
||
|
|
let oracle_cards = load_oracle_cards(&conn, http_reader(&oracle_uri)?)?;
|
||
|
|
|
||
|
|
progress("Loading printings…");
|
||
|
|
let printings = load_printings(&conn, http_reader(&default_uri)?)?;
|
||
|
|
|
||
|
|
progress("Optimizing…");
|
||
|
|
conn.execute_batch("INSERT INTO oracle_names_fts(oracle_names_fts) VALUES('optimize');")?;
|
||
|
|
|
||
|
|
Ok(IngestStats {
|
||
|
|
oracle_cards,
|
||
|
|
printings,
|
||
|
|
})
|
||
|
|
}
|
||
|
|
|
||
|
|
// ---- Scryfall bulk index -------------------------------------------------
|
||
|
|
|
||
|
|
#[derive(Deserialize)]
|
||
|
|
struct BulkIndex {
|
||
|
|
data: Vec<BulkEntry>,
|
||
|
|
}
|
||
|
|
|
||
|
|
#[derive(Deserialize)]
|
||
|
|
struct BulkEntry {
|
||
|
|
#[serde(rename = "type")]
|
||
|
|
kind: String,
|
||
|
|
download_uri: String,
|
||
|
|
}
|
||
|
|
|
||
|
|
fn fetch_bulk_uris(progress: &mut impl FnMut(&str)) -> Result<(String, String)> {
|
||
|
|
progress("Fetching Scryfall bulk index…");
|
||
|
|
let index: BulkIndex = ureq::get(BULK_INDEX_URL)
|
||
|
|
.set("User-Agent", USER_AGENT)
|
||
|
|
.set("Accept", "application/json")
|
||
|
|
.call()?
|
||
|
|
.into_json()?;
|
||
|
|
|
||
|
|
let find = |kind: &str| {
|
||
|
|
index
|
||
|
|
.data
|
||
|
|
.iter()
|
||
|
|
.find(|e| e.kind == kind)
|
||
|
|
.map(|e| e.download_uri.clone())
|
||
|
|
.ok_or_else(|| CoreError::Other(format!("Scryfall bulk index missing '{kind}'")))
|
||
|
|
};
|
||
|
|
Ok((find("oracle_cards")?, find("default_cards")?))
|
||
|
|
}
|
||
|
|
|
||
|
|
fn http_reader(url: &str) -> Result<impl Read> {
|
||
|
|
let resp = ureq::get(url)
|
||
|
|
.set("User-Agent", USER_AGENT)
|
||
|
|
.call()?;
|
||
|
|
Ok(BufReader::with_capacity(1 << 20, resp.into_reader()))
|
||
|
|
}
|
||
|
|
|
||
|
|
// ---- Streaming array parse ----------------------------------------------
|
||
|
|
|
||
|
|
/// Deserialize a top-level JSON array element-by-element, invoking `f` per item
|
||
|
|
/// without holding the whole array in memory.
|
||
|
|
fn stream_json_array<R, T, F>(reader: R, f: F) -> Result<()>
|
||
|
|
where
|
||
|
|
R: Read,
|
||
|
|
T: DeserializeOwned,
|
||
|
|
F: FnMut(T) -> Result<()>,
|
||
|
|
{
|
||
|
|
struct SeqVisitor<T, F> {
|
||
|
|
f: F,
|
||
|
|
_marker: PhantomData<fn() -> T>,
|
||
|
|
}
|
||
|
|
|
||
|
|
impl<'de, T, F> Visitor<'de> for SeqVisitor<T, F>
|
||
|
|
where
|
||
|
|
T: DeserializeOwned,
|
||
|
|
F: FnMut(T) -> Result<()>,
|
||
|
|
{
|
||
|
|
type Value = ();
|
||
|
|
|
||
|
|
fn expecting(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
|
||
|
|
fmt.write_str("a JSON array of objects")
|
||
|
|
}
|
||
|
|
|
||
|
|
fn visit_seq<A>(mut self, mut seq: A) -> std::result::Result<(), A::Error>
|
||
|
|
where
|
||
|
|
A: SeqAccess<'de>,
|
||
|
|
{
|
||
|
|
while let Some(item) = seq.next_element::<T>()? {
|
||
|
|
(self.f)(item).map_err(serde::de::Error::custom)?;
|
||
|
|
}
|
||
|
|
Ok(())
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
let mut de = serde_json::Deserializer::from_reader(reader);
|
||
|
|
serde::de::Deserializer::deserialize_seq(
|
||
|
|
&mut de,
|
||
|
|
SeqVisitor {
|
||
|
|
f,
|
||
|
|
_marker: PhantomData,
|
||
|
|
},
|
||
|
|
)?;
|
||
|
|
Ok(())
|
||
|
|
}
|
||
|
|
|
||
|
|
// ---- Oracle pass ---------------------------------------------------------
|
||
|
|
|
||
|
|
#[derive(Deserialize)]
|
||
|
|
struct ScryOracle {
|
||
|
|
oracle_id: Option<String>,
|
||
|
|
name: String,
|
||
|
|
#[serde(default)]
|
||
|
|
cmc: f64,
|
||
|
|
#[serde(default)]
|
||
|
|
colors: Option<Vec<String>>,
|
||
|
|
#[serde(default)]
|
||
|
|
color_identity: Vec<String>,
|
||
|
|
#[serde(default)]
|
||
|
|
type_line: Option<String>,
|
||
|
|
#[serde(default)]
|
||
|
|
oracle_text: Option<String>,
|
||
|
|
#[serde(default)]
|
||
|
|
keywords: Vec<String>,
|
||
|
|
#[serde(default)]
|
||
|
|
card_faces: Option<Vec<ScryFace>>,
|
||
|
|
}
|
||
|
|
|
||
|
|
#[derive(Deserialize)]
|
||
|
|
struct ScryFace {
|
||
|
|
#[serde(default)]
|
||
|
|
type_line: Option<String>,
|
||
|
|
#[serde(default)]
|
||
|
|
colors: Option<Vec<String>>,
|
||
|
|
}
|
||
|
|
|
||
|
|
fn load_oracle_cards(conn: &Connection, reader: impl Read) -> Result<usize> {
|
||
|
|
conn.execute_batch("BEGIN")?;
|
||
|
|
let mut count = 0usize;
|
||
|
|
{
|
||
|
|
let mut ins_card = conn.prepare(
|
||
|
|
"INSERT OR IGNORE INTO oracle_cards
|
||
|
|
(oracle_id,name,name_lower,mana_value,colors,color_identity,type_line,oracle_text)
|
||
|
|
VALUES (?1,?2,?3,?4,?5,?6,?7,?8)",
|
||
|
|
)?;
|
||
|
|
let mut ins_fts =
|
||
|
|
conn.prepare("INSERT INTO oracle_names_fts(name,oracle_id) VALUES (?1,?2)")?;
|
||
|
|
let mut ins_type =
|
||
|
|
conn.prepare("INSERT INTO card_types(oracle_id,type) VALUES (?1,?2)")?;
|
||
|
|
let mut ins_sub =
|
||
|
|
conn.prepare("INSERT INTO card_subtypes(oracle_id,subtype) VALUES (?1,?2)")?;
|
||
|
|
let mut ins_kw =
|
||
|
|
conn.prepare("INSERT INTO card_keywords(oracle_id,keyword) VALUES (?1,?2)")?;
|
||
|
|
|
||
|
|
stream_json_array::<_, ScryOracle, _>(reader, |c| {
|
||
|
|
let Some(oracle_id) = c.oracle_id.clone() else {
|
||
|
|
return Ok(()); // rare face-only records without a top-level oracle_id
|
||
|
|
};
|
||
|
|
|
||
|
|
// Colors: prefer top-level, else union of faces.
|
||
|
|
let colors = match &c.colors {
|
||
|
|
Some(cs) => ColorSet::from_letters(cs),
|
||
|
|
None => {
|
||
|
|
let mut m = 0u8;
|
||
|
|
if let Some(faces) = &c.card_faces {
|
||
|
|
for f in faces {
|
||
|
|
if let Some(cs) = &f.colors {
|
||
|
|
m |= ColorSet::from_letters(cs).0;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
ColorSet(m)
|
||
|
|
}
|
||
|
|
};
|
||
|
|
let identity = ColorSet::from_letters(&c.color_identity);
|
||
|
|
|
||
|
|
// Type line: top-level, else join faces.
|
||
|
|
let type_line = c.type_line.clone().unwrap_or_else(|| {
|
||
|
|
c.card_faces
|
||
|
|
.as_ref()
|
||
|
|
.map(|faces| {
|
||
|
|
faces
|
||
|
|
.iter()
|
||
|
|
.filter_map(|f| f.type_line.clone())
|
||
|
|
.collect::<Vec<_>>()
|
||
|
|
.join(" // ")
|
||
|
|
})
|
||
|
|
.unwrap_or_default()
|
||
|
|
});
|
||
|
|
|
||
|
|
let changed = ins_card.execute(params![
|
||
|
|
oracle_id,
|
||
|
|
c.name,
|
||
|
|
c.name.to_lowercase(),
|
||
|
|
c.cmc,
|
||
|
|
colors.0 as i64,
|
||
|
|
identity.0 as i64,
|
||
|
|
type_line,
|
||
|
|
c.oracle_text.clone().unwrap_or_default(),
|
||
|
|
])?;
|
||
|
|
// Only emit satellite rows the first time we see this oracle_id.
|
||
|
|
if changed == 1 {
|
||
|
|
ins_fts.execute(params![c.name, oracle_id])?;
|
||
|
|
let (types, subtypes) = parse_type_line(&type_line);
|
||
|
|
for t in types {
|
||
|
|
ins_type.execute(params![oracle_id, t])?;
|
||
|
|
}
|
||
|
|
for s in subtypes {
|
||
|
|
ins_sub.execute(params![oracle_id, s])?;
|
||
|
|
}
|
||
|
|
let mut kws = c.keywords.clone();
|
||
|
|
kws.sort();
|
||
|
|
kws.dedup();
|
||
|
|
for k in kws {
|
||
|
|
ins_kw.execute(params![oracle_id, k])?;
|
||
|
|
}
|
||
|
|
count += 1;
|
||
|
|
}
|
||
|
|
Ok(())
|
||
|
|
})?;
|
||
|
|
}
|
||
|
|
conn.execute_batch("COMMIT")?;
|
||
|
|
Ok(count)
|
||
|
|
}
|
||
|
|
|
||
|
|
/// Split a Scryfall type line into (types+supertypes, subtypes). Handles split /
|
||
|
|
/// double-faced lines joined by " // ". The separator is the em dash '—'.
|
||
|
|
pub fn parse_type_line(type_line: &str) -> (Vec<String>, Vec<String>) {
|
||
|
|
let mut types = Vec::new();
|
||
|
|
let mut subtypes = Vec::new();
|
||
|
|
for part in type_line.split("//") {
|
||
|
|
let part = part.trim();
|
||
|
|
let mut halves = part.splitn(2, '—');
|
||
|
|
let left = halves.next().unwrap_or("").trim();
|
||
|
|
let right = halves.next().unwrap_or("").trim();
|
||
|
|
for t in left.split_whitespace() {
|
||
|
|
types.push(t.to_string());
|
||
|
|
}
|
||
|
|
for s in right.split_whitespace() {
|
||
|
|
subtypes.push(s.to_string());
|
||
|
|
}
|
||
|
|
}
|
||
|
|
types.sort();
|
||
|
|
types.dedup();
|
||
|
|
subtypes.sort();
|
||
|
|
subtypes.dedup();
|
||
|
|
(types, subtypes)
|
||
|
|
}
|
||
|
|
|
||
|
|
// ---- Printings pass ------------------------------------------------------
|
||
|
|
|
||
|
|
#[derive(Deserialize)]
|
||
|
|
struct ScryPrinting {
|
||
|
|
id: String,
|
||
|
|
oracle_id: Option<String>,
|
||
|
|
#[serde(default)]
|
||
|
|
set: String,
|
||
|
|
#[serde(default)]
|
||
|
|
collector_number: String,
|
||
|
|
#[serde(default)]
|
||
|
|
finishes: Vec<String>,
|
||
|
|
#[serde(default)]
|
||
|
|
rarity: String,
|
||
|
|
#[serde(default)]
|
||
|
|
image_uris: Option<ImageUris>,
|
||
|
|
#[serde(default)]
|
||
|
|
card_faces: Option<Vec<PrintFace>>,
|
||
|
|
}
|
||
|
|
|
||
|
|
#[derive(Deserialize)]
|
||
|
|
struct ImageUris {
|
||
|
|
#[serde(default)]
|
||
|
|
normal: Option<String>,
|
||
|
|
}
|
||
|
|
|
||
|
|
#[derive(Deserialize)]
|
||
|
|
struct PrintFace {
|
||
|
|
#[serde(default)]
|
||
|
|
image_uris: Option<ImageUris>,
|
||
|
|
}
|
||
|
|
|
||
|
|
fn load_printings(conn: &Connection, reader: impl Read) -> Result<usize> {
|
||
|
|
conn.execute_batch("BEGIN")?;
|
||
|
|
let mut count = 0usize;
|
||
|
|
{
|
||
|
|
let mut ins = conn.prepare(
|
||
|
|
"INSERT OR IGNORE INTO printings
|
||
|
|
(scryfall_id,oracle_id,set_code,collector_number,finishes,rarity,image_uri)
|
||
|
|
VALUES (?1,?2,?3,?4,?5,?6,?7)",
|
||
|
|
)?;
|
||
|
|
stream_json_array::<_, ScryPrinting, _>(reader, |p| {
|
||
|
|
let Some(oracle_id) = p.oracle_id.clone() else {
|
||
|
|
return Ok(());
|
||
|
|
};
|
||
|
|
let image = p
|
||
|
|
.image_uris
|
||
|
|
.as_ref()
|
||
|
|
.and_then(|i| i.normal.clone())
|
||
|
|
.or_else(|| {
|
||
|
|
p.card_faces
|
||
|
|
.as_ref()
|
||
|
|
.and_then(|f| f.first())
|
||
|
|
.and_then(|f| f.image_uris.as_ref())
|
||
|
|
.and_then(|i| i.normal.clone())
|
||
|
|
})
|
||
|
|
.unwrap_or_default();
|
||
|
|
ins.execute(params![
|
||
|
|
p.id,
|
||
|
|
oracle_id,
|
||
|
|
p.set,
|
||
|
|
p.collector_number,
|
||
|
|
p.finishes.join(","),
|
||
|
|
p.rarity,
|
||
|
|
image,
|
||
|
|
])?;
|
||
|
|
count += 1;
|
||
|
|
Ok(())
|
||
|
|
})?;
|
||
|
|
}
|
||
|
|
conn.execute_batch("COMMIT")?;
|
||
|
|
Ok(count)
|
||
|
|
}
|
||
|
|
|
||
|
|
#[cfg(test)]
|
||
|
|
mod tests {
|
||
|
|
use super::*;
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn parses_type_lines() {
|
||
|
|
let (t, s) = parse_type_line("Legendary Creature — Elf Druid");
|
||
|
|
assert_eq!(t, vec!["Creature", "Legendary"]);
|
||
|
|
assert_eq!(s, vec!["Druid", "Elf"]);
|
||
|
|
|
||
|
|
let (t, s) = parse_type_line("Instant");
|
||
|
|
assert_eq!(t, vec!["Instant"]);
|
||
|
|
assert!(s.is_empty());
|
||
|
|
|
||
|
|
// Split card.
|
||
|
|
let (t, s) = parse_type_line("Sorcery // Sorcery");
|
||
|
|
assert_eq!(t, vec!["Sorcery"]);
|
||
|
|
assert!(s.is_empty());
|
||
|
|
}
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn streams_array_of_objects() {
|
||
|
|
#[derive(serde::Deserialize)]
|
||
|
|
struct Item {
|
||
|
|
n: i32,
|
||
|
|
}
|
||
|
|
let json = br#"[{"n":1},{"n":2},{"n":3}]"#;
|
||
|
|
let mut sum = 0;
|
||
|
|
stream_json_array::<_, Item, _>(&json[..], |it| {
|
||
|
|
sum += it.n;
|
||
|
|
Ok(())
|
||
|
|
})
|
||
|
|
.unwrap();
|
||
|
|
assert_eq!(sum, 6);
|
||
|
|
}
|
||
|
|
|
||
|
|
#[test]
|
||
|
|
fn oracle_and_printing_load_from_fixture_json() {
|
||
|
|
// Minimal Scryfall-shaped fixtures exercising the insert paths.
|
||
|
|
let oracle = r#"[
|
||
|
|
{"oracle_id":"oid-1","name":"Llanowar Elves","cmc":1.0,
|
||
|
|
"colors":["G"],"color_identity":["G"],
|
||
|
|
"type_line":"Creature — Elf Druid","keywords":[]},
|
||
|
|
{"oracle_id":"oid-2","name":"Serra Angel","cmc":5.0,
|
||
|
|
"colors":["W"],"color_identity":["W"],
|
||
|
|
"type_line":"Creature — Angel","keywords":["Flying","Vigilance"]}
|
||
|
|
]"#;
|
||
|
|
let printings = r#"[
|
||
|
|
{"id":"p-1","oracle_id":"oid-1","set":"m19","collector_number":"310",
|
||
|
|
"finishes":["nonfoil","foil"],"rarity":"common",
|
||
|
|
"image_uris":{"normal":"http://img/1"}}
|
||
|
|
]"#;
|
||
|
|
|
||
|
|
let dir = tempfile::tempdir().unwrap();
|
||
|
|
let dest = dir.path().join("reference.db");
|
||
|
|
let conn = open_reference_for_build(&dest).unwrap();
|
||
|
|
|
||
|
|
let oc = load_oracle_cards(&conn, oracle.as_bytes()).unwrap();
|
||
|
|
let pc = load_printings(&conn, printings.as_bytes()).unwrap();
|
||
|
|
assert_eq!(oc, 2);
|
||
|
|
assert_eq!(pc, 1);
|
||
|
|
|
||
|
|
// Keyword table populated.
|
||
|
|
let flyers: i64 = conn
|
||
|
|
.query_row(
|
||
|
|
"SELECT count(*) FROM card_keywords WHERE keyword='Flying'",
|
||
|
|
[],
|
||
|
|
|r| r.get(0),
|
||
|
|
)
|
||
|
|
.unwrap();
|
||
|
|
assert_eq!(flyers, 1);
|
||
|
|
|
||
|
|
// Bitmask colors stored.
|
||
|
|
let g: i64 = conn
|
||
|
|
.query_row(
|
||
|
|
"SELECT colors FROM oracle_cards WHERE oracle_id='oid-1'",
|
||
|
|
[],
|
||
|
|
|r| r.get(0),
|
||
|
|
)
|
||
|
|
.unwrap();
|
||
|
|
assert_eq!(g as u8, crate::model::G);
|
||
|
|
}
|
||
|
|
}
|