Accent-insensitive card search

A fold() (lowercase + strip accents, using the same table as nucleo's own
normalization) is registered as a SQLite function and used by the name,
oracle-text and location filters, so "khazad-dum" finds Troll of
Khazad-dûm. The fuzzy name search folds the query too (accented queries
used to match nothing) and its exact-name promotion; decklist/ManaBox
import falls back to a folded name match; the deck quick filter and tag
suggestions fold client-side.

Closes #17

Cargo.toml

@@ -8,7 +8,7 @@ edition = "2021"
 license = "MIT"
 
 [workspace.dependencies]
-rusqlite = { version = "0.31", features = ["bundled"] }
+rusqlite = { version = "0.31", features = ["bundled", "functions"] }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 nucleo = "0.5"

crates/mtgc-core/src/db.rs

@@ -134,6 +134,7 @@ pub fn open_library(library_path: &Path) -> Result<Connection> {
 pub fn open(library_path: &Path, reference_path: &Path) -> Result<Connection> {
     let conn = Connection::open(library_path)?;
     conn.pragma_update(None, "foreign_keys", true)?;
+    crate::text::register_sql_fold(&conn)?;
     migrate_library(&conn)?;
     attach_reference(&conn, reference_path)?;
     Ok(conn)

crates/mtgc-core/src/decklist.rs

@@ -310,23 +310,31 @@ pub fn format_library_text(entries: &[LibraryExportLine]) -> String {
 
 impl Catalog {
     /// Case-insensitive exact-name lookup against the reference db's
-    /// precomputed `name_lower` column. No fuzzy matching — an unresolved
+    /// precomputed `name_lower` column, falling back to an accent-insensitive
-    /// name is reported back to the caller rather than guessed at, so a
+    /// match so a list written without them ("Lim-Dul's Cohort") still
-    /// typo never silently imports the wrong card. When several cards share
+    /// resolves. No fuzzy matching — an unresolved name is reported back to
-    /// the name, a real card wins over a token or an art-series-style "Card".
+    /// the caller rather than guessed at, so a typo never silently imports
+    /// the wrong card. When several cards share the name, a real card wins
+    /// over a token or an art-series-style "Card".
     pub fn find_oracle_id_by_name(&self, name: &str) -> Result<Option<String>> {
-        Ok(self
+        let lookup = |column: &str, key: String| -> Result<Option<String>> {
-            .conn
+            Ok(self
-            .query_row(
+                .conn
-                &format!(
+                .query_row(
-                    "SELECT oracle_id FROM {REF_DB}.oracle_cards WHERE name_lower = ?1
+                    &format!(
-                     ORDER BY (type_line LIKE 'Token%' OR type_line = 'Card'), rowid
+                        "SELECT oracle_id FROM {REF_DB}.oracle_cards WHERE {column} = ?1
-                     LIMIT 1"
+                         ORDER BY (type_line LIKE 'Token%' OR type_line = 'Card'), rowid
-                ),
+                         LIMIT 1"
-                params![name.trim().to_lowercase()],
+                    ),
-                |r| r.get(0),
+                    params![key],
-            )
+                    |r| r.get(0),
-            .optional()?)
+                )
+                .optional()?)
+        };
+        match lookup("name_lower", name.trim().to_lowercase())? {
+            Some(oid) => Ok(Some(oid)),
+            None => lookup("fold(name)", crate::text::fold(name.trim())),
+        }
     }
 
     /// Resolve a decklist line to `(oracle_id, scryfall_id of its printing)`.
@@ -897,6 +905,23 @@ mod tests {
         assert!(angel.is_commander);
     }
 
+    #[test]
+    fn import_resolves_a_name_written_without_its_accents() {
+        let fx = test_support::make();
+        let cat = &fx.catalog;
+        cat.connection()
+            .execute(
+                "UPDATE ref.oracle_cards SET name = 'Lim-Dûl''s Cohort', name_lower = 'lim-dûl''s cohort'
+                  WHERE oracle_id = 'serra-angel'",
+                [],
+            )
+            .unwrap();
+        let (deck, unresolved) = cat.import_deck("Plain", "1 Lim-Dul's Cohort\n").unwrap();
+        assert!(unresolved.is_empty());
+        let contents = cat.deck_contents(deck.id).unwrap();
+        assert_eq!(contents[0].card.oracle_id, "serra-angel");
+    }
+
     #[test]
     fn import_deck_takes_two_commanders_and_folds_the_rest_into_mainboard() {
         let fx = test_support::make();

crates/mtgc-core/src/filters.rs

@@ -7,6 +7,7 @@ use rusqlite::types::Value;
 use crate::db::REF_DB;
 use crate::error::{CoreError, Result};
 use crate::model::{LibraryCard, LibraryFilter};
+use crate::text::fold;
 use crate::Catalog;
 
 /// Append the card-data predicates shared by [`Catalog::search_library`] and
@@ -26,8 +27,8 @@ fn push_collection_filters(sql: &mut String, vals: &mut Vec<Value>, filter: &Lib
         vals.push(Value::Integer(deck_id));
     }
     if let Some(loc) = filter.location.as_ref().map(|l| l.trim()).filter(|l| !l.is_empty()) {
-        sql.push_str(" AND LOWER(lc.location) LIKE ?");
+        sql.push_str(" AND fold(lc.location) LIKE ?");
-        vals.push(Value::Text(format!("%{}%", loc.to_lowercase())));
+        vals.push(Value::Text(format!("%{}%", fold(loc))));
     }
     if filter.spares_only {
         sql.push_str(SPARES_PREDICATE);
@@ -106,8 +107,9 @@ fn push_card_filters(
         );
     }
     if let Some(text) = filter.text.as_ref().map(|t| t.trim()).filter(|t| !t.is_empty()) {
-        sql.push_str(" AND oc.name_lower LIKE ?");
+        // Folded on both sides, so "khazad-dum" finds Troll of Khazad-dûm.
-        vals.push(Value::Text(format!("%{}%", text.to_lowercase())));
+        sql.push_str(" AND fold(oc.name) LIKE ?");
+        vals.push(Value::Text(format!("%{}%", fold(text))));
     }
     if let Some(code) = filter.set_code.as_ref().filter(|c| !c.is_empty()) {
         sql.push_str(&format!(
@@ -188,8 +190,8 @@ fn push_card_filters(
         vals.push(Value::Text(k.clone()));
     }
     if let Some(txt) = filter.oracle_text.as_ref().map(|t| t.trim()).filter(|t| !t.is_empty()) {
-        sql.push_str(" AND LOWER(oc.oracle_text) LIKE ?");
+        sql.push_str(" AND fold(oc.oracle_text) LIKE ?");
-        vals.push(Value::Text(format!("%{}%", txt.to_lowercase())));
+        vals.push(Value::Text(format!("%{}%", fold(txt))));
     }
 }
 
@@ -304,8 +306,8 @@ impl Catalog {
         // off by the LIMIT. Bound after the WHERE values and before the LIMIT, so
         // the positional order still lines up.
         if let Some(text) = filter.text.as_ref().map(|t| t.trim()).filter(|t| !t.is_empty()) {
-            sql.push_str("oc.name_lower <> ?, ");
+            sql.push_str("fold(oc.name) <> ?, ");
-            vals.push(Value::Text(text.to_lowercase()));
+            vals.push(Value::Text(fold(text)));
         }
         sql.push_str("oc.edhrec_rank IS NULL, oc.edhrec_rank, oc.name COLLATE NOCASE LIMIT ?");
         vals.push(Value::Integer(limit as i64));
@@ -340,6 +342,36 @@ mod tests {
         cards.iter().map(|c| c.card.name.clone()).collect()
     }
 
+    #[test]
+    fn name_and_oracle_text_match_regardless_of_accents() {
+        let fx = test_support::make();
+        let cat = &fx.catalog;
+        cat.connection()
+            .execute(
+                "UPDATE ref.oracle_cards
+                    SET name = 'Troll of Khazad-dûm', name_lower = 'troll of khazad-dûm',
+                        oracle_text = 'Swampcycling. Lim-Dûl approves.'
+                  WHERE oracle_id = 'serra-angel'",
+                [],
+            )
+            .unwrap();
+        cat.add_owned("serra-angel", 1).unwrap();
+        cat.add_owned("lightning-bolt", 1).unwrap();
+
+        let by_name = |text: &str| LibraryFilter { text: Some(text.into()), ..Default::default() };
+        let by_text = |text: &str| LibraryFilter { oracle_text: Some(text.into()), ..Default::default() };
+
+        // Plain query, accented card — in the library and the whole card pool.
+        assert_eq!(names(&cat.search_library(&by_name("khazad-dum")).unwrap()), vec!["Troll of Khazad-dûm"]);
+        assert_eq!(
+            names(&cat.search_reference(&by_name("KHAZAD-DUM"), 10).unwrap()),
+            vec!["Troll of Khazad-dûm"]
+        );
+        // Accented query, plain card.
+        assert_eq!(names(&cat.search_library(&by_name("líghtning")).unwrap()), vec!["Lightning Bolt"]);
+        assert_eq!(names(&cat.search_reference(&by_text("lim-dul"), 10).unwrap()), vec!["Troll of Khazad-dûm"]);
+    }
+
     #[test]
     fn empty_filter_returns_all_owned() {
         let fx = owned_fixture();

crates/mtgc-core/src/lib.rs

@@ -21,6 +21,7 @@ pub mod printings;
 pub mod search;
 pub mod sync;
 pub mod synergy;
+pub mod text;
 
 use std::path::Path;
 

crates/mtgc-core/src/search.rs

@@ -91,17 +91,21 @@ impl NameIndex {
             return Vec::new();
         }
         let mut matcher = Matcher::new(Config::DEFAULT);
-        let pattern = Pattern::parse(query, CaseMatching::Ignore, Normalization::Smart);
+        // nucleo's Smart normalization turns itself off when the query has an
+        // accent, so fold it first: "Élves" finds "Elves" as well as the
+        // reverse. A query without accents passes through unchanged.
+        let pattern = Pattern::parse(&crate::text::fold(query), CaseMatching::Ignore, Normalization::Smart);
         let matches = pattern.match_list(self.entries.iter(), &mut matcher);
         // Typing a card's full name means *that* card, whatever the fuzzy score
         // says. "Explore" otherwise sits below "Explorer's Scope", "Explore the
         // Vastlands" and a dozen other names that merely contain the word — and
         // with a small `limit` it can fall off the end entirely, making the card
         // unreachable. Promote before truncating, for that reason.
-        let needle = query.trim().to_lowercase();
+        // Folded, so typing an accented name without its accents still counts.
+        let needle = crate::text::fold(query.trim());
         let (mut ranked, rest): (Vec<_>, Vec<_>) = matches
             .into_iter()
-            .partition(|(e, _)| e.card.name.to_lowercase() == needle);
+            .partition(|(e, _)| crate::text::fold(&e.card.name) == needle);
         ranked.extend(rest);
         ranked.truncate(limit);
         ranked.into_iter().map(|(e, _score)| e.card.clone()).collect()
@@ -174,6 +178,25 @@ mod tests {
         assert_eq!(hits[0].name, "Explore");
     }
 
+    #[test]
+    fn accents_dont_matter_in_either_direction() {
+        let fx = test_support::make();
+        fx.catalog
+            .conn
+            .execute(
+                "UPDATE ref.oracle_cards SET name = 'Lim-Dûl''s Cohort', name_lower = 'lim-dûl''s cohort'
+                  WHERE oracle_id = 'serra-angel'",
+                [],
+            )
+            .unwrap();
+        let idx = fx.catalog.build_name_index().unwrap();
+
+        assert_eq!(idx.search("lim-dul", 5)[0].name, "Lim-Dûl's Cohort");
+        // Typed in full without the accent, it's still the exact match.
+        assert_eq!(idx.search("lim-dul's cohort", 1)[0].name, "Lim-Dûl's Cohort");
+        assert_eq!(idx.search("Llanowar Élves", 5)[0].name, "Llanowar Elves");
+    }
+
     #[test]
     fn prefix_and_limit() {
         let fx = test_support::make();

crates/mtgc-core/src/text.rs

@@ -0,0 +1,42 @@
+//! Accent- and case-insensitive text matching.
+
+/// Lowercase `s` and strip accents ("Lim-Dûl's Cohort" → "lim-dul's cohort"),
+/// so typed text matches card names however they're accented. Uses the same
+/// per-character table as the fuzzy name search's own normalization, so both
+/// search paths agree on what counts as a match.
+pub fn fold(s: &str) -> String {
+    s.chars()
+        .map(|c| nucleo::chars::normalize(nucleo::chars::to_lower_case(c)))
+        .collect()
+}
+
+/// Register [`fold`] as the SQL function `fold(text)` on `conn`, so queries
+/// can compare `fold(oc.name)` against a folded pattern.
+pub(crate) fn register_sql_fold(conn: &rusqlite::Connection) -> rusqlite::Result<()> {
+    use rusqlite::functions::FunctionFlags;
+    conn.create_scalar_function(
+        "fold",
+        1,
+        FunctionFlags::SQLITE_UTF8 | FunctionFlags::SQLITE_DETERMINISTIC,
+        |ctx| {
+            Ok(match ctx.get::<Option<String>>(0)? {
+                Some(s) => Some(fold(&s)),
+                None => None,
+            })
+        },
+    )
+}
+
+#[cfg(test)]
+mod tests {
+    use super::fold;
+
+    #[test]
+    fn folds_case_and_accents_and_leaves_ascii_alone() {
+        assert_eq!(fold("Lim-Dûl's Cohort"), "lim-dul's cohort");
+        assert_eq!(fold("Éowyn, Lady of Rohan"), "eowyn, lady of rohan");
+        assert_eq!(fold("Hamato Ninpō"), "hamato ninpo");
+        assert_eq!(fold("Jakub Šlemr"), "jakub slemr");
+        assert_eq!(fold("Lightning Bolt"), "lightning bolt");
+    }
+}

src/components.tsx

@@ -632,6 +632,12 @@ export const CardSearchBox = forwardRef<
   );
 });
 
+/** Lowercase and strip accents, so "khazad-dum" matches "Khazad-dûm". The
+ *  frontend twin of `mtgc_core::text::fold`, for matching done client-side. */
+export function fold(s: string): string {
+  return s.normalize("NFD").replace(/\p{M}/gu, "").toLowerCase();
+}
+
 /**
  * A chip-based multi-value text filter with autocomplete: selected values
  * render as removable chips, and typing filters `suggestions` into a
@@ -653,11 +659,11 @@ export function TagInput({
   const [text, setText] = useState("");
   const [sel, setSel] = useState(0);
 
-  const query = text.trim().toLowerCase();
+  const query = fold(text.trim());
   const filtered = query
     ? suggestions
         .filter((s) => !values.some((v) => v.toLowerCase() === s.toLowerCase()))
-        .filter((s) => s.toLowerCase().includes(query))
+        .filter((s) => fold(s).includes(query))
         .slice(0, 8)
     : [];
 

src/views/DeckView.tsx

@@ -37,6 +37,7 @@ import {
   usePersistedState,
   useTileColumns,
   useViewMode,
+  fold,
 } from "../components";
 import { SuggestionsModal } from "../SuggestionsModal";
 import { useBackButton } from "../backButton";
@@ -878,9 +879,9 @@ export function DeckView({ refreshSignal }: { refreshSignal?: number }) {
   // filter only, distinct from the CardSearchBox boxes below (which search
   // the library/reference db to find NEW cards to add).
   function matchesFilter(e: DeckEntry): boolean {
-    const q = filterQuery.trim().toLowerCase();
+    const q = fold(filterQuery.trim());
     if (!q) return true;
-    return e.name.toLowerCase().includes(q) || e.type_line.toLowerCase().includes(q);
+    return fold(e.name).includes(q) || fold(e.type_line).includes(q);
   }
 
   const owned = contents.filter((e) => !e.is_maybeboard && e.quantity_owned > 0 && matchesFilter(e));