diff --git a/CHANGELOG.md b/CHANGELOG.md index a952e0b..aff8a63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Queries now ignore diacritics, e.g. `z cafe` matches `/Café`. + ### Fixed - Bash/Zsh: fix `z` failing on Cygwin/MSYS2 due to `cygpath` being passed a bad string. diff --git a/Cargo.lock b/Cargo.lock index 16edf6e..d39a0c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -475,6 +475,16 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "nucleo-matcher" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf33f538733d1a5a3494b836ba913207f14d9d4a1d3cd67030c5061bdd2cac85" +dependencies = [ + "memchr", + "unicode-segmentation", +] + [[package]] name = "num-conv" version = "0.2.1" @@ -906,6 +916,12 @@ version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + [[package]] name = "unicode-xid" version = "0.2.6" @@ -1167,6 +1183,7 @@ dependencies = [ "dunce", "fastrand", "glob", + "nucleo-matcher", "ouroboros", "rstest", "rstest_reuse", diff --git a/Cargo.toml b/Cargo.toml index bfcf408..c6b745d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,7 @@ dirs = "6.0.0" dunce = "1.0.1" fastrand = "2.0.0" glob = "0.3.0" +nucleo-matcher = "0.3.1" ouroboros = "0.18.3" serde = { version = "1.0.116", features = ["derive"] } time = { version = "0.3.47", default-features = false, features = ["parsing", "macros", "std"] } diff --git a/src/db/stream.rs b/src/db/stream.rs index 24c84e0..57643c9 100644 --- a/src/db/stream.rs +++ b/src/db/stream.rs @@ -83,7 +83,7 @@ impl<'a> Stream<'a> { None => return true, }; - let path = util::to_lowercase(path); + let path = util::normalize(path); let mut path = path.as_str(); match path.rfind(keywords_last) { Some(idx) => { @@ -149,7 +149,7 @@ impl StreamOptions { I: IntoIterator, I::Item: AsRef, { - self.keywords = keywords.into_iter().map(util::to_lowercase).collect(); + self.keywords = keywords.into_iter().map(util::normalize).collect(); self } @@ -185,6 +185,26 @@ mod tests { #[rstest] // Case normalization #[case(&["fOo", "bAr"], "/foo/bar", true)] + // Diacritics are folded on both sides + #[case(&["malaga"], "/Málaga", true)] + #[case(&["málaga"], "/malaga", true)] + #[case(&["munchen"], "/München", true)] + #[case(&["lodz"], "/łódź", true)] + // Ligatures are expanded + #[case(&["aether"], "/Æther", true)] + #[case(&["oeuvre"], "/Œuvre", true)] + #[case(&["strasse"], "/Straße", true)] + // Combining diacritical marks are removed + #[case(&["cafe"], "/cafe\u{0301}", true)] + #[case(&["café"], "/cafe", true)] + #[case(&["malaga"], "/Ma\u{0301}laga", true)] + // Characters outside the folding range are only case-normalized (not + // mis-folded) + #[case(&["ṁ"], "/Ṁ", true)] + #[case(&["m"], "/Ṁ", false)] + // Non-Latin scripts are only case-normalized + #[case(&["папка"], "/ПАПКА", true)] + #[case(&["市场"], "/市场", true)] // Last component #[case(&["ba"], "/foo/bar", true)] #[case(&["fo"], "/foo/bar", false)] diff --git a/src/util.rs b/src/util.rs index 4c0a27c..71b7212 100644 --- a/src/util.rs +++ b/src/util.rs @@ -373,8 +373,39 @@ pub fn resolve_path(path: impl AsRef) -> Result { Ok(stack.iter().collect()) } -/// Convert a string to lowercase, with a fast path for ASCII strings. -pub fn to_lowercase(s: impl AsRef) -> String { +/// Normalizes a string for matching, with a fast path for ASCII strings: +/// Latin characters which are variants of ASCII characters (e.g. `á`, `ø`) +/// are folded to their ASCII equivalent, ligatures are expanded (e.g. `æ` to +/// `ae`), combining diacritical marks are removed (macOS stores filenames in +/// decomposed form, where `á` is an `a` followed by U+0301), and the string +/// is converted to lowercase. +pub fn normalize(s: impl AsRef) -> String { let s = s.as_ref(); - if s.is_ascii() { s.to_ascii_lowercase() } else { s.to_lowercase() } + if s.is_ascii() { + return s.to_ascii_lowercase(); + } + + let mut result = String::with_capacity(s.len()); + for c in s.chars() { + match c { + // Combining marks are not handled by nucleo-matcher, and must be + // removed separately. + '\u{0300}'..='\u{036f}' | '\u{1ab0}'..='\u{1aff}' | '\u{1dc0}'..='\u{1dff}' => {} + // Ligatures expand to multiple characters, which nucleo-matcher's + // char-to-char normalization cannot represent. + 'æ' | 'Æ' => result.push_str("ae"), + 'œ' | 'Œ' => result.push_str("oe"), + 'ß' | 'ẞ' => result.push_str("ss"), + // nucleo-matcher 0.3.1 only folds the Latin-1 Supplement, Latin + // Extended-A, and Latin Extended-B blocks correctly. Its Latin + // Extended Additional and super/subscript tables are corrupt due to + // a codegen bug and return arbitrary letters, so we restrict folding + // to the correct range and leave everything else untouched. + '\u{00c0}'..='\u{024f}' => { + result.extend(nucleo_matcher::chars::normalize(c).to_lowercase()); + } + _ => result.extend(c.to_lowercase()), + } + } + result }