diff --git a/Cargo.lock b/Cargo.lock index 27da609..355185d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,18 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.3" @@ -17,6 +29,12 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "anstream" version = "0.6.18" @@ -126,15 +144,6 @@ dependencies = [ "wait-timeout", ] -[[package]] -name = "bincode" -version = "1.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" -dependencies = [ - "serde", -] - [[package]] name = "bitflags" version = "2.9.1" @@ -321,6 +330,18 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "fallible-iterator" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" + +[[package]] +name = "fallible-streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" + [[package]] name = "fastrand" version = "2.3.0" @@ -356,6 +377,25 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +dependencies = [ + "ahash", + "allocator-api2", +] + +[[package]] +name = "hashlink" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7" +dependencies = [ + "hashbrown", +] + [[package]] name = "heck" version = "0.4.1" @@ -396,6 +436,16 @@ dependencies = [ "libc", ] +[[package]] +name = "libsqlite3-sys" +version = "0.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716" +dependencies = [ + "pkg-config", + "vcpkg", +] + [[package]] name = "linux-raw-sys" version = "0.9.4" @@ -484,6 +534,12 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -670,6 +726,20 @@ dependencies = [ "syn", ] +[[package]] +name = "rusqlite" +version = "0.30.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a78046161564f5e7cd9008aff3b2990b3850dc8e0349119b98e8f251e099f24d" +dependencies = [ + "bitflags", + "fallible-iterator", + "fallible-streaming-iterator", + "hashlink", + "libsqlite3-sys", + "smallvec", +] + [[package]] name = "rustc-hash" version = "2.1.1" @@ -742,6 +812,12 @@ dependencies = [ "serde", ] +[[package]] +name = "smallvec" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" + [[package]] name = "static_assertions" version = "1.1.0" @@ -816,6 +892,12 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.5" @@ -988,7 +1070,6 @@ dependencies = [ "anyhow", "askama", "assert_cmd", - "bincode", "clap", "clap_complete", "clap_complete_fig", @@ -1002,6 +1083,7 @@ dependencies = [ "ouroboros", "rstest", "rstest_reuse", + "rusqlite", "serde", "tempfile", "which", diff --git a/Cargo.toml b/Cargo.toml index d137115..54bec44 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ askama = { version = "0.14.0", default-features = false, features = [ "derive", "std", ] } -bincode = "1.3.1" +rusqlite = "0.30.0" clap = { version = "4.3.0", features = ["derive"] } color-print = "0.3.4" dirs = "6.0.0" diff --git a/src/db/mod.rs b/src/db/mod.rs index 1856fda..e0d31a6 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -1,24 +1,19 @@ mod dir; mod stream; +use std::fs; use std::path::{Path, PathBuf}; -use std::{fs, io}; -use anyhow::{Context, Result, bail}; -use bincode::Options; -use ouroboros::self_referencing; +use anyhow::{Context, Result}; +use rusqlite::{Connection, OptionalExtension, params}; +use crate::config; pub use crate::db::dir::{Dir, Epoch, Rank}; pub use crate::db::stream::{Stream, StreamOptions}; -use crate::{config, util}; -#[self_referencing] pub struct Database { path: PathBuf, - bytes: Vec, - #[borrows(bytes)] - #[covariant] - pub dirs: Vec>, + conn: Connection, dirty: bool, } @@ -32,47 +27,73 @@ impl Database { pub fn open_dir(data_dir: impl AsRef) -> Result { let data_dir = data_dir.as_ref(); - let path = data_dir.join("db.zo"); + let path = data_dir.join("db.sqlite3"); let path = fs::canonicalize(&path).unwrap_or(path); - match fs::read(&path) { - Ok(bytes) => Self::try_new(path, bytes, |bytes| Self::deserialize(bytes), false), - Err(e) if e.kind() == io::ErrorKind::NotFound => { - // Create data directory, but don't create any file yet. The file will be - // created later by [`Database::save`] if any data is modified. - fs::create_dir_all(data_dir).with_context(|| { - format!("unable to create data directory: {}", data_dir.display()) - })?; - Ok(Self::new(path, Vec::new(), |_| Vec::new(), false)) - } - Err(e) => { - Err(e).with_context(|| format!("could not read from database: {}", path.display())) - } - } + fs::create_dir_all(data_dir) + .with_context(|| format!("unable to create data directory: {}", data_dir.display()))?; + + // Open or create sqlite database file. + let conn = Connection::open(&path) + .with_context(|| format!("could not open database: {}", path.display()))?; + + // Enable WAL for better concurrency and durability. + conn.pragma_update(None, "journal_mode", &"WAL").ok(); + + // Create table if it doesn't exist. + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS dirs ( + path TEXT PRIMARY KEY, + rank REAL NOT NULL, + last_accessed INTEGER NOT NULL + );", + )?; + + Ok(Database { path, conn, dirty: false }) } pub fn save(&mut self) -> Result<()> { - // Only write to disk if the database is modified. - if !self.dirty() { - return Ok(()); - } - - let bytes = Self::serialize(self.dirs())?; - util::write(self.borrow_path(), bytes).context("could not write to database")?; - self.with_dirty_mut(|dirty| *dirty = false); - + // For SQLite, write operations are applied immediately via transactions. + // Keep save() for compatibility; do nothing. + self.dirty = false; Ok(()) } /// Increments the rank of a directory, or creates it if it does not exist. pub fn add(&mut self, path: impl AsRef + Into, by: Rank, now: Epoch) { - self.with_dirs_mut(|dirs| match dirs.iter_mut().find(|dir| dir.path == path.as_ref()) { - Some(dir) => dir.rank = (dir.rank + by).max(0.0), - None => { - dirs.push(Dir { path: path.into().into(), rank: by.max(0.0), last_accessed: now }) + let path_s: String = path.into(); + let tx = match self.conn.transaction() { + Ok(t) => t, + Err(_) => return, + }; + + let existing: Option<(f64, u64)> = tx + .query_row( + "SELECT rank, last_accessed FROM dirs WHERE path = ?1", + params![&path_s], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional() + .unwrap_or(None); + + match existing { + Some((rank, _last)) => { + let new_rank = (rank + by).max(0.0); + let _ = tx.execute( + "UPDATE dirs SET rank = ?1 WHERE path = ?2", + params![new_rank, &path_s], + ); } - }); - self.with_dirty_mut(|dirty| *dirty = true); + None => { + let _ = tx.execute( + "INSERT INTO dirs (path, rank, last_accessed) VALUES (?1, ?2, ?3)", + params![&path_s, by.max(0.0), now], + ); + } + } + + let _ = tx.commit(); + self.dirty = true; } /// Creates a new directory. This will create a duplicate entry if this @@ -80,155 +101,157 @@ impl Database { /// either does a check before calling this, or calls `dedup()` /// afterward. pub fn add_unchecked(&mut self, path: impl AsRef + Into, rank: Rank, now: Epoch) { - self.with_dirs_mut(|dirs| { - dirs.push(Dir { path: path.into().into(), rank, last_accessed: now }) - }); - self.with_dirty_mut(|dirty| *dirty = true); + let path_s: String = path.into(); + let _ = self.conn.execute( + "INSERT OR REPLACE INTO dirs (path, rank, last_accessed) VALUES (?1, ?2, ?3)", + params![&path_s, rank, now], + ); + self.dirty = true; } /// Increments the rank and updates the last_accessed of a directory, or /// creates it if it does not exist. pub fn add_update(&mut self, path: impl AsRef + Into, by: Rank, now: Epoch) { - self.with_dirs_mut(|dirs| match dirs.iter_mut().find(|dir| dir.path == path.as_ref()) { - Some(dir) => { - dir.rank = (dir.rank + by).max(0.0); - dir.last_accessed = now; + let path_s: String = path.into(); + let tx = match self.conn.transaction() { + Ok(t) => t, + Err(_) => return, + }; + + let existing: Option<(f64, u64)> = tx + .query_row( + "SELECT rank, last_accessed FROM dirs WHERE path = ?1", + params![&path_s], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional() + .unwrap_or(None); + + match existing { + Some((rank, _)) => { + let new_rank = (rank + by).max(0.0); + let _ = tx.execute( + "UPDATE dirs SET rank = ?1, last_accessed = ?2 WHERE path = ?3", + params![new_rank, now, &path_s], + ); } None => { - dirs.push(Dir { path: path.into().into(), rank: by.max(0.0), last_accessed: now }) + let _ = tx.execute( + "INSERT INTO dirs (path, rank, last_accessed) VALUES (?1, ?2, ?3)", + params![&path_s, by.max(0.0), now], + ); } - }); - self.with_dirty_mut(|dirty| *dirty = true); + } + + let _ = tx.commit(); + self.dirty = true; } - /// Removes the directory with `path` from the store. This does not preserve - /// ordering, but is O(1). + /// Removes the directory with `path` from the store. Returns true if an + /// entry was deleted. pub fn remove(&mut self, path: impl AsRef) -> bool { - match self.dirs().iter().position(|dir| dir.path == path.as_ref()) { - Some(idx) => { - self.swap_remove(idx); - true + let path_s = path.as_ref(); + match self.conn.execute("DELETE FROM dirs WHERE path = ?1", params![path_s]) { + Ok(count) => { + if count > 0 { + self.dirty = true; + true + } else { + false + } } - None => false, + Err(_) => false, } } - pub fn swap_remove(&mut self, idx: usize) { - self.with_dirs_mut(|dirs| dirs.swap_remove(idx)); - self.with_dirty_mut(|dirty| *dirty = true); + pub fn swap_remove(&mut self, _idx: usize) { + // In the sqlite-backed implementation we don't maintain an in-memory + // vector, so this is a no-op. Higher-level code that relies on + // indices shouldn't be calling this directly except within the + // streaming logic which uses Database::dirs(). For compatibility, keep + // the method but do nothing. + self.dirty = true; } pub fn age(&mut self, max_age: Rank) { - let mut dirty = false; - self.with_dirs_mut(|dirs| { - let total_age = dirs.iter().map(|dir| dir.rank).sum::(); - if total_age > max_age { - let factor = 0.9 * max_age / total_age; - for idx in (0..dirs.len()).rev() { - let dir = &mut dirs[idx]; - dir.rank *= factor; - if dir.rank < 1.0 { - dirs.swap_remove(idx); + // Apply the aging algorithm to all rows. + // Collect entries first to avoid holding a Statement borrow while starting + // a transaction on the connection. + let mut entries = Vec::new(); + if let Ok(mut stmt) = self.conn.prepare("SELECT path, rank FROM dirs") { + if let Ok(rows) = + stmt.query_map([], |row| Ok((row.get::<_, String>(0)?, row.get::<_, f64>(1)?))) + { + for r in rows { + if let Ok((path, rank)) = r { + entries.push((path, rank)); } } - dirty = true; } - }); - self.with_dirty_mut(|dirty_prev| *dirty_prev |= dirty); + } + + let total_age: f64 = entries.iter().map(|(_, rank)| *rank).sum(); + if total_age > max_age { + let factor = 0.9 * max_age / total_age; + if let Ok(tx) = self.conn.transaction() { + for (path, rank) in entries { + let new_rank = rank * factor; + if new_rank < 1.0 { + let _ = tx.execute("DELETE FROM dirs WHERE path = ?1", params![path]); + } else { + let _ = tx.execute( + "UPDATE dirs SET rank = ?1 WHERE path = ?2", + params![new_rank, path], + ); + } + } + let _ = tx.commit(); + self.dirty = true; + } + } } pub fn dedup(&mut self) { - // Sort by path, so that equal paths are next to each other. - self.sort_by_path(); - - let mut dirty = false; - self.with_dirs_mut(|dirs| { - for idx in (1..dirs.len()).rev() { - // Check if curr_dir and next_dir have equal paths. - let curr_dir = &dirs[idx]; - let next_dir = &dirs[idx - 1]; - if next_dir.path != curr_dir.path { - continue; - } - - // Merge curr_dir's rank and last_accessed into next_dir. - let rank = curr_dir.rank; - let last_accessed = curr_dir.last_accessed; - let next_dir = &mut dirs[idx - 1]; - next_dir.last_accessed = next_dir.last_accessed.max(last_accessed); - next_dir.rank += rank; - - // Delete curr_dir. - dirs.swap_remove(idx); - dirty = true; - } - }); - self.with_dirty_mut(|dirty_prev| *dirty_prev |= dirty); + // Using path as PRIMARY KEY ensures uniqueness, nothing to do here. } pub fn sort_by_path(&mut self) { - self.with_dirs_mut(|dirs| dirs.sort_unstable_by(|dir1, dir2| dir1.path.cmp(&dir2.path))); - self.with_dirty_mut(|dirty| *dirty = true); + // Sorting is done at query time in the sqlite-backed implementation. } - pub fn sort_by_score(&mut self, now: Epoch) { - self.with_dirs_mut(|dirs| { - dirs.sort_unstable_by(|dir1: &Dir, dir2: &Dir| { - dir1.score(now).total_cmp(&dir2.score(now)) - }) - }); - self.with_dirty_mut(|dirty| *dirty = true); + pub fn sort_by_score(&mut self, _now: Epoch) { + // Sorting is done at query time in the sqlite-backed implementation. } pub fn dirty(&self) -> bool { - *self.borrow_dirty() + self.dirty } - pub fn dirs(&self) -> &[Dir<'_>] { - self.borrow_dirs() - } - - fn serialize(dirs: &[Dir<'_>]) -> Result> { - (|| -> bincode::Result<_> { - // Preallocate buffer with combined size of sections. - let buffer_size = - bincode::serialized_size(&Self::VERSION)? + bincode::serialized_size(&dirs)?; - let mut buffer = Vec::with_capacity(buffer_size as usize); - - // Serialize sections into buffer. - bincode::serialize_into(&mut buffer, &Self::VERSION)?; - bincode::serialize_into(&mut buffer, &dirs)?; - - Ok(buffer) - })() - .context("could not serialize database") - } - - fn deserialize(bytes: &[u8]) -> Result>> { - // Assume a maximum size for the database. This prevents bincode from throwing - // strange errors when it encounters invalid data. - const MAX_SIZE: u64 = 32 << 20; // 32 MiB - let deserializer = &mut bincode::options().with_fixint_encoding().with_limit(MAX_SIZE); - - // Split bytes into sections. - let version_size = deserializer.serialized_size(&Self::VERSION).unwrap() as _; - if bytes.len() < version_size { - bail!("could not deserialize database: corrupted data"); - } - let (bytes_version, bytes_dirs) = bytes.split_at(version_size); - - // Deserialize sections. - let version = deserializer.deserialize(bytes_version)?; - let dirs = match version { - Self::VERSION => { - deserializer.deserialize(bytes_dirs).context("could not deserialize database")? - } - version => { - bail!("unsupported version (got {version}, supports {})", Self::VERSION) - } + pub fn dirs(&self) -> Vec> { + // Load all dirs from the database into an owned Vec. + let mut stmt = match self.conn.prepare("SELECT path, rank, last_accessed FROM dirs") { + Ok(s) => s, + Err(_) => return Vec::new(), }; - Ok(dirs) + let rows = stmt.query_map([], |row| { + Ok(Dir { + path: row.get::<_, String>(0)?.into(), + rank: row.get::<_, f64>(1)?, + last_accessed: row.get::<_, u64>(2)?, + }) + }); + + let mut out = Vec::new(); + if let Ok(map) = rows { + for r in map { + if let Ok(dir) = r { + out.push(dir); + } + } + } + + out } } @@ -253,7 +276,8 @@ mod tests { let db = Database::open_dir(data_dir.path()).unwrap(); assert_eq!(db.dirs().len(), 1); - let dir = &db.dirs()[0]; + let dirs = db.dirs(); + let dir = &dirs[0]; assert_eq!(dir.path, path); assert!((dir.rank - 2.0).abs() < 0.01); assert_eq!(dir.last_accessed, now); diff --git a/src/db/stream.rs b/src/db/stream.rs index 24c84e0..3d48bef 100644 --- a/src/db/stream.rs +++ b/src/db/stream.rs @@ -1,5 +1,3 @@ -use std::iter::Rev; -use std::ops::Range; use std::path::Path; use std::{fs, path}; @@ -10,20 +8,25 @@ use crate::util::{self, MONTH}; pub struct Stream<'a> { db: &'a mut Database, - idxs: Rev>, + entries: Vec>, + pos: usize, options: StreamOptions, } impl<'a> Stream<'a> { pub fn new(db: &'a mut Database, options: StreamOptions) -> Self { - db.sort_by_score(options.now); - let idxs = (0..db.dirs().len()).rev(); - Stream { db, idxs, options } + // Load entries and sort by score. + let mut entries = db.dirs(); + entries.sort_unstable_by(|a, b| a.score(options.now).total_cmp(&b.score(options.now))); + // iterate from highest to lowest + entries.reverse(); + Stream { db, entries, pos: 0, options } } pub fn next(&mut self) -> Option<&Dir<'_>> { - while let Some(idx) = self.idxs.next() { - let dir = &self.db.dirs()[idx]; + while self.pos < self.entries.len() { + let dir = &self.entries[self.pos]; + self.pos += 1; if !self.filter_by_keywords(&dir.path) { continue; @@ -34,20 +37,20 @@ impl<'a> Stream<'a> { } if !self.filter_by_exclude(&dir.path) { - self.db.swap_remove(idx); + // lazily remove from database + let _ = self.db.remove(&*dir.path); continue; } // Exists queries are slow, this should always be checked last. if !self.filter_by_exists(&dir.path) { if dir.last_accessed < self.options.ttl { - self.db.swap_remove(idx); + let _ = self.db.remove(&*dir.path); } continue; } - let dir = &self.db.dirs()[idx]; - return Some(dir); + return Some(&self.entries[self.pos - 1]); } None @@ -203,9 +206,10 @@ mod tests { #[case(&["/foo/", "/bar"], "/foo/bar", false)] #[case(&["/foo/", "/bar"], "/foo/baz/bar", true)] fn query(#[case] keywords: &[&str], #[case] path: &str, #[case] is_match: bool) { - let db = &mut Database::new(PathBuf::new(), Vec::new(), |_| Vec::new(), false); + let data_dir = tempfile::tempdir().unwrap(); + let mut db = Database::open_dir(data_dir.path()).unwrap(); let options = StreamOptions::new(0).with_keywords(keywords.iter()); - let stream = Stream::new(db, options); + let stream = Stream::new(&mut db, options); assert_eq!(is_match, stream.filter_by_keywords(path)); } } diff --git a/src/util.rs b/src/util.rs index 996f61d..5f6161c 100644 --- a/src/util.rs +++ b/src/util.rs @@ -1,5 +1,4 @@ use std::ffi::OsStr; -use std::fs::{self, File, OpenOptions}; use std::io::{self, Read, Write}; use std::path::{Component, Path, PathBuf}; use std::process::{Child, Command, Stdio};