wip
This commit is contained in:
parent
f3a07cd3d1
commit
0dc4a72500
|
|
@ -2,6 +2,18 @@
|
||||||
# It is not intended for manual editing.
|
# It is not intended for manual editing.
|
||||||
version = 4
|
version = 4
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "ahash"
|
||||||
|
version = "0.8.12"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
|
||||||
|
dependencies = [
|
||||||
|
"cfg-if",
|
||||||
|
"once_cell",
|
||||||
|
"version_check",
|
||||||
|
"zerocopy",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "aho-corasick"
|
name = "aho-corasick"
|
||||||
version = "1.1.3"
|
version = "1.1.3"
|
||||||
|
|
@ -17,6 +29,12 @@ version = "0.1.3"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd"
|
checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "allocator-api2"
|
||||||
|
version = "0.2.21"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "anstream"
|
name = "anstream"
|
||||||
version = "0.6.18"
|
version = "0.6.18"
|
||||||
|
|
@ -126,15 +144,6 @@ dependencies = [
|
||||||
"wait-timeout",
|
"wait-timeout",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
|
||||||
name = "bincode"
|
|
||||||
version = "1.3.3"
|
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
||||||
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
|
|
||||||
dependencies = [
|
|
||||||
"serde",
|
|
||||||
]
|
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "bitflags"
|
name = "bitflags"
|
||||||
version = "2.9.1"
|
version = "2.9.1"
|
||||||
|
|
@ -321,6 +330,18 @@ dependencies = [
|
||||||
"windows-sys",
|
"windows-sys",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fallible-iterator"
|
||||||
|
version = "0.3.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "fallible-streaming-iterator"
|
||||||
|
version = "0.1.9"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "fastrand"
|
name = "fastrand"
|
||||||
version = "2.3.0"
|
version = "2.3.0"
|
||||||
|
|
@ -356,6 +377,25 @@ version = "0.3.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2"
|
checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "hashbrown"
|
||||||
|
version = "0.14.5"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
|
||||||
|
dependencies = [
|
||||||
|
"ahash",
|
||||||
|
"allocator-api2",
|
||||||
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "hashlink"
|
||||||
|
version = "0.8.4"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7"
|
||||||
|
dependencies = [
|
||||||
|
"hashbrown",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "heck"
|
name = "heck"
|
||||||
version = "0.4.1"
|
version = "0.4.1"
|
||||||
|
|
@ -396,6 +436,16 @@ dependencies = [
|
||||||
"libc",
|
"libc",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "libsqlite3-sys"
|
||||||
|
version = "0.27.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716"
|
||||||
|
dependencies = [
|
||||||
|
"pkg-config",
|
||||||
|
"vcpkg",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "linux-raw-sys"
|
name = "linux-raw-sys"
|
||||||
version = "0.9.4"
|
version = "0.9.4"
|
||||||
|
|
@ -484,6 +534,12 @@ version = "2.3.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
|
checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "pkg-config"
|
||||||
|
version = "0.3.32"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "ppv-lite86"
|
name = "ppv-lite86"
|
||||||
version = "0.2.21"
|
version = "0.2.21"
|
||||||
|
|
@ -670,6 +726,20 @@ dependencies = [
|
||||||
"syn",
|
"syn",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "rusqlite"
|
||||||
|
version = "0.30.0"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "a78046161564f5e7cd9008aff3b2990b3850dc8e0349119b98e8f251e099f24d"
|
||||||
|
dependencies = [
|
||||||
|
"bitflags",
|
||||||
|
"fallible-iterator",
|
||||||
|
"fallible-streaming-iterator",
|
||||||
|
"hashlink",
|
||||||
|
"libsqlite3-sys",
|
||||||
|
"smallvec",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "rustc-hash"
|
name = "rustc-hash"
|
||||||
version = "2.1.1"
|
version = "2.1.1"
|
||||||
|
|
@ -742,6 +812,12 @@ dependencies = [
|
||||||
"serde",
|
"serde",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "smallvec"
|
||||||
|
version = "1.15.1"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "static_assertions"
|
name = "static_assertions"
|
||||||
version = "1.1.0"
|
version = "1.1.0"
|
||||||
|
|
@ -816,6 +892,12 @@ version = "0.2.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "vcpkg"
|
||||||
|
version = "0.2.15"
|
||||||
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
|
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "version_check"
|
name = "version_check"
|
||||||
version = "0.9.5"
|
version = "0.9.5"
|
||||||
|
|
@ -988,7 +1070,6 @@ dependencies = [
|
||||||
"anyhow",
|
"anyhow",
|
||||||
"askama",
|
"askama",
|
||||||
"assert_cmd",
|
"assert_cmd",
|
||||||
"bincode",
|
|
||||||
"clap",
|
"clap",
|
||||||
"clap_complete",
|
"clap_complete",
|
||||||
"clap_complete_fig",
|
"clap_complete_fig",
|
||||||
|
|
@ -1002,6 +1083,7 @@ dependencies = [
|
||||||
"ouroboros",
|
"ouroboros",
|
||||||
"rstest",
|
"rstest",
|
||||||
"rstest_reuse",
|
"rstest_reuse",
|
||||||
|
"rusqlite",
|
||||||
"serde",
|
"serde",
|
||||||
"tempfile",
|
"tempfile",
|
||||||
"which",
|
"which",
|
||||||
|
|
|
||||||
|
|
@ -21,7 +21,7 @@ askama = { version = "0.14.0", default-features = false, features = [
|
||||||
"derive",
|
"derive",
|
||||||
"std",
|
"std",
|
||||||
] }
|
] }
|
||||||
bincode = "1.3.1"
|
rusqlite = "0.30.0"
|
||||||
clap = { version = "4.3.0", features = ["derive"] }
|
clap = { version = "4.3.0", features = ["derive"] }
|
||||||
color-print = "0.3.4"
|
color-print = "0.3.4"
|
||||||
dirs = "6.0.0"
|
dirs = "6.0.0"
|
||||||
|
|
|
||||||
332
src/db/mod.rs
332
src/db/mod.rs
|
|
@ -1,24 +1,19 @@
|
||||||
mod dir;
|
mod dir;
|
||||||
mod stream;
|
mod stream;
|
||||||
|
|
||||||
|
use std::fs;
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
use std::{fs, io};
|
|
||||||
|
|
||||||
use anyhow::{Context, Result, bail};
|
use anyhow::{Context, Result};
|
||||||
use bincode::Options;
|
use rusqlite::{Connection, OptionalExtension, params};
|
||||||
use ouroboros::self_referencing;
|
|
||||||
|
|
||||||
|
use crate::config;
|
||||||
pub use crate::db::dir::{Dir, Epoch, Rank};
|
pub use crate::db::dir::{Dir, Epoch, Rank};
|
||||||
pub use crate::db::stream::{Stream, StreamOptions};
|
pub use crate::db::stream::{Stream, StreamOptions};
|
||||||
use crate::{config, util};
|
|
||||||
|
|
||||||
#[self_referencing]
|
|
||||||
pub struct Database {
|
pub struct Database {
|
||||||
path: PathBuf,
|
path: PathBuf,
|
||||||
bytes: Vec<u8>,
|
conn: Connection,
|
||||||
#[borrows(bytes)]
|
|
||||||
#[covariant]
|
|
||||||
pub dirs: Vec<Dir<'this>>,
|
|
||||||
dirty: bool,
|
dirty: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -32,47 +27,73 @@ impl Database {
|
||||||
|
|
||||||
pub fn open_dir(data_dir: impl AsRef<Path>) -> Result<Self> {
|
pub fn open_dir(data_dir: impl AsRef<Path>) -> Result<Self> {
|
||||||
let data_dir = data_dir.as_ref();
|
let data_dir = data_dir.as_ref();
|
||||||
let path = data_dir.join("db.zo");
|
let path = data_dir.join("db.sqlite3");
|
||||||
let path = fs::canonicalize(&path).unwrap_or(path);
|
let path = fs::canonicalize(&path).unwrap_or(path);
|
||||||
|
|
||||||
match fs::read(&path) {
|
fs::create_dir_all(data_dir)
|
||||||
Ok(bytes) => Self::try_new(path, bytes, |bytes| Self::deserialize(bytes), false),
|
.with_context(|| format!("unable to create data directory: {}", data_dir.display()))?;
|
||||||
Err(e) if e.kind() == io::ErrorKind::NotFound => {
|
|
||||||
// Create data directory, but don't create any file yet. The file will be
|
// Open or create sqlite database file.
|
||||||
// created later by [`Database::save`] if any data is modified.
|
let conn = Connection::open(&path)
|
||||||
fs::create_dir_all(data_dir).with_context(|| {
|
.with_context(|| format!("could not open database: {}", path.display()))?;
|
||||||
format!("unable to create data directory: {}", data_dir.display())
|
|
||||||
})?;
|
// Enable WAL for better concurrency and durability.
|
||||||
Ok(Self::new(path, Vec::new(), |_| Vec::new(), false))
|
conn.pragma_update(None, "journal_mode", &"WAL").ok();
|
||||||
}
|
|
||||||
Err(e) => {
|
// Create table if it doesn't exist.
|
||||||
Err(e).with_context(|| format!("could not read from database: {}", path.display()))
|
conn.execute_batch(
|
||||||
}
|
"CREATE TABLE IF NOT EXISTS dirs (
|
||||||
}
|
path TEXT PRIMARY KEY,
|
||||||
|
rank REAL NOT NULL,
|
||||||
|
last_accessed INTEGER NOT NULL
|
||||||
|
);",
|
||||||
|
)?;
|
||||||
|
|
||||||
|
Ok(Database { path, conn, dirty: false })
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn save(&mut self) -> Result<()> {
|
pub fn save(&mut self) -> Result<()> {
|
||||||
// Only write to disk if the database is modified.
|
// For SQLite, write operations are applied immediately via transactions.
|
||||||
if !self.dirty() {
|
// Keep save() for compatibility; do nothing.
|
||||||
return Ok(());
|
self.dirty = false;
|
||||||
}
|
|
||||||
|
|
||||||
let bytes = Self::serialize(self.dirs())?;
|
|
||||||
util::write(self.borrow_path(), bytes).context("could not write to database")?;
|
|
||||||
self.with_dirty_mut(|dirty| *dirty = false);
|
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Increments the rank of a directory, or creates it if it does not exist.
|
/// Increments the rank of a directory, or creates it if it does not exist.
|
||||||
pub fn add(&mut self, path: impl AsRef<str> + Into<String>, by: Rank, now: Epoch) {
|
pub fn add(&mut self, path: impl AsRef<str> + Into<String>, by: Rank, now: Epoch) {
|
||||||
self.with_dirs_mut(|dirs| match dirs.iter_mut().find(|dir| dir.path == path.as_ref()) {
|
let path_s: String = path.into();
|
||||||
Some(dir) => dir.rank = (dir.rank + by).max(0.0),
|
let tx = match self.conn.transaction() {
|
||||||
None => {
|
Ok(t) => t,
|
||||||
dirs.push(Dir { path: path.into().into(), rank: by.max(0.0), last_accessed: now })
|
Err(_) => return,
|
||||||
|
};
|
||||||
|
|
||||||
|
let existing: Option<(f64, u64)> = tx
|
||||||
|
.query_row(
|
||||||
|
"SELECT rank, last_accessed FROM dirs WHERE path = ?1",
|
||||||
|
params![&path_s],
|
||||||
|
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||||
|
)
|
||||||
|
.optional()
|
||||||
|
.unwrap_or(None);
|
||||||
|
|
||||||
|
match existing {
|
||||||
|
Some((rank, _last)) => {
|
||||||
|
let new_rank = (rank + by).max(0.0);
|
||||||
|
let _ = tx.execute(
|
||||||
|
"UPDATE dirs SET rank = ?1 WHERE path = ?2",
|
||||||
|
params![new_rank, &path_s],
|
||||||
|
);
|
||||||
}
|
}
|
||||||
});
|
None => {
|
||||||
self.with_dirty_mut(|dirty| *dirty = true);
|
let _ = tx.execute(
|
||||||
|
"INSERT INTO dirs (path, rank, last_accessed) VALUES (?1, ?2, ?3)",
|
||||||
|
params![&path_s, by.max(0.0), now],
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let _ = tx.commit();
|
||||||
|
self.dirty = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Creates a new directory. This will create a duplicate entry if this
|
/// Creates a new directory. This will create a duplicate entry if this
|
||||||
|
|
@ -80,155 +101,157 @@ impl Database {
|
||||||
/// either does a check before calling this, or calls `dedup()`
|
/// either does a check before calling this, or calls `dedup()`
|
||||||
/// afterward.
|
/// afterward.
|
||||||
pub fn add_unchecked(&mut self, path: impl AsRef<str> + Into<String>, rank: Rank, now: Epoch) {
|
pub fn add_unchecked(&mut self, path: impl AsRef<str> + Into<String>, rank: Rank, now: Epoch) {
|
||||||
self.with_dirs_mut(|dirs| {
|
let path_s: String = path.into();
|
||||||
dirs.push(Dir { path: path.into().into(), rank, last_accessed: now })
|
let _ = self.conn.execute(
|
||||||
});
|
"INSERT OR REPLACE INTO dirs (path, rank, last_accessed) VALUES (?1, ?2, ?3)",
|
||||||
self.with_dirty_mut(|dirty| *dirty = true);
|
params![&path_s, rank, now],
|
||||||
|
);
|
||||||
|
self.dirty = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Increments the rank and updates the last_accessed of a directory, or
|
/// Increments the rank and updates the last_accessed of a directory, or
|
||||||
/// creates it if it does not exist.
|
/// creates it if it does not exist.
|
||||||
pub fn add_update(&mut self, path: impl AsRef<str> + Into<String>, by: Rank, now: Epoch) {
|
pub fn add_update(&mut self, path: impl AsRef<str> + Into<String>, by: Rank, now: Epoch) {
|
||||||
self.with_dirs_mut(|dirs| match dirs.iter_mut().find(|dir| dir.path == path.as_ref()) {
|
let path_s: String = path.into();
|
||||||
Some(dir) => {
|
let tx = match self.conn.transaction() {
|
||||||
dir.rank = (dir.rank + by).max(0.0);
|
Ok(t) => t,
|
||||||
dir.last_accessed = now;
|
Err(_) => return,
|
||||||
|
};
|
||||||
|
|
||||||
|
let existing: Option<(f64, u64)> = tx
|
||||||
|
.query_row(
|
||||||
|
"SELECT rank, last_accessed FROM dirs WHERE path = ?1",
|
||||||
|
params![&path_s],
|
||||||
|
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||||
|
)
|
||||||
|
.optional()
|
||||||
|
.unwrap_or(None);
|
||||||
|
|
||||||
|
match existing {
|
||||||
|
Some((rank, _)) => {
|
||||||
|
let new_rank = (rank + by).max(0.0);
|
||||||
|
let _ = tx.execute(
|
||||||
|
"UPDATE dirs SET rank = ?1, last_accessed = ?2 WHERE path = ?3",
|
||||||
|
params![new_rank, now, &path_s],
|
||||||
|
);
|
||||||
}
|
}
|
||||||
None => {
|
None => {
|
||||||
dirs.push(Dir { path: path.into().into(), rank: by.max(0.0), last_accessed: now })
|
let _ = tx.execute(
|
||||||
|
"INSERT INTO dirs (path, rank, last_accessed) VALUES (?1, ?2, ?3)",
|
||||||
|
params![&path_s, by.max(0.0), now],
|
||||||
|
);
|
||||||
}
|
}
|
||||||
});
|
}
|
||||||
self.with_dirty_mut(|dirty| *dirty = true);
|
|
||||||
|
let _ = tx.commit();
|
||||||
|
self.dirty = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Removes the directory with `path` from the store. This does not preserve
|
/// Removes the directory with `path` from the store. Returns true if an
|
||||||
/// ordering, but is O(1).
|
/// entry was deleted.
|
||||||
pub fn remove(&mut self, path: impl AsRef<str>) -> bool {
|
pub fn remove(&mut self, path: impl AsRef<str>) -> bool {
|
||||||
match self.dirs().iter().position(|dir| dir.path == path.as_ref()) {
|
let path_s = path.as_ref();
|
||||||
Some(idx) => {
|
match self.conn.execute("DELETE FROM dirs WHERE path = ?1", params![path_s]) {
|
||||||
self.swap_remove(idx);
|
Ok(count) => {
|
||||||
true
|
if count > 0 {
|
||||||
|
self.dirty = true;
|
||||||
|
true
|
||||||
|
} else {
|
||||||
|
false
|
||||||
|
}
|
||||||
}
|
}
|
||||||
None => false,
|
Err(_) => false,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn swap_remove(&mut self, idx: usize) {
|
pub fn swap_remove(&mut self, _idx: usize) {
|
||||||
self.with_dirs_mut(|dirs| dirs.swap_remove(idx));
|
// In the sqlite-backed implementation we don't maintain an in-memory
|
||||||
self.with_dirty_mut(|dirty| *dirty = true);
|
// vector, so this is a no-op. Higher-level code that relies on
|
||||||
|
// indices shouldn't be calling this directly except within the
|
||||||
|
// streaming logic which uses Database::dirs(). For compatibility, keep
|
||||||
|
// the method but do nothing.
|
||||||
|
self.dirty = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn age(&mut self, max_age: Rank) {
|
pub fn age(&mut self, max_age: Rank) {
|
||||||
let mut dirty = false;
|
// Apply the aging algorithm to all rows.
|
||||||
self.with_dirs_mut(|dirs| {
|
// Collect entries first to avoid holding a Statement borrow while starting
|
||||||
let total_age = dirs.iter().map(|dir| dir.rank).sum::<Rank>();
|
// a transaction on the connection.
|
||||||
if total_age > max_age {
|
let mut entries = Vec::new();
|
||||||
let factor = 0.9 * max_age / total_age;
|
if let Ok(mut stmt) = self.conn.prepare("SELECT path, rank FROM dirs") {
|
||||||
for idx in (0..dirs.len()).rev() {
|
if let Ok(rows) =
|
||||||
let dir = &mut dirs[idx];
|
stmt.query_map([], |row| Ok((row.get::<_, String>(0)?, row.get::<_, f64>(1)?)))
|
||||||
dir.rank *= factor;
|
{
|
||||||
if dir.rank < 1.0 {
|
for r in rows {
|
||||||
dirs.swap_remove(idx);
|
if let Ok((path, rank)) = r {
|
||||||
|
entries.push((path, rank));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
dirty = true;
|
|
||||||
}
|
}
|
||||||
});
|
}
|
||||||
self.with_dirty_mut(|dirty_prev| *dirty_prev |= dirty);
|
|
||||||
|
let total_age: f64 = entries.iter().map(|(_, rank)| *rank).sum();
|
||||||
|
if total_age > max_age {
|
||||||
|
let factor = 0.9 * max_age / total_age;
|
||||||
|
if let Ok(tx) = self.conn.transaction() {
|
||||||
|
for (path, rank) in entries {
|
||||||
|
let new_rank = rank * factor;
|
||||||
|
if new_rank < 1.0 {
|
||||||
|
let _ = tx.execute("DELETE FROM dirs WHERE path = ?1", params![path]);
|
||||||
|
} else {
|
||||||
|
let _ = tx.execute(
|
||||||
|
"UPDATE dirs SET rank = ?1 WHERE path = ?2",
|
||||||
|
params![new_rank, path],
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let _ = tx.commit();
|
||||||
|
self.dirty = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn dedup(&mut self) {
|
pub fn dedup(&mut self) {
|
||||||
// Sort by path, so that equal paths are next to each other.
|
// Using path as PRIMARY KEY ensures uniqueness, nothing to do here.
|
||||||
self.sort_by_path();
|
|
||||||
|
|
||||||
let mut dirty = false;
|
|
||||||
self.with_dirs_mut(|dirs| {
|
|
||||||
for idx in (1..dirs.len()).rev() {
|
|
||||||
// Check if curr_dir and next_dir have equal paths.
|
|
||||||
let curr_dir = &dirs[idx];
|
|
||||||
let next_dir = &dirs[idx - 1];
|
|
||||||
if next_dir.path != curr_dir.path {
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Merge curr_dir's rank and last_accessed into next_dir.
|
|
||||||
let rank = curr_dir.rank;
|
|
||||||
let last_accessed = curr_dir.last_accessed;
|
|
||||||
let next_dir = &mut dirs[idx - 1];
|
|
||||||
next_dir.last_accessed = next_dir.last_accessed.max(last_accessed);
|
|
||||||
next_dir.rank += rank;
|
|
||||||
|
|
||||||
// Delete curr_dir.
|
|
||||||
dirs.swap_remove(idx);
|
|
||||||
dirty = true;
|
|
||||||
}
|
|
||||||
});
|
|
||||||
self.with_dirty_mut(|dirty_prev| *dirty_prev |= dirty);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn sort_by_path(&mut self) {
|
pub fn sort_by_path(&mut self) {
|
||||||
self.with_dirs_mut(|dirs| dirs.sort_unstable_by(|dir1, dir2| dir1.path.cmp(&dir2.path)));
|
// Sorting is done at query time in the sqlite-backed implementation.
|
||||||
self.with_dirty_mut(|dirty| *dirty = true);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn sort_by_score(&mut self, now: Epoch) {
|
pub fn sort_by_score(&mut self, _now: Epoch) {
|
||||||
self.with_dirs_mut(|dirs| {
|
// Sorting is done at query time in the sqlite-backed implementation.
|
||||||
dirs.sort_unstable_by(|dir1: &Dir, dir2: &Dir| {
|
|
||||||
dir1.score(now).total_cmp(&dir2.score(now))
|
|
||||||
})
|
|
||||||
});
|
|
||||||
self.with_dirty_mut(|dirty| *dirty = true);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn dirty(&self) -> bool {
|
pub fn dirty(&self) -> bool {
|
||||||
*self.borrow_dirty()
|
self.dirty
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn dirs(&self) -> &[Dir<'_>] {
|
pub fn dirs(&self) -> Vec<Dir<'static>> {
|
||||||
self.borrow_dirs()
|
// Load all dirs from the database into an owned Vec.
|
||||||
}
|
let mut stmt = match self.conn.prepare("SELECT path, rank, last_accessed FROM dirs") {
|
||||||
|
Ok(s) => s,
|
||||||
fn serialize(dirs: &[Dir<'_>]) -> Result<Vec<u8>> {
|
Err(_) => return Vec::new(),
|
||||||
(|| -> bincode::Result<_> {
|
|
||||||
// Preallocate buffer with combined size of sections.
|
|
||||||
let buffer_size =
|
|
||||||
bincode::serialized_size(&Self::VERSION)? + bincode::serialized_size(&dirs)?;
|
|
||||||
let mut buffer = Vec::with_capacity(buffer_size as usize);
|
|
||||||
|
|
||||||
// Serialize sections into buffer.
|
|
||||||
bincode::serialize_into(&mut buffer, &Self::VERSION)?;
|
|
||||||
bincode::serialize_into(&mut buffer, &dirs)?;
|
|
||||||
|
|
||||||
Ok(buffer)
|
|
||||||
})()
|
|
||||||
.context("could not serialize database")
|
|
||||||
}
|
|
||||||
|
|
||||||
fn deserialize(bytes: &[u8]) -> Result<Vec<Dir<'_>>> {
|
|
||||||
// Assume a maximum size for the database. This prevents bincode from throwing
|
|
||||||
// strange errors when it encounters invalid data.
|
|
||||||
const MAX_SIZE: u64 = 32 << 20; // 32 MiB
|
|
||||||
let deserializer = &mut bincode::options().with_fixint_encoding().with_limit(MAX_SIZE);
|
|
||||||
|
|
||||||
// Split bytes into sections.
|
|
||||||
let version_size = deserializer.serialized_size(&Self::VERSION).unwrap() as _;
|
|
||||||
if bytes.len() < version_size {
|
|
||||||
bail!("could not deserialize database: corrupted data");
|
|
||||||
}
|
|
||||||
let (bytes_version, bytes_dirs) = bytes.split_at(version_size);
|
|
||||||
|
|
||||||
// Deserialize sections.
|
|
||||||
let version = deserializer.deserialize(bytes_version)?;
|
|
||||||
let dirs = match version {
|
|
||||||
Self::VERSION => {
|
|
||||||
deserializer.deserialize(bytes_dirs).context("could not deserialize database")?
|
|
||||||
}
|
|
||||||
version => {
|
|
||||||
bail!("unsupported version (got {version}, supports {})", Self::VERSION)
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
Ok(dirs)
|
let rows = stmt.query_map([], |row| {
|
||||||
|
Ok(Dir {
|
||||||
|
path: row.get::<_, String>(0)?.into(),
|
||||||
|
rank: row.get::<_, f64>(1)?,
|
||||||
|
last_accessed: row.get::<_, u64>(2)?,
|
||||||
|
})
|
||||||
|
});
|
||||||
|
|
||||||
|
let mut out = Vec::new();
|
||||||
|
if let Ok(map) = rows {
|
||||||
|
for r in map {
|
||||||
|
if let Ok(dir) = r {
|
||||||
|
out.push(dir);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
out
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -253,7 +276,8 @@ mod tests {
|
||||||
let db = Database::open_dir(data_dir.path()).unwrap();
|
let db = Database::open_dir(data_dir.path()).unwrap();
|
||||||
assert_eq!(db.dirs().len(), 1);
|
assert_eq!(db.dirs().len(), 1);
|
||||||
|
|
||||||
let dir = &db.dirs()[0];
|
let dirs = db.dirs();
|
||||||
|
let dir = &dirs[0];
|
||||||
assert_eq!(dir.path, path);
|
assert_eq!(dir.path, path);
|
||||||
assert!((dir.rank - 2.0).abs() < 0.01);
|
assert!((dir.rank - 2.0).abs() < 0.01);
|
||||||
assert_eq!(dir.last_accessed, now);
|
assert_eq!(dir.last_accessed, now);
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,3 @@
|
||||||
use std::iter::Rev;
|
|
||||||
use std::ops::Range;
|
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::{fs, path};
|
use std::{fs, path};
|
||||||
|
|
||||||
|
|
@ -10,20 +8,25 @@ use crate::util::{self, MONTH};
|
||||||
|
|
||||||
pub struct Stream<'a> {
|
pub struct Stream<'a> {
|
||||||
db: &'a mut Database,
|
db: &'a mut Database,
|
||||||
idxs: Rev<Range<usize>>,
|
entries: Vec<Dir<'static>>,
|
||||||
|
pos: usize,
|
||||||
options: StreamOptions,
|
options: StreamOptions,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<'a> Stream<'a> {
|
impl<'a> Stream<'a> {
|
||||||
pub fn new(db: &'a mut Database, options: StreamOptions) -> Self {
|
pub fn new(db: &'a mut Database, options: StreamOptions) -> Self {
|
||||||
db.sort_by_score(options.now);
|
// Load entries and sort by score.
|
||||||
let idxs = (0..db.dirs().len()).rev();
|
let mut entries = db.dirs();
|
||||||
Stream { db, idxs, options }
|
entries.sort_unstable_by(|a, b| a.score(options.now).total_cmp(&b.score(options.now)));
|
||||||
|
// iterate from highest to lowest
|
||||||
|
entries.reverse();
|
||||||
|
Stream { db, entries, pos: 0, options }
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn next(&mut self) -> Option<&Dir<'_>> {
|
pub fn next(&mut self) -> Option<&Dir<'_>> {
|
||||||
while let Some(idx) = self.idxs.next() {
|
while self.pos < self.entries.len() {
|
||||||
let dir = &self.db.dirs()[idx];
|
let dir = &self.entries[self.pos];
|
||||||
|
self.pos += 1;
|
||||||
|
|
||||||
if !self.filter_by_keywords(&dir.path) {
|
if !self.filter_by_keywords(&dir.path) {
|
||||||
continue;
|
continue;
|
||||||
|
|
@ -34,20 +37,20 @@ impl<'a> Stream<'a> {
|
||||||
}
|
}
|
||||||
|
|
||||||
if !self.filter_by_exclude(&dir.path) {
|
if !self.filter_by_exclude(&dir.path) {
|
||||||
self.db.swap_remove(idx);
|
// lazily remove from database
|
||||||
|
let _ = self.db.remove(&*dir.path);
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
// Exists queries are slow, this should always be checked last.
|
// Exists queries are slow, this should always be checked last.
|
||||||
if !self.filter_by_exists(&dir.path) {
|
if !self.filter_by_exists(&dir.path) {
|
||||||
if dir.last_accessed < self.options.ttl {
|
if dir.last_accessed < self.options.ttl {
|
||||||
self.db.swap_remove(idx);
|
let _ = self.db.remove(&*dir.path);
|
||||||
}
|
}
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
let dir = &self.db.dirs()[idx];
|
return Some(&self.entries[self.pos - 1]);
|
||||||
return Some(dir);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
None
|
None
|
||||||
|
|
@ -203,9 +206,10 @@ mod tests {
|
||||||
#[case(&["/foo/", "/bar"], "/foo/bar", false)]
|
#[case(&["/foo/", "/bar"], "/foo/bar", false)]
|
||||||
#[case(&["/foo/", "/bar"], "/foo/baz/bar", true)]
|
#[case(&["/foo/", "/bar"], "/foo/baz/bar", true)]
|
||||||
fn query(#[case] keywords: &[&str], #[case] path: &str, #[case] is_match: bool) {
|
fn query(#[case] keywords: &[&str], #[case] path: &str, #[case] is_match: bool) {
|
||||||
let db = &mut Database::new(PathBuf::new(), Vec::new(), |_| Vec::new(), false);
|
let data_dir = tempfile::tempdir().unwrap();
|
||||||
|
let mut db = Database::open_dir(data_dir.path()).unwrap();
|
||||||
let options = StreamOptions::new(0).with_keywords(keywords.iter());
|
let options = StreamOptions::new(0).with_keywords(keywords.iter());
|
||||||
let stream = Stream::new(db, options);
|
let stream = Stream::new(&mut db, options);
|
||||||
assert_eq!(is_match, stream.filter_by_keywords(path));
|
assert_eq!(is_match, stream.filter_by_keywords(path));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,5 +1,4 @@
|
||||||
use std::ffi::OsStr;
|
use std::ffi::OsStr;
|
||||||
use std::fs::{self, File, OpenOptions};
|
|
||||||
use std::io::{self, Read, Write};
|
use std::io::{self, Read, Write};
|
||||||
use std::path::{Component, Path, PathBuf};
|
use std::path::{Component, Path, PathBuf};
|
||||||
use std::process::{Child, Command, Stdio};
|
use std::process::{Child, Command, Stdio};
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue