wip
This commit is contained in:
lengyijun
parent
f3a07cd3d1
commit
0dc4a72500
|
|
@ -2,6 +2,18 @@
|
|||
# It is not intended for manual editing.
|
||||
version = 4
|
||||
|
||||
[[package]]
|
||||
name = "ahash"
|
||||
version = "0.8.12"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"once_cell",
|
||||
"version_check",
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aho-corasick"
|
||||
version = "1.1.3"
|
||||
|
|
@ -17,6 +29,12 @@ version = "0.1.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd"
|
||||
|
||||
[[package]]
|
||||
name = "allocator-api2"
|
||||
version = "0.2.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
|
||||
|
||||
[[package]]
|
||||
name = "anstream"
|
||||
version = "0.6.18"
|
||||
|
|
@ -126,15 +144,6 @@ dependencies = [
|
|||
"wait-timeout",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bincode"
|
||||
version = "1.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bitflags"
|
||||
version = "2.9.1"
|
||||
|
|
@ -321,6 +330,18 @@ dependencies = [
|
|||
"windows-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "fallible-iterator"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649"
|
||||
|
||||
[[package]]
|
||||
name = "fallible-streaming-iterator"
|
||||
version = "0.1.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
|
||||
|
||||
[[package]]
|
||||
name = "fastrand"
|
||||
version = "2.3.0"
|
||||
|
|
@ -356,6 +377,25 @@ version = "0.3.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2"
|
||||
|
||||
[[package]]
|
||||
name = "hashbrown"
|
||||
version = "0.14.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"allocator-api2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "hashlink"
|
||||
version = "0.8.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e8094feaf31ff591f651a2664fb9cfd92bba7a60ce3197265e9482ebe753c8f7"
|
||||
dependencies = [
|
||||
"hashbrown",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "heck"
|
||||
version = "0.4.1"
|
||||
|
|
@ -396,6 +436,16 @@ dependencies = [
|
|||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "libsqlite3-sys"
|
||||
version = "0.27.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf4e226dcd58b4be396f7bd3c20da8fdee2911400705297ba7d2d7cc2c30f716"
|
||||
dependencies = [
|
||||
"pkg-config",
|
||||
"vcpkg",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "linux-raw-sys"
|
||||
version = "0.9.4"
|
||||
|
|
@ -484,6 +534,12 @@ version = "2.3.1"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
|
||||
|
||||
[[package]]
|
||||
name = "pkg-config"
|
||||
version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.21"
|
||||
|
|
@ -670,6 +726,20 @@ dependencies = [
|
|||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rusqlite"
|
||||
version = "0.30.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a78046161564f5e7cd9008aff3b2990b3850dc8e0349119b98e8f251e099f24d"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"fallible-iterator",
|
||||
"fallible-streaming-iterator",
|
||||
"hashlink",
|
||||
"libsqlite3-sys",
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-hash"
|
||||
version = "2.1.1"
|
||||
|
|
@ -742,6 +812,12 @@ dependencies = [
|
|||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "smallvec"
|
||||
version = "1.15.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
|
||||
|
||||
[[package]]
|
||||
name = "static_assertions"
|
||||
version = "1.1.0"
|
||||
|
|
@ -816,6 +892,12 @@ version = "0.2.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
|
||||
|
||||
[[package]]
|
||||
name = "vcpkg"
|
||||
version = "0.2.15"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
|
||||
|
||||
[[package]]
|
||||
name = "version_check"
|
||||
version = "0.9.5"
|
||||
|
|
@ -988,7 +1070,6 @@ dependencies = [
|
|||
"anyhow",
|
||||
"askama",
|
||||
"assert_cmd",
|
||||
"bincode",
|
||||
"clap",
|
||||
"clap_complete",
|
||||
"clap_complete_fig",
|
||||
|
|
@ -1002,6 +1083,7 @@ dependencies = [
|
|||
"ouroboros",
|
||||
"rstest",
|
||||
"rstest_reuse",
|
||||
"rusqlite",
|
||||
"serde",
|
||||
"tempfile",
|
||||
"which",
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ askama = { version = "0.14.0", default-features = false, features = [
|
|||
"derive",
|
||||
"std",
|
||||
] }
|
||||
bincode = "1.3.1"
|
||||
rusqlite = "0.30.0"
|
||||
clap = { version = "4.3.0", features = ["derive"] }
|
||||
color-print = "0.3.4"
|
||||
dirs = "6.0.0"
|
||||
|
|
|
|||
332
src/db/mod.rs
332
src/db/mod.rs
|
|
@ -1,24 +1,19 @@
|
|||
mod dir;
|
||||
mod stream;
|
||||
|
||||
use std::fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::{fs, io};
|
||||
|
||||
use anyhow::{Context, Result, bail};
|
||||
use bincode::Options;
|
||||
use ouroboros::self_referencing;
|
||||
use anyhow::{Context, Result};
|
||||
use rusqlite::{Connection, OptionalExtension, params};
|
||||
|
||||
use crate::config;
|
||||
pub use crate::db::dir::{Dir, Epoch, Rank};
|
||||
pub use crate::db::stream::{Stream, StreamOptions};
|
||||
use crate::{config, util};
|
||||
|
||||
#[self_referencing]
|
||||
pub struct Database {
|
||||
path: PathBuf,
|
||||
bytes: Vec<u8>,
|
||||
#[borrows(bytes)]
|
||||
#[covariant]
|
||||
pub dirs: Vec<Dir<'this>>,
|
||||
conn: Connection,
|
||||
dirty: bool,
|
||||
}
|
||||
|
||||
|
|
@ -32,47 +27,73 @@ impl Database {
|
|||
|
||||
pub fn open_dir(data_dir: impl AsRef<Path>) -> Result<Self> {
|
||||
let data_dir = data_dir.as_ref();
|
||||
let path = data_dir.join("db.zo");
|
||||
let path = data_dir.join("db.sqlite3");
|
||||
let path = fs::canonicalize(&path).unwrap_or(path);
|
||||
|
||||
match fs::read(&path) {
|
||||
Ok(bytes) => Self::try_new(path, bytes, |bytes| Self::deserialize(bytes), false),
|
||||
Err(e) if e.kind() == io::ErrorKind::NotFound => {
|
||||
// Create data directory, but don't create any file yet. The file will be
|
||||
// created later by [`Database::save`] if any data is modified.
|
||||
fs::create_dir_all(data_dir).with_context(|| {
|
||||
format!("unable to create data directory: {}", data_dir.display())
|
||||
})?;
|
||||
Ok(Self::new(path, Vec::new(), |_| Vec::new(), false))
|
||||
}
|
||||
Err(e) => {
|
||||
Err(e).with_context(|| format!("could not read from database: {}", path.display()))
|
||||
}
|
||||
}
|
||||
fs::create_dir_all(data_dir)
|
||||
.with_context(|| format!("unable to create data directory: {}", data_dir.display()))?;
|
||||
|
||||
// Open or create sqlite database file.
|
||||
let conn = Connection::open(&path)
|
||||
.with_context(|| format!("could not open database: {}", path.display()))?;
|
||||
|
||||
// Enable WAL for better concurrency and durability.
|
||||
conn.pragma_update(None, "journal_mode", &"WAL").ok();
|
||||
|
||||
// Create table if it doesn't exist.
|
||||
conn.execute_batch(
|
||||
"CREATE TABLE IF NOT EXISTS dirs (
|
||||
path TEXT PRIMARY KEY,
|
||||
rank REAL NOT NULL,
|
||||
last_accessed INTEGER NOT NULL
|
||||
);",
|
||||
)?;
|
||||
|
||||
Ok(Database { path, conn, dirty: false })
|
||||
}
|
||||
|
||||
pub fn save(&mut self) -> Result<()> {
|
||||
// Only write to disk if the database is modified.
|
||||
if !self.dirty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let bytes = Self::serialize(self.dirs())?;
|
||||
util::write(self.borrow_path(), bytes).context("could not write to database")?;
|
||||
self.with_dirty_mut(|dirty| *dirty = false);
|
||||
|
||||
// For SQLite, write operations are applied immediately via transactions.
|
||||
// Keep save() for compatibility; do nothing.
|
||||
self.dirty = false;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Increments the rank of a directory, or creates it if it does not exist.
|
||||
pub fn add(&mut self, path: impl AsRef<str> + Into<String>, by: Rank, now: Epoch) {
|
||||
self.with_dirs_mut(|dirs| match dirs.iter_mut().find(|dir| dir.path == path.as_ref()) {
|
||||
Some(dir) => dir.rank = (dir.rank + by).max(0.0),
|
||||
None => {
|
||||
dirs.push(Dir { path: path.into().into(), rank: by.max(0.0), last_accessed: now })
|
||||
let path_s: String = path.into();
|
||||
let tx = match self.conn.transaction() {
|
||||
Ok(t) => t,
|
||||
Err(_) => return,
|
||||
};
|
||||
|
||||
let existing: Option<(f64, u64)> = tx
|
||||
.query_row(
|
||||
"SELECT rank, last_accessed FROM dirs WHERE path = ?1",
|
||||
params![&path_s],
|
||||
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||
)
|
||||
.optional()
|
||||
.unwrap_or(None);
|
||||
|
||||
match existing {
|
||||
Some((rank, _last)) => {
|
||||
let new_rank = (rank + by).max(0.0);
|
||||
let _ = tx.execute(
|
||||
"UPDATE dirs SET rank = ?1 WHERE path = ?2",
|
||||
params![new_rank, &path_s],
|
||||
);
|
||||
}
|
||||
});
|
||||
self.with_dirty_mut(|dirty| *dirty = true);
|
||||
None => {
|
||||
let _ = tx.execute(
|
||||
"INSERT INTO dirs (path, rank, last_accessed) VALUES (?1, ?2, ?3)",
|
||||
params![&path_s, by.max(0.0), now],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let _ = tx.commit();
|
||||
self.dirty = true;
|
||||
}
|
||||
|
||||
/// Creates a new directory. This will create a duplicate entry if this
|
||||
|
|
@ -80,155 +101,157 @@ impl Database {
|
|||
/// either does a check before calling this, or calls `dedup()`
|
||||
/// afterward.
|
||||
pub fn add_unchecked(&mut self, path: impl AsRef<str> + Into<String>, rank: Rank, now: Epoch) {
|
||||
self.with_dirs_mut(|dirs| {
|
||||
dirs.push(Dir { path: path.into().into(), rank, last_accessed: now })
|
||||
});
|
||||
self.with_dirty_mut(|dirty| *dirty = true);
|
||||
let path_s: String = path.into();
|
||||
let _ = self.conn.execute(
|
||||
"INSERT OR REPLACE INTO dirs (path, rank, last_accessed) VALUES (?1, ?2, ?3)",
|
||||
params![&path_s, rank, now],
|
||||
);
|
||||
self.dirty = true;
|
||||
}
|
||||
|
||||
/// Increments the rank and updates the last_accessed of a directory, or
|
||||
/// creates it if it does not exist.
|
||||
pub fn add_update(&mut self, path: impl AsRef<str> + Into<String>, by: Rank, now: Epoch) {
|
||||
self.with_dirs_mut(|dirs| match dirs.iter_mut().find(|dir| dir.path == path.as_ref()) {
|
||||
Some(dir) => {
|
||||
dir.rank = (dir.rank + by).max(0.0);
|
||||
dir.last_accessed = now;
|
||||
let path_s: String = path.into();
|
||||
let tx = match self.conn.transaction() {
|
||||
Ok(t) => t,
|
||||
Err(_) => return,
|
||||
};
|
||||
|
||||
let existing: Option<(f64, u64)> = tx
|
||||
.query_row(
|
||||
"SELECT rank, last_accessed FROM dirs WHERE path = ?1",
|
||||
params![&path_s],
|
||||
|row| Ok((row.get(0)?, row.get(1)?)),
|
||||
)
|
||||
.optional()
|
||||
.unwrap_or(None);
|
||||
|
||||
match existing {
|
||||
Some((rank, _)) => {
|
||||
let new_rank = (rank + by).max(0.0);
|
||||
let _ = tx.execute(
|
||||
"UPDATE dirs SET rank = ?1, last_accessed = ?2 WHERE path = ?3",
|
||||
params![new_rank, now, &path_s],
|
||||
);
|
||||
}
|
||||
None => {
|
||||
dirs.push(Dir { path: path.into().into(), rank: by.max(0.0), last_accessed: now })
|
||||
let _ = tx.execute(
|
||||
"INSERT INTO dirs (path, rank, last_accessed) VALUES (?1, ?2, ?3)",
|
||||
params![&path_s, by.max(0.0), now],
|
||||
);
|
||||
}
|
||||
});
|
||||
self.with_dirty_mut(|dirty| *dirty = true);
|
||||
}
|
||||
|
||||
let _ = tx.commit();
|
||||
self.dirty = true;
|
||||
}
|
||||
|
||||
/// Removes the directory with `path` from the store. This does not preserve
|
||||
/// ordering, but is O(1).
|
||||
/// Removes the directory with `path` from the store. Returns true if an
|
||||
/// entry was deleted.
|
||||
pub fn remove(&mut self, path: impl AsRef<str>) -> bool {
|
||||
match self.dirs().iter().position(|dir| dir.path == path.as_ref()) {
|
||||
Some(idx) => {
|
||||
self.swap_remove(idx);
|
||||
true
|
||||
let path_s = path.as_ref();
|
||||
match self.conn.execute("DELETE FROM dirs WHERE path = ?1", params![path_s]) {
|
||||
Ok(count) => {
|
||||
if count > 0 {
|
||||
self.dirty = true;
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
None => false,
|
||||
Err(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn swap_remove(&mut self, idx: usize) {
|
||||
self.with_dirs_mut(|dirs| dirs.swap_remove(idx));
|
||||
self.with_dirty_mut(|dirty| *dirty = true);
|
||||
pub fn swap_remove(&mut self, _idx: usize) {
|
||||
// In the sqlite-backed implementation we don't maintain an in-memory
|
||||
// vector, so this is a no-op. Higher-level code that relies on
|
||||
// indices shouldn't be calling this directly except within the
|
||||
// streaming logic which uses Database::dirs(). For compatibility, keep
|
||||
// the method but do nothing.
|
||||
self.dirty = true;
|
||||
}
|
||||
|
||||
pub fn age(&mut self, max_age: Rank) {
|
||||
let mut dirty = false;
|
||||
self.with_dirs_mut(|dirs| {
|
||||
let total_age = dirs.iter().map(|dir| dir.rank).sum::<Rank>();
|
||||
if total_age > max_age {
|
||||
let factor = 0.9 * max_age / total_age;
|
||||
for idx in (0..dirs.len()).rev() {
|
||||
let dir = &mut dirs[idx];
|
||||
dir.rank *= factor;
|
||||
if dir.rank < 1.0 {
|
||||
dirs.swap_remove(idx);
|
||||
// Apply the aging algorithm to all rows.
|
||||
// Collect entries first to avoid holding a Statement borrow while starting
|
||||
// a transaction on the connection.
|
||||
let mut entries = Vec::new();
|
||||
if let Ok(mut stmt) = self.conn.prepare("SELECT path, rank FROM dirs") {
|
||||
if let Ok(rows) =
|
||||
stmt.query_map([], |row| Ok((row.get::<_, String>(0)?, row.get::<_, f64>(1)?)))
|
||||
{
|
||||
for r in rows {
|
||||
if let Ok((path, rank)) = r {
|
||||
entries.push((path, rank));
|
||||
}
|
||||
}
|
||||
dirty = true;
|
||||
}
|
||||
});
|
||||
self.with_dirty_mut(|dirty_prev| *dirty_prev |= dirty);
|
||||
}
|
||||
|
||||
let total_age: f64 = entries.iter().map(|(_, rank)| *rank).sum();
|
||||
if total_age > max_age {
|
||||
let factor = 0.9 * max_age / total_age;
|
||||
if let Ok(tx) = self.conn.transaction() {
|
||||
for (path, rank) in entries {
|
||||
let new_rank = rank * factor;
|
||||
if new_rank < 1.0 {
|
||||
let _ = tx.execute("DELETE FROM dirs WHERE path = ?1", params![path]);
|
||||
} else {
|
||||
let _ = tx.execute(
|
||||
"UPDATE dirs SET rank = ?1 WHERE path = ?2",
|
||||
params![new_rank, path],
|
||||
);
|
||||
}
|
||||
}
|
||||
let _ = tx.commit();
|
||||
self.dirty = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn dedup(&mut self) {
|
||||
// Sort by path, so that equal paths are next to each other.
|
||||
self.sort_by_path();
|
||||
|
||||
let mut dirty = false;
|
||||
self.with_dirs_mut(|dirs| {
|
||||
for idx in (1..dirs.len()).rev() {
|
||||
// Check if curr_dir and next_dir have equal paths.
|
||||
let curr_dir = &dirs[idx];
|
||||
let next_dir = &dirs[idx - 1];
|
||||
if next_dir.path != curr_dir.path {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Merge curr_dir's rank and last_accessed into next_dir.
|
||||
let rank = curr_dir.rank;
|
||||
let last_accessed = curr_dir.last_accessed;
|
||||
let next_dir = &mut dirs[idx - 1];
|
||||
next_dir.last_accessed = next_dir.last_accessed.max(last_accessed);
|
||||
next_dir.rank += rank;
|
||||
|
||||
// Delete curr_dir.
|
||||
dirs.swap_remove(idx);
|
||||
dirty = true;
|
||||
}
|
||||
});
|
||||
self.with_dirty_mut(|dirty_prev| *dirty_prev |= dirty);
|
||||
// Using path as PRIMARY KEY ensures uniqueness, nothing to do here.
|
||||
}
|
||||
|
||||
pub fn sort_by_path(&mut self) {
|
||||
self.with_dirs_mut(|dirs| dirs.sort_unstable_by(|dir1, dir2| dir1.path.cmp(&dir2.path)));
|
||||
self.with_dirty_mut(|dirty| *dirty = true);
|
||||
// Sorting is done at query time in the sqlite-backed implementation.
|
||||
}
|
||||
|
||||
pub fn sort_by_score(&mut self, now: Epoch) {
|
||||
self.with_dirs_mut(|dirs| {
|
||||
dirs.sort_unstable_by(|dir1: &Dir, dir2: &Dir| {
|
||||
dir1.score(now).total_cmp(&dir2.score(now))
|
||||
})
|
||||
});
|
||||
self.with_dirty_mut(|dirty| *dirty = true);
|
||||
pub fn sort_by_score(&mut self, _now: Epoch) {
|
||||
// Sorting is done at query time in the sqlite-backed implementation.
|
||||
}
|
||||
|
||||
pub fn dirty(&self) -> bool {
|
||||
*self.borrow_dirty()
|
||||
self.dirty
|
||||
}
|
||||
|
||||
pub fn dirs(&self) -> &[Dir<'_>] {
|
||||
self.borrow_dirs()
|
||||
}
|
||||
|
||||
fn serialize(dirs: &[Dir<'_>]) -> Result<Vec<u8>> {
|
||||
(|| -> bincode::Result<_> {
|
||||
// Preallocate buffer with combined size of sections.
|
||||
let buffer_size =
|
||||
bincode::serialized_size(&Self::VERSION)? + bincode::serialized_size(&dirs)?;
|
||||
let mut buffer = Vec::with_capacity(buffer_size as usize);
|
||||
|
||||
// Serialize sections into buffer.
|
||||
bincode::serialize_into(&mut buffer, &Self::VERSION)?;
|
||||
bincode::serialize_into(&mut buffer, &dirs)?;
|
||||
|
||||
Ok(buffer)
|
||||
})()
|
||||
.context("could not serialize database")
|
||||
}
|
||||
|
||||
fn deserialize(bytes: &[u8]) -> Result<Vec<Dir<'_>>> {
|
||||
// Assume a maximum size for the database. This prevents bincode from throwing
|
||||
// strange errors when it encounters invalid data.
|
||||
const MAX_SIZE: u64 = 32 << 20; // 32 MiB
|
||||
let deserializer = &mut bincode::options().with_fixint_encoding().with_limit(MAX_SIZE);
|
||||
|
||||
// Split bytes into sections.
|
||||
let version_size = deserializer.serialized_size(&Self::VERSION).unwrap() as _;
|
||||
if bytes.len() < version_size {
|
||||
bail!("could not deserialize database: corrupted data");
|
||||
}
|
||||
let (bytes_version, bytes_dirs) = bytes.split_at(version_size);
|
||||
|
||||
// Deserialize sections.
|
||||
let version = deserializer.deserialize(bytes_version)?;
|
||||
let dirs = match version {
|
||||
Self::VERSION => {
|
||||
deserializer.deserialize(bytes_dirs).context("could not deserialize database")?
|
||||
}
|
||||
version => {
|
||||
bail!("unsupported version (got {version}, supports {})", Self::VERSION)
|
||||
}
|
||||
pub fn dirs(&self) -> Vec<Dir<'static>> {
|
||||
// Load all dirs from the database into an owned Vec.
|
||||
let mut stmt = match self.conn.prepare("SELECT path, rank, last_accessed FROM dirs") {
|
||||
Ok(s) => s,
|
||||
Err(_) => return Vec::new(),
|
||||
};
|
||||
|
||||
Ok(dirs)
|
||||
let rows = stmt.query_map([], |row| {
|
||||
Ok(Dir {
|
||||
path: row.get::<_, String>(0)?.into(),
|
||||
rank: row.get::<_, f64>(1)?,
|
||||
last_accessed: row.get::<_, u64>(2)?,
|
||||
})
|
||||
});
|
||||
|
||||
let mut out = Vec::new();
|
||||
if let Ok(map) = rows {
|
||||
for r in map {
|
||||
if let Ok(dir) = r {
|
||||
out.push(dir);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -253,7 +276,8 @@ mod tests {
|
|||
let db = Database::open_dir(data_dir.path()).unwrap();
|
||||
assert_eq!(db.dirs().len(), 1);
|
||||
|
||||
let dir = &db.dirs()[0];
|
||||
let dirs = db.dirs();
|
||||
let dir = &dirs[0];
|
||||
assert_eq!(dir.path, path);
|
||||
assert!((dir.rank - 2.0).abs() < 0.01);
|
||||
assert_eq!(dir.last_accessed, now);
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
use std::iter::Rev;
|
||||
use std::ops::Range;
|
||||
use std::path::Path;
|
||||
use std::{fs, path};
|
||||
|
||||
|
|
@ -10,20 +8,25 @@ use crate::util::{self, MONTH};
|
|||
|
||||
pub struct Stream<'a> {
|
||||
db: &'a mut Database,
|
||||
idxs: Rev<Range<usize>>,
|
||||
entries: Vec<Dir<'static>>,
|
||||
pos: usize,
|
||||
options: StreamOptions,
|
||||
}
|
||||
|
||||
impl<'a> Stream<'a> {
|
||||
pub fn new(db: &'a mut Database, options: StreamOptions) -> Self {
|
||||
db.sort_by_score(options.now);
|
||||
let idxs = (0..db.dirs().len()).rev();
|
||||
Stream { db, idxs, options }
|
||||
// Load entries and sort by score.
|
||||
let mut entries = db.dirs();
|
||||
entries.sort_unstable_by(|a, b| a.score(options.now).total_cmp(&b.score(options.now)));
|
||||
// iterate from highest to lowest
|
||||
entries.reverse();
|
||||
Stream { db, entries, pos: 0, options }
|
||||
}
|
||||
|
||||
pub fn next(&mut self) -> Option<&Dir<'_>> {
|
||||
while let Some(idx) = self.idxs.next() {
|
||||
let dir = &self.db.dirs()[idx];
|
||||
while self.pos < self.entries.len() {
|
||||
let dir = &self.entries[self.pos];
|
||||
self.pos += 1;
|
||||
|
||||
if !self.filter_by_keywords(&dir.path) {
|
||||
continue;
|
||||
|
|
@ -34,20 +37,20 @@ impl<'a> Stream<'a> {
|
|||
}
|
||||
|
||||
if !self.filter_by_exclude(&dir.path) {
|
||||
self.db.swap_remove(idx);
|
||||
// lazily remove from database
|
||||
let _ = self.db.remove(&*dir.path);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Exists queries are slow, this should always be checked last.
|
||||
if !self.filter_by_exists(&dir.path) {
|
||||
if dir.last_accessed < self.options.ttl {
|
||||
self.db.swap_remove(idx);
|
||||
let _ = self.db.remove(&*dir.path);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
let dir = &self.db.dirs()[idx];
|
||||
return Some(dir);
|
||||
return Some(&self.entries[self.pos - 1]);
|
||||
}
|
||||
|
||||
None
|
||||
|
|
@ -203,9 +206,10 @@ mod tests {
|
|||
#[case(&["/foo/", "/bar"], "/foo/bar", false)]
|
||||
#[case(&["/foo/", "/bar"], "/foo/baz/bar", true)]
|
||||
fn query(#[case] keywords: &[&str], #[case] path: &str, #[case] is_match: bool) {
|
||||
let db = &mut Database::new(PathBuf::new(), Vec::new(), |_| Vec::new(), false);
|
||||
let data_dir = tempfile::tempdir().unwrap();
|
||||
let mut db = Database::open_dir(data_dir.path()).unwrap();
|
||||
let options = StreamOptions::new(0).with_keywords(keywords.iter());
|
||||
let stream = Stream::new(db, options);
|
||||
let stream = Stream::new(&mut db, options);
|
||||
assert_eq!(is_match, stream.filter_by_keywords(path));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
use std::ffi::OsStr;
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::io::{self, Read, Write};
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
use std::process::{Child, Command, Stdio};
|
||||
|
|
|
|||
Loading…
Reference in New Issue