From 3d3d56440a4a55bb63cbec7805ea109db892bcb6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tristan=20Dani=C3=ABl=20Maat?= <tm@tlater.net>
Date: Wed, 26 Mar 2025 02:27:39 +0800
Subject: [PATCH] feat(card_db): Only fetch database when upstream has updated
---
.cargo/config.toml | 2 +
Cargo.lock | 52 ++++++++++++++++
Cargo.toml | 6 +-
flake.nix | 2 +
src/bin/database_worker.rs | 1 +
src/database.rs | 20 +-----
src/database/worker.rs | 67 ++++++++++++++++++--
src/utils.rs | 122 +++++++++++++++++++++++++++++++++++++
8 files changed, 249 insertions(+), 23 deletions(-)
create mode 100644 .cargo/config.toml
diff --git a/.cargo/config.toml b/.cargo/config.toml
new file mode 100644
index 0000000..f4e8c00
--- /dev/null
+++ b/.cargo/config.toml
@@ -0,0 +1,2 @@
+[build]
+target = "wasm32-unknown-unknown"
diff --git a/Cargo.lock b/Cargo.lock
index 4a72acf..128cf41 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -155,6 +155,15 @@ version = "1.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3"
+[[package]]
+name = "cc"
+version = "1.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fcb57c740ae1daf453ae85f16e37396f672b039e00d9d866e07ddb24e328e3a"
+dependencies = [
+ "shlex",
+]
+
[[package]]
name = "cfg-if"
version = "1.0.0"
@@ -427,7 +436,9 @@ dependencies = [
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
+ "wasm-bindgen-test",
"wasm-logger",
+ "web-sys",
]
[[package]]
@@ -1328,6 +1339,16 @@ dependencies = [
"quote",
]
+[[package]]
+name = "minicov"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f27fe9f1cc3c22e1687f9446c2083c4c5fc7f0bcf1c7a86bdbded14985895b4b"
+dependencies = [
+ "cc",
+ "walkdir",
+]
+
[[package]]
name = "minimal-lexical"
version = "0.2.1"
@@ -1893,6 +1914,12 @@ dependencies = [
"syn",
]
+[[package]]
+name = "shlex"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
+
[[package]]
name = "slab"
version = "0.4.9"
@@ -2093,6 +2120,7 @@ checksum = "dad298b01a40a23aac4580b67e3dbedb7cc8402f3592d7f49469de2ea4aecdd8"
dependencies = [
"deranged",
"itoa",
+ "js-sys",
"num-conv",
"powerfmt",
"serde",
@@ -2366,6 +2394,30 @@ dependencies = [
"unicode-ident",
]
+[[package]]
+name = "wasm-bindgen-test"
+version = "0.3.50"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "66c8d5e33ca3b6d9fa3b4676d774c5778031d27a578c2b007f905acf816152c3"
+dependencies = [
+ "js-sys",
+ "minicov",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "wasm-bindgen-test-macro",
+]
+
+[[package]]
+name = "wasm-bindgen-test-macro"
+version = "0.3.50"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "17d5042cc5fa009658f9a7333ef24291b1291a25b6382dd68862a7f3b969f69b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
[[package]]
name = "wasm-logger"
version = "0.2.0"
diff --git a/Cargo.toml b/Cargo.toml
index 7a33316..aa1f690 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -18,7 +18,7 @@ wasm-bindgen = "0.2.100"
wasm-bindgen-futures = { version = "0.4.50", features = ["futures-core-03-stream"] }
diesel-derive-enum = { version = "2.1.0", features = ["sqlite"] }
diesel_migrations = { git = "https://github.com/diesel-rs/diesel.git", features = ["sqlite"] }
-time = { version = "0.3.39", features = ["parsing"] }
+time = { version = "0.3.39", features = ["formatting", "parsing", "wasm-bindgen"] }
wasm-logger = "0.2.0"
log = "0.4.26"
sqlite-wasm-rs = { version = ">=0.3.0, <0.4.0" , default-features = false, features = ["precompiled"]}
@@ -26,3 +26,7 @@ thiserror-ext = "0.2.1"
futures-util = "0.3.31"
gloo = { version = "0.11.0", features = ["futures"] }
gloo-net = { version = "0.6.0" }
+web-sys = { version = "0.3.77", features = ["FileSystemWritableFileStream", "WorkerGlobalScope"] }
+
+[target.'cfg(all(any(target_arch = "wasm32", target_arch = "wasm64"), target_os = "unknown"))'.dev-dependencies]
+wasm-bindgen-test = "0.3.50"
diff --git a/flake.nix b/flake.nix
index dc2446d..eb79fea 100644
--- a/flake.nix
+++ b/flake.nix
@@ -33,6 +33,8 @@
leptosfmt
rust-analyzer
trunk
+ wasm-pack
+ nodePackages.npm
lld
diff --git a/src/bin/database_worker.rs b/src/bin/database_worker.rs
index 4cecf83..e857e2d 100644
--- a/src/bin/database_worker.rs
+++ b/src/bin/database_worker.rs
@@ -9,6 +9,7 @@ fn main() {
if let Err(e) = a {
log::error!("{:?}", e);
+ panic!("{}", e);
}
})
}
diff --git a/src/database.rs b/src/database.rs
index a4c652c..64f0dd9 100644
--- a/src/database.rs
+++ b/src/database.rs
@@ -4,11 +4,7 @@ pub mod models;
pub mod schema;
mod worker;
-use sqlite_wasm_rs::export::{install_opfs_sahpool, OpfsSAHError};
-use thiserror_ext::AsReport;
-use worker::{DatabaseWorker, Query};
-
-use crate::utils::github_fetch_file;
+use worker::{BrowserDatabaseError, DatabaseWorker, Query};
pub struct BrowserDatabase {
worker: WorkerBridge<DatabaseWorker>,
@@ -34,19 +30,9 @@ impl BrowserDatabase {
/// Initialize the database worker.
///
/// Must be executed in a worker script, not on the main thread.
- pub async fn init_worker() -> Result<(), OpfsSAHError> {
+ pub async fn init_worker() -> Result<(), BrowserDatabaseError> {
log::debug!("Initializing database worker...");
- let sah_pool_util = install_opfs_sahpool(None, false).await?;
-
- let babel_cdb = match github_fetch_file("ProjectIgnis", "BabelCDB", "cards.cdb").await {
- Ok(cdb) => cdb,
- Err(err) => {
- log::error!("{}", err.to_report_string());
- panic!("{:?}", err);
- }
- };
-
- sah_pool_util.import_db("/cards.db", &babel_cdb)?;
+ DatabaseWorker::initialize_db().await?;
DatabaseWorker::registrar().register();
log::debug!("Database worker set up!");
Ok(())
diff --git a/src/database/worker.rs b/src/database/worker.rs
index 9d6916c..7ee7eb2 100644
--- a/src/database/worker.rs
+++ b/src/database/worker.rs
@@ -2,16 +2,22 @@ use diesel::{prelude::*, Connection};
use futures::TryFutureExt;
use gloo::worker::{HandlerId, Worker, WorkerScope};
use serde::{Deserialize, Serialize};
+use sqlite_wasm_rs::export::{install_opfs_sahpool, OpfsSAHPoolUtil};
use thiserror::Error;
use thiserror_ext::AsReport;
+use time::{format_description::well_known::Rfc3339, OffsetDateTime};
use wasm_bindgen_futures::spawn_local;
+use crate::utils::{
+ self, github_fetch_file, github_fetch_last_updated, read_opfs_file, write_opfs_file,
+};
+
use super::{
models::CardDetails,
schema::{datas, texts},
};
-const DB_URL: &str = "file:/cards.db?vfs=opfs-sahpool&mode=ro";
+const DB_URL: &str = "file:/cards.cdb?vfs=opfs-sahpool&mode=ro";
#[derive(Serialize, Deserialize)]
pub enum Query {
@@ -20,6 +26,7 @@ pub enum Query {
#[derive(Serialize, Deserialize, Debug)]
pub enum Response {
+ Update,
ByPassword(Option<CardDetails>),
Error(String),
}
@@ -27,6 +34,13 @@ pub enum Response {
pub struct DatabaseWorker {}
impl DatabaseWorker {
+ pub async fn initialize_db() -> Result<()> {
+ let sah_pool_util = install_opfs_sahpool(None, false).await?;
+ update_card_db(&sah_pool_util).await?;
+
+ Ok(())
+ }
+
async fn run_query(query: &Query) -> Result<Response> {
let res = match query {
Query::ByPassword(pw) => Response::ByPassword(Self::get_card(*pw).await?),
@@ -126,15 +140,58 @@ impl Worker for DatabaseWorker {
}
}
+async fn should_update_card_db() -> Result<bool> {
+ let last_update = {
+ let ts = read_opfs_file("last_update.txt").await?;
+ if !ts.is_empty() {
+ OffsetDateTime::parse(&ts, &Rfc3339).map_err(BrowserDatabaseError::from)
+ } else {
+ Ok(OffsetDateTime::UNIX_EPOCH)
+ }
+ }?;
+
+ log::debug!("Last database update: {}", last_update);
+
+ let latest_version = github_fetch_last_updated("ProjectIgnis", "BabelCDB", "cards.cdb").await?;
+
+ log::debug!("Latest database version: {}", latest_version);
+
+ Ok(last_update < latest_version)
+}
+
+async fn update_card_db(sah_pool_util: &OpfsSAHPoolUtil) -> Result<()> {
+ if should_update_card_db().await? {
+ log::info!("Updating card database...");
+ let db = github_fetch_file("ProjectIgnis", "BabelCDB", "cards.cdb").await?;
+ sah_pool_util.import_db("/cards.cdb", &db)?;
+ write_opfs_file(
+ "last_update.txt",
+ &OffsetDateTime::now_utc().format(&Rfc3339)?,
+ )
+ .await?;
+ } else {
+ log::debug!("Skipping card database updte");
+ }
+
+ Ok(())
+}
+
#[derive(Debug, Error)]
pub enum BrowserDatabaseError {
- // #[error("failed to set up browser database storage")]
- // VFSError(#[from] sqlite_wasm_rs::export::OpfsSAHError),
- // #[error("could not fetch updated card database")]
- // CardDbFetchError(#[from] GithubError),
#[error("failed to connect to database")]
DieselConnectionError(#[from] diesel::ConnectionError),
#[error("failed to query database")]
DieselQueryError(#[from] diesel::result::Error),
+
+ #[error("failed to write database to local storage")]
+ FileStorage(#[from] sqlite_wasm_rs::export::OpfsSAHError),
+ #[error("failed to fetch Babel DB")]
+ GitHubAPI(#[from] crate::utils::GithubError),
+ #[error("failed to retrieve last Babel DB update date")]
+ LocalStorage(#[from] utils::OpfsError),
+ #[error("failed to retrieve last Babel DB update date")]
+ TimeParseError(#[from] time::error::Parse),
+ #[error("failed to write last Babel DB update date")]
+ TimeFormatError(#[from] time::error::Format),
}
type Result<T> = std::result::Result<T, BrowserDatabaseError>;
diff --git a/src/utils.rs b/src/utils.rs
index 31e459d..2be9eaa 100644
--- a/src/utils.rs
+++ b/src/utils.rs
@@ -2,6 +2,13 @@ use gloo::net::http;
use thiserror::Error;
use time::{format_description::well_known::Rfc3339, OffsetDateTime};
+use wasm_bindgen::{JsCast, JsValue};
+use wasm_bindgen_futures::{js_sys, JsFuture};
+use web_sys::{
+ js_sys::Object, FileSystemDirectoryHandle, FileSystemFileHandle, FileSystemGetFileOptions,
+ FileSystemWritableFileStream, WorkerGlobalScope,
+};
+
const GITHUB_API: &str = "https://api.github.com";
pub async fn github_fetch_last_updated(
@@ -58,3 +65,118 @@ pub enum GithubError {
#[error("date/time format reported for GitHub commit is invalid")]
InvalidDateTimeError(#[from] time::error::Parse),
}
+
+/// Read a file from the in-browser OPFS filesystem, creating the file
+/// if it does not exist.
+pub async fn read_opfs_file(path: &str) -> Result<String, OpfsError> {
+ let dir: FileSystemDirectoryHandle = JsFuture::from(
+ js_sys::global()
+ .dyn_into::<WorkerGlobalScope>()
+ .map_err(OpfsError::NotSupported)?
+ .navigator()
+ .storage()
+ .get_directory(),
+ )
+ .await
+ .map_err(OpfsError::HandleAccess)?
+ .into();
+
+ let opt = FileSystemGetFileOptions::new();
+ opt.set_create(true);
+ let file_handle: FileSystemFileHandle =
+ JsFuture::from(dir.get_file_handle_with_options(path, &opt))
+ .await
+ .map_err(OpfsError::HandleAccess)?
+ .into();
+
+ let file: web_sys::File = JsFuture::from(file_handle.get_file())
+ .await
+ .map_err(OpfsError::HandleAccess)?
+ .into();
+
+ Ok(gloo::file::futures::read_as_text(&gloo::file::Blob::from(file)).await?)
+}
+
+/// Write contents to a file in the in-browser OPFS, creating the file
+/// if it does not exist.
+pub async fn write_opfs_file(path: &str, contents: &str) -> Result<(), OpfsError> {
+ let dir: FileSystemDirectoryHandle = JsFuture::from(
+ js_sys::global()
+ .dyn_into::<WorkerGlobalScope>()
+ .map_err(OpfsError::NotSupported)?
+ .navigator()
+ .storage()
+ .get_directory(),
+ )
+ .await
+ .map_err(OpfsError::HandleAccess)?
+ .into();
+
+ let opt = FileSystemGetFileOptions::new();
+ opt.set_create(true);
+ let file_handle: FileSystemFileHandle =
+ JsFuture::from(dir.get_file_handle_with_options(path, &opt))
+ .await
+ .map_err(OpfsError::HandleAccess)?
+ .into();
+
+ let write_handle: FileSystemWritableFileStream = JsFuture::from(file_handle.create_writable())
+ .await
+ .map_err(OpfsError::HandleAccess)?
+ .into();
+
+ JsFuture::from(
+ write_handle
+ .write_with_str(contents)
+ .map_err(OpfsError::WriteFile)?,
+ )
+ .await
+ .map_err(OpfsError::WriteFile)?;
+
+ JsFuture::from(write_handle.close())
+ .await
+ .map_err(OpfsError::WriteFile)?;
+
+ Ok(())
+}
+
+#[derive(Debug, Error)]
+pub enum OpfsError {
+ #[error("OPFS is not supported by this browser: {0:?}")]
+ NotSupported(Object),
+ #[error("could not get OPFS directory path: {0:?}")]
+ HandleAccess(JsValue),
+ #[error("could not read OPFS file")]
+ ReadFile(#[from] gloo::file::FileReadError),
+ #[error("could not write OPFS file: {0:?}")]
+ WriteFile(JsValue),
+}
+
+#[cfg(all(
+ any(target_arch = "wasm32", target_arch = "wasm64"),
+ target_os = "unknown",
+ test
+))]
+mod worker_tests {
+ use wasm_bindgen_test::*;
+ wasm_bindgen_test_configure!(run_in_dedicated_worker);
+
+ use super::{read_opfs_file, write_opfs_file};
+
+ #[wasm_bindgen_test]
+ async fn test_read_write_opfs() {
+ let empty = read_opfs_file("test.txt")
+ .await
+ .expect("file read must succeed");
+ assert_eq!(empty, "");
+
+ write_opfs_file("test.txt", "contents")
+ .await
+ .expect("file write must succeed");
+
+ let nonempty = read_opfs_file("test.txt")
+ .await
+ .expect("file read must succeed");
+ assert_eq!(nonempty, "contents");
+ }
+}