feat(card_db): Only fetch database when upstream has updated

.cargo/config.toml

@@ -0,0 +1,2 @@
+[build]
+target = "wasm32-unknown-unknown"

Cargo.lock

@@ -155,6 +155,15 @@ version = "1.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3"
 
+[[package]]
+name = "cc"
+version = "1.2.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fcb57c740ae1daf453ae85f16e37396f672b039e00d9d866e07ddb24e328e3a"
+dependencies = [
+ "shlex",
+]
+
 [[package]]
 name = "cfg-if"
 version = "1.0.0"
@@ -427,7 +436,9 @@ dependencies = [
  "url",
  "wasm-bindgen",
  "wasm-bindgen-futures",
+ "wasm-bindgen-test",
  "wasm-logger",
+ "web-sys",
 ]
 
 [[package]]
@@ -1328,6 +1339,16 @@ dependencies = [
  "quote",
 ]
 
+[[package]]
+name = "minicov"
+version = "0.3.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f27fe9f1cc3c22e1687f9446c2083c4c5fc7f0bcf1c7a86bdbded14985895b4b"
+dependencies = [
+ "cc",
+ "walkdir",
+]
+
 [[package]]
 name = "minimal-lexical"
 version = "0.2.1"
@@ -1893,6 +1914,12 @@ dependencies = [
  "syn",
 ]
 
+[[package]]
+name = "shlex"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
+
 [[package]]
 name = "slab"
 version = "0.4.9"
@@ -2093,6 +2120,7 @@ checksum = "dad298b01a40a23aac4580b67e3dbedb7cc8402f3592d7f49469de2ea4aecdd8"
 dependencies = [
  "deranged",
  "itoa",
+ "js-sys",
  "num-conv",
  "powerfmt",
  "serde",
@@ -2366,6 +2394,30 @@ dependencies = [
  "unicode-ident",
 ]
 
+[[package]]
+name = "wasm-bindgen-test"
+version = "0.3.50"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "66c8d5e33ca3b6d9fa3b4676d774c5778031d27a578c2b007f905acf816152c3"
+dependencies = [
+ "js-sys",
+ "minicov",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "wasm-bindgen-test-macro",
+]
+
+[[package]]
+name = "wasm-bindgen-test-macro"
+version = "0.3.50"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "17d5042cc5fa009658f9a7333ef24291b1291a25b6382dd68862a7f3b969f69b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "wasm-logger"
 version = "0.2.0"

Cargo.toml

@@ -18,7 +18,7 @@ wasm-bindgen = "0.2.100"
 wasm-bindgen-futures = { version = "0.4.50", features = ["futures-core-03-stream"] }
 diesel-derive-enum = { version = "2.1.0", features = ["sqlite"] }
 diesel_migrations = { git = "https://github.com/diesel-rs/diesel.git", features = ["sqlite"] }
-time = { version = "0.3.39", features = ["parsing"] }
+time = { version = "0.3.39", features = ["formatting", "parsing", "wasm-bindgen"] }
 wasm-logger = "0.2.0"
 log = "0.4.26"
 sqlite-wasm-rs = { version = ">=0.3.0, <0.4.0" , default-features = false, features = ["precompiled"]}
@@ -26,3 +26,7 @@ thiserror-ext = "0.2.1"
 futures-util = "0.3.31"
 gloo = { version = "0.11.0", features = ["futures"] }
 gloo-net = { version = "0.6.0" }
+web-sys = { version = "0.3.77", features = ["FileSystemWritableFileStream", "WorkerGlobalScope"] }
+
+[target.'cfg(all(any(target_arch = "wasm32", target_arch = "wasm64"), target_os = "unknown"))'.dev-dependencies]
+wasm-bindgen-test = "0.3.50"

flake.nix

@@ -33,6 +33,8 @@
           leptosfmt
           rust-analyzer
           trunk
+          wasm-pack
+          nodePackages.npm
 
           lld
 

src/bin/database_worker.rs

@@ -9,6 +9,7 @@ fn main() {
 
         if let Err(e) = a {
             log::error!("{:?}", e);
+            panic!("{}", e);
         }
     })
 }

src/database.rs

@@ -4,11 +4,7 @@ pub mod models;
 pub mod schema;
 mod worker;
 
-use sqlite_wasm_rs::export::{install_opfs_sahpool, OpfsSAHError};
+use worker::{BrowserDatabaseError, DatabaseWorker, Query};
-use thiserror_ext::AsReport;
-use worker::{DatabaseWorker, Query};
-
-use crate::utils::github_fetch_file;
 
 pub struct BrowserDatabase {
     worker: WorkerBridge<DatabaseWorker>,
@@ -34,19 +30,9 @@ impl BrowserDatabase {
     /// Initialize the database worker.
     ///
     /// Must be executed in a worker script, not on the main thread.
-    pub async fn init_worker() -> Result<(), OpfsSAHError> {
+    pub async fn init_worker() -> Result<(), BrowserDatabaseError> {
         log::debug!("Initializing database worker...");
-        let sah_pool_util = install_opfs_sahpool(None, false).await?;
+        DatabaseWorker::initialize_db().await?;
-
-        let babel_cdb = match github_fetch_file("ProjectIgnis", "BabelCDB", "cards.cdb").await {
-            Ok(cdb) => cdb,
-            Err(err) => {
-                log::error!("{}", err.to_report_string());
-                panic!("{:?}", err);
-            }
-        };
-
-        sah_pool_util.import_db("/cards.db", &babel_cdb)?;
         DatabaseWorker::registrar().register();
         log::debug!("Database worker set up!");
         Ok(())

src/database/worker.rs

@@ -2,16 +2,22 @@ use diesel::{prelude::*, Connection};
 use futures::TryFutureExt;
 use gloo::worker::{HandlerId, Worker, WorkerScope};
 use serde::{Deserialize, Serialize};
+use sqlite_wasm_rs::export::{install_opfs_sahpool, OpfsSAHPoolUtil};
 use thiserror::Error;
 use thiserror_ext::AsReport;
+use time::{format_description::well_known::Rfc3339, OffsetDateTime};
 use wasm_bindgen_futures::spawn_local;
 
+use crate::utils::{
+    self, github_fetch_file, github_fetch_last_updated, read_opfs_file, write_opfs_file,
+};
+
 use super::{
     models::CardDetails,
     schema::{datas, texts},
 };
 
-const DB_URL: &str = "file:/cards.db?vfs=opfs-sahpool&mode=ro";
+const DB_URL: &str = "file:/cards.cdb?vfs=opfs-sahpool&mode=ro";
 
 #[derive(Serialize, Deserialize)]
 pub enum Query {
@@ -20,6 +26,7 @@ pub enum Query {
 
 #[derive(Serialize, Deserialize, Debug)]
 pub enum Response {
+    Update,
     ByPassword(Option<CardDetails>),
     Error(String),
 }
@@ -27,6 +34,13 @@ pub enum Response {
 pub struct DatabaseWorker {}
 
 impl DatabaseWorker {
+    pub async fn initialize_db() -> Result<()> {
+        let sah_pool_util = install_opfs_sahpool(None, false).await?;
+        update_card_db(&sah_pool_util).await?;
+
+        Ok(())
+    }
+
     async fn run_query(query: &Query) -> Result<Response> {
         let res = match query {
             Query::ByPassword(pw) => Response::ByPassword(Self::get_card(*pw).await?),
@@ -126,15 +140,58 @@ impl Worker for DatabaseWorker {
     }
 }
 
+async fn should_update_card_db() -> Result<bool> {
+    let last_update = {
+        let ts = read_opfs_file("last_update.txt").await?;
+        if !ts.is_empty() {
+            OffsetDateTime::parse(&ts, &Rfc3339).map_err(BrowserDatabaseError::from)
+        } else {
+            Ok(OffsetDateTime::UNIX_EPOCH)
+        }
+    }?;
+
+    log::debug!("Last database update: {}", last_update);
+
+    let latest_version = github_fetch_last_updated("ProjectIgnis", "BabelCDB", "cards.cdb").await?;
+
+    log::debug!("Latest database version: {}", latest_version);
+
+    Ok(last_update < latest_version)
+}
+
+async fn update_card_db(sah_pool_util: &OpfsSAHPoolUtil) -> Result<()> {
+    if should_update_card_db().await? {
+        log::info!("Updating card database...");
+        let db = github_fetch_file("ProjectIgnis", "BabelCDB", "cards.cdb").await?;
+        sah_pool_util.import_db("/cards.cdb", &db)?;
+        write_opfs_file(
+            "last_update.txt",
+            &OffsetDateTime::now_utc().format(&Rfc3339)?,
+        )
+        .await?;
+    } else {
+        log::debug!("Skipping card database updte");
+    }
+
+    Ok(())
+}
+
 #[derive(Debug, Error)]
 pub enum BrowserDatabaseError {
-    // #[error("failed to set up browser database storage")]
-    // VFSError(#[from] sqlite_wasm_rs::export::OpfsSAHError),
-    // #[error("could not fetch updated card database")]
-    // CardDbFetchError(#[from] GithubError),
     #[error("failed to connect to database")]
     DieselConnectionError(#[from] diesel::ConnectionError),
     #[error("failed to query database")]
     DieselQueryError(#[from] diesel::result::Error),
+
+    #[error("failed to write database to local storage")]
+    FileStorage(#[from] sqlite_wasm_rs::export::OpfsSAHError),
+    #[error("failed to fetch Babel DB")]
+    GitHubAPI(#[from] crate::utils::GithubError),
+    #[error("failed to retrieve last Babel DB update date")]
+    LocalStorage(#[from] utils::OpfsError),
+    #[error("failed to retrieve last Babel DB update date")]
+    TimeParseError(#[from] time::error::Parse),
+    #[error("failed to write last Babel DB update date")]
+    TimeFormatError(#[from] time::error::Format),
 }
 type Result<T> = std::result::Result<T, BrowserDatabaseError>;

src/utils.rs

@@ -2,6 +2,13 @@ use gloo::net::http;
 use thiserror::Error;
 use time::{format_description::well_known::Rfc3339, OffsetDateTime};
 
+use wasm_bindgen::{JsCast, JsValue};
+use wasm_bindgen_futures::{js_sys, JsFuture};
+use web_sys::{
+    js_sys::Object, FileSystemDirectoryHandle, FileSystemFileHandle, FileSystemGetFileOptions,
+    FileSystemWritableFileStream, WorkerGlobalScope,
+};
+
 const GITHUB_API: &str = "https://api.github.com";
 
 pub async fn github_fetch_last_updated(
@@ -58,3 +65,118 @@ pub enum GithubError {
     #[error("date/time format reported for GitHub commit is invalid")]
     InvalidDateTimeError(#[from] time::error::Parse),
 }
+
+/// Read a file from the in-browser OPFS filesystem, creating the file
+/// if it does not exist.
+pub async fn read_opfs_file(path: &str) -> Result<String, OpfsError> {
+    let dir: FileSystemDirectoryHandle = JsFuture::from(
+        js_sys::global()
+            .dyn_into::<WorkerGlobalScope>()
+            .map_err(OpfsError::NotSupported)?
+            .navigator()
+            .storage()
+            .get_directory(),
+    )
+    .await
+    .map_err(OpfsError::HandleAccess)?
+    .into();
+
+    let opt = FileSystemGetFileOptions::new();
+    opt.set_create(true);
+    let file_handle: FileSystemFileHandle =
+        JsFuture::from(dir.get_file_handle_with_options(path, &opt))
+            .await
+            .map_err(OpfsError::HandleAccess)?
+            .into();
+
+    let file: web_sys::File = JsFuture::from(file_handle.get_file())
+        .await
+        .map_err(OpfsError::HandleAccess)?
+        .into();
+
+    Ok(gloo::file::futures::read_as_text(&gloo::file::Blob::from(file)).await?)
+}
+
+/// Write contents to a file in the in-browser OPFS, creating the file
+/// if it does not exist.
+pub async fn write_opfs_file(path: &str, contents: &str) -> Result<(), OpfsError> {
+    let dir: FileSystemDirectoryHandle = JsFuture::from(
+        js_sys::global()
+            .dyn_into::<WorkerGlobalScope>()
+            .map_err(OpfsError::NotSupported)?
+            .navigator()
+            .storage()
+            .get_directory(),
+    )
+    .await
+    .map_err(OpfsError::HandleAccess)?
+    .into();
+
+    let opt = FileSystemGetFileOptions::new();
+    opt.set_create(true);
+    let file_handle: FileSystemFileHandle =
+        JsFuture::from(dir.get_file_handle_with_options(path, &opt))
+            .await
+            .map_err(OpfsError::HandleAccess)?
+            .into();
+
+    let write_handle: FileSystemWritableFileStream = JsFuture::from(file_handle.create_writable())
+        .await
+        .map_err(OpfsError::HandleAccess)?
+        .into();
+
+    JsFuture::from(
+        write_handle
+            .write_with_str(contents)
+            .map_err(OpfsError::WriteFile)?,
+    )
+    .await
+    .map_err(OpfsError::WriteFile)?;
+
+    JsFuture::from(write_handle.close())
+        .await
+        .map_err(OpfsError::WriteFile)?;
+
+    Ok(())
+}
+
+#[derive(Debug, Error)]
+pub enum OpfsError {
+    #[error("OPFS is not supported by this browser: {0:?}")]
+    NotSupported(Object),
+    #[error("could not get OPFS directory path: {0:?}")]
+    HandleAccess(JsValue),
+    #[error("could not read OPFS file")]
+    ReadFile(#[from] gloo::file::FileReadError),
+    #[error("could not write OPFS file: {0:?}")]
+    WriteFile(JsValue),
+}
+
+#[cfg(all(
+    any(target_arch = "wasm32", target_arch = "wasm64"),
+    target_os = "unknown",
+    test
+))]
+mod worker_tests {
+    use wasm_bindgen_test::*;
+    wasm_bindgen_test_configure!(run_in_dedicated_worker);
+
+    use super::{read_opfs_file, write_opfs_file};
+
+    #[wasm_bindgen_test]
+    async fn test_read_write_opfs() {
+        let empty = read_opfs_file("test.txt")
+            .await
+            .expect("file read must succeed");
+        assert_eq!(empty, "");
+
+        write_opfs_file("test.txt", "contents")
+            .await
+            .expect("file write must succeed");
+
+        let nonempty = read_opfs_file("test.txt")
+            .await
+            .expect("file read must succeed");
+        assert_eq!(nonempty, "contents");
+    }
+}