Migrate trailing slash base URLs #895

Author: joepio

lib/src/db.rs

@@ -16,6 +16,7 @@ use std::{
 };
 
 use mail_send::{Connected, Transport};
+use migrations::{REFERENCE_INDEX_CURRENT, RESOURCE_TREE_CURRENT};
 use tracing::{info, instrument};
 
 use crate::{
@@ -100,8 +101,8 @@ impl Db {
     /// It is used for distinguishing locally defined items from externally defined ones.
     pub fn init(path: &std::path::Path, server_url: &str) -> AtomicResult<Db> {
         let db = sled::open(path).map_err(|e|format!("Failed opening DB at this location: {:?} . Is another instance of Atomic Server running? {}", path, e))?;
-        let resources = db.open_tree("resources_v1").map_err(|e|format!("Failed building resources. Your DB might be corrupt. Go back to a previous version and export your data. {}", e))?;
-        let reference_index = db.open_tree("reference_index_v1")?;
+        let resources = db.open_tree(RESOURCE_TREE_CURRENT).map_err(|e|format!("Failed building resources. Your DB might be corrupt. Go back to a previous version and export your data. {}", e))?;
+        let reference_index = db.open_tree(REFERENCE_INDEX_CURRENT)?;
         let query_index = db.open_tree("members_index")?;
         let prop_val_sub_index = db.open_tree("prop_val_sub_index")?;
         let watched_queries = db.open_tree("watched_queries")?;

lib/src/db/migrations.rs

@@ -12,33 +12,42 @@ Therefore, we need migrations to convert the old schema to the new one.
 - Update the Tree key used in [Db::init]
  */
 
-use crate::{errors::AtomicResult, Db, Storelike};
+use crate::{errors::AtomicResult, resources::PropVals, Db, Storelike};
+
+const RESOURCE_TREE_V0: &str = "resources";
+const RESOURCE_TREE_V1: &str = "resources_v1";
+const RESOURCE_TREE_V2: &str = "resources_v2";
+pub const RESOURCE_TREE_CURRENT: &str = RESOURCE_TREE_V2;
+
+const REFERENCE_INDEX_V0: &str = "reference_index";
+const REFERENCE_INDEX_V1: &str = "reference_index_v1";
+pub const REFERENCE_INDEX_CURRENT: &str = REFERENCE_INDEX_V1;
 
 /// Checks the current version(s) of the internal Store, and performs migrations if needed.
 pub fn migrate_maybe(store: &Db) -> AtomicResult<()> {
     for tree in store.db.tree_names() {
         match String::from_utf8_lossy(&tree).as_ref() {
             // Add migrations for outdated Trees to this list
-            "resources" => v0_to_v1(store)?,
-            "reference_index" => ref_v0_to_v1(store)?,
+            RESOURCE_TREE_V0 => res_v0_to_v1(store)?,
+            RESOURCE_TREE_V1 => res_v1_to_v2(store)?,
+            REFERENCE_INDEX_V0 => ref_v0_to_v1(store)?,
             _other => {}
         }
     }
     Ok(())
 }
 
 /// Change the subjects from `bincode` to `.as_bytes()`
-fn v0_to_v1(store: &Db) -> AtomicResult<()> {
+fn res_v0_to_v1(store: &Db) -> AtomicResult<()> {
     tracing::warn!("Migrating resources schema from v0 to v1...");
-    let new = store.db.open_tree("resources_v1")?;
-    let old_key = "resources";
+    let new = store.db.open_tree(RESOURCE_TREE_V1)?;
+    let old_key = RESOURCE_TREE_V0;
     let old = store.db.open_tree(old_key)?;
     let mut count = 0;
 
     for item in old.into_iter() {
         let (subject, resource_bin) = item.expect("Unable to convert into iterable");
-        let subject: String =
-            bincode::deserialize(&subject).expect("Unable to deserialize subject");
+        let subject: String = String::from_utf8_lossy(&subject).to_string();
         new.insert(subject.as_bytes(), resource_bin)?;
         count += 1;
     }
@@ -73,6 +82,72 @@ fn v0_to_v1(store: &Db) -> AtomicResult<()> {
     Ok(())
 }
 
+/// add a trailing slash to all "home" subjects
+fn res_v1_to_v2(store: &Db) -> AtomicResult<()> {
+    tracing::warn!("Migrating resources schema from v1 to v2...");
+    let new = store.db.open_tree(RESOURCE_TREE_V2)?;
+    let old_key = RESOURCE_TREE_V1;
+    let old = store.db.open_tree(old_key)?;
+    let mut count = 0;
+
+    fn migrate_subject(subject: &str) -> String {
+        let url = url::Url::parse(subject).expect("Unable to parse subject URL");
+        if subject != url.to_string() {
+            println!("Migrating: {} -> {}", subject, url.to_string())
+        };
+        url.to_string()
+    }
+
+    for item in old.into_iter() {
+        let (subject, resource_bin) = item.expect("Unable to convert into iterable");
+        let subject: String = String::from_utf8_lossy(&subject).to_string();
+
+        let mut propvals: PropVals = bincode::deserialize(&resource_bin)?;
+
+        for (_prop, val) in propvals.iter_mut() {
+            match val {
+                crate::Value::AtomicUrl(a) => {
+                    *a = migrate_subject(a);
+                }
+                crate::Value::ResourceArray(arr) => {
+                    for url in arr.iter_mut() {
+                        match url {
+                            crate::values::SubResource::Subject(s) => {
+                                *s = migrate_subject(s);
+                            }
+                            // This skips nested resources
+                            _other => {}
+                        }
+                    }
+                }
+                // This skips nested resources
+                _other => {}
+            };
+        }
+        new.insert(migrate_subject(&subject), bincode::serialize(&propvals)?)?;
+        count += 1;
+    }
+
+    assert_eq!(
+        new.len(),
+        store.resources.len(),
+        "Not all resources were migrated."
+    );
+
+    assert!(
+        store.db.drop_tree(old_key)?,
+        "Old resources tree not properly removed."
+    );
+
+    tracing::warn!("Rebuilding indexes due to migrating to new version...");
+    store.db.drop_tree(old_key)?;
+    store.build_index(true)?;
+    tracing::warn!("Rebuilding index finished!");
+
+    tracing::warn!("Finished migration of {} resources", count);
+    Ok(())
+}
+
 /// Add `prop_val_sub` index
 fn ref_v0_to_v1(store: &Db) -> AtomicResult<()> {
     tracing::warn!("Rebuilding indexes due to migrating to new version...");