[all resource export] add content verify

Author: welbon

Cargo.lock

@@ -11,6 +11,7 @@ rust-version = { workspace = true }
 
 [dependencies]
 anyhow = { workspace = true }
+bcs-ext = { workspace = true }
 clap = { workspace = true }
 csv = { workspace = true }
 serde_json = { workspace = true }
@@ -19,6 +20,7 @@ starcoin-statedb = { workspace = true }
 starcoin-storage = { workspace = true }
 starcoin-types = { workspace = true }
 tempfile = "3.19.1"
+log = "0.4.27"
 
 
 [dev-dependencies]

cmd/resource-code-exporter/Cargo.toml

@@ -6,7 +6,7 @@ use starcoin_statedb::{ChainStateDB, ChainStateReader};
 use starcoin_storage::{
     db_storage::DBStorage, storage::StorageInstance, BlockStore, Storage, StorageVersion,
 };
- use std::{io::Write, path::Path, sync::Arc};
+use std::{io::Write, path::Path, sync::Arc};
 
 /// Export resources and code from storage for a specific block
 pub fn export(db: &str, output: &Path, block_id: HashValue) -> anyhow::Result<()> {
@@ -30,22 +30,23 @@ pub fn export(db: &str, output: &Path, block_id: HashValue) -> anyhow::Result<()
 
     // Create writer and export
     let mut csv_writer = csv::WriterBuilder::new().from_path(output)?;
-    export_from_statedb(&statedb, root, &mut csv_writer)?;
+    export_from_statedb(&statedb, &mut csv_writer)?;
 
     Ok(())
 }
 
 /// Export resources and code from StateDB to a writer
 pub fn export_from_statedb<W: Write>(
     statedb: &ChainStateDB,
-    root: HashValue,
     writer: &mut csv::Writer<W>,
 ) -> anyhow::Result<()> {
     // write csv header
     {
         writer.write_field("address")?;
-        writer.write_field("state_root")?;
-        writer.write_field("account_state")?;
+        writer.write_field("code_blob_hash")?;
+        writer.write_field("code_blob")?;
+        writer.write_field("resource_blob_hash")?;
+        writer.write_field("resrouce_blob")?;
         writer.write_record(None::<&[u8]>)?;
     }
 
@@ -56,15 +57,39 @@ pub fn export_from_statedb<W: Write>(
     let now = Instant::now();
     let mut processed = 0;
 
-    for (account_address, account_state) in global_states.into_iter() {
-        // Serialize the entire account state
-        let account_state_json = serde_json::to_string(&account_state)?;
+    for (account_address, account_state_set) in global_states.into_iter() {
+        // Process codes
+        let (code_state_set_hash, code_state_set) = match account_state_set.code_set() {
+            Some(state_set) => {
+                let code_state_set = serde_json::to_string(&state_set).unwrap();
+                (
+                    HashValue::sha3_256_of(&code_state_set.as_bytes()).to_hex_literal(),
+                    code_state_set,
+                )
+            }
+            None => (String::new(), String::new()),
+        };
+
+        // Process resources
+        let (resource_state_set_hash, resource_state_set) = match account_state_set.resource_set() {
+            Some(state_set) => {
+                let resource_state_set = serde_json::to_string(&state_set).unwrap();
+                (
+                    HashValue::sha3_256_of(&resource_state_set.as_bytes()).to_hex_literal(),
+                    resource_state_set,
+                )
+            }
+            None => (String::new(), String::new()),
+        };
 
         // write csv record
         let record = vec![
+            // account address
             serde_json::to_string(&account_address)?,
-            serde_json::to_string(&root)?,
-            account_state_json,
+            code_state_set_hash,
+            code_state_set,
+            resource_state_set_hash,
+            resource_state_set,
         ];
 
         writer.serialize(record)?;
@@ -93,7 +118,7 @@ mod test {
         let mut buffer = Cursor::new(Vec::new());
         {
             let mut csv_writer = csv::WriterBuilder::new().from_writer(&mut buffer);
-            export_from_statedb(&chain_statedb, chain_statedb.state_root(), &mut csv_writer)?;
+            export_from_statedb(&chain_statedb, &mut csv_writer)?;
         }
 
         // Get the written data

cmd/resource-code-exporter/src/export.rs

@@ -4,14 +4,15 @@
 use starcoin_crypto::HashValue;
 use starcoin_statedb::{ChainStateDB, ChainStateReader, ChainStateWriter};
 use starcoin_storage::{db_storage::DBStorage, storage::StorageInstance, Storage, StorageVersion};
+use starcoin_types::state_set::StateSet;
 use starcoin_types::{
     account_address::AccountAddress,
     state_set::{AccountStateSet, ChainStateSet},
 };
 use std::path::Path;
 use std::sync::Arc;
 
-pub fn import(csv_path: &Path, db_path: &Path) -> anyhow::Result<()> {
+pub fn import(csv_path: &Path, db_path: &Path, expect_root_hash: HashValue) -> anyhow::Result<()> {
     let db_storage = DBStorage::open_with_cfs(
         db_path,
         StorageVersion::current_version()
@@ -24,42 +25,63 @@ pub fn import(csv_path: &Path, db_path: &Path) -> anyhow::Result<()> {
     let storage = Storage::new(StorageInstance::new_db_instance(db_storage))?;
     let storage = Arc::new(storage);
     let statedb = ChainStateDB::new(storage.clone(), None);
-    import_from_statedb(&statedb, csv_path)
+    import_from_statedb(&statedb, csv_path, expect_root_hash)
 }
 
 /// Import resources and code from CSV file to a new statedb
-pub fn import_from_statedb(statedb: &ChainStateDB, csv_path: &Path) -> anyhow::Result<()> {
+pub fn import_from_statedb(
+    statedb: &ChainStateDB,
+    csv_path: &Path,
+    expect_state_root_hash: HashValue,
+) -> anyhow::Result<()> {
     // Read CSV file
     let mut csv_reader = csv::Reader::from_path(csv_path)?;
-    let mut expected_state_root = None;
-    let mut state_sets = Vec::new();
+    let mut chain_state_set_data = Vec::new();
 
     for result in csv_reader.records() {
         let record = result?;
-        let address: AccountAddress = serde_json::from_str(&record[0])?;
-        let state_root: HashValue = serde_json::from_str(&record[1])?;
-        let account_state: AccountStateSet = serde_json::from_str(&record[2])?;
+        let account_address: AccountAddress = serde_json::from_str(&record[0])?;
+        println!("record len: {:?}", record.len());
 
-        // Store the first state root as expected
-        if expected_state_root.is_none() {
-            expected_state_root = Some(state_root);
-        }
+        let code_state_set = if !record[1].is_empty() && !record[2].is_empty() {
+            let code_state_hash: String = serde_json::from_str(&record[1])?;
+            let code_state_set_str = &record[2];
+            assert_eq!(
+                code_state_hash,
+                HashValue::sha3_256_of(code_state_set_str.as_bytes()).to_hex_literal()
+            );
+            Some(serde_json::from_str::<StateSet>(code_state_set_str)?)
+        } else {
+            None
+        };
+
+        let resource_state_set = if !record[3].is_empty() && !record[4].is_empty() {
+            let resrouce_blob_hash: String = serde_json::from_str(&record[3])?;
+            let resource_state_set_str = &record[4];
+            assert_eq!(
+                resrouce_blob_hash,
+                HashValue::sha3_256_of(resource_state_set_str.as_bytes()).to_hex_literal()
+            );
+            Some(serde_json::from_str(resource_state_set_str)?)
+        } else {
+            None
+        };
 
-        // Add to state sets
-        state_sets.push((address, account_state));
+        chain_state_set_data.push((
+            account_address,
+            AccountStateSet::new(vec![code_state_set, resource_state_set]),
+        ));
     }
 
-    // Create chain state set and apply it
-    let chain_state_set = ChainStateSet::new(state_sets);
-    statedb.apply(chain_state_set)?;
+    statedb.apply(ChainStateSet::new(chain_state_set_data))?;
 
     // Get new state root
     let new_state_root = statedb.state_root();
 
     // Verify state root matches
-    if let Some(expected) = expected_state_root {
+    {
         assert_eq!(
-            new_state_root, expected,
+            expect_state_root_hash, new_state_root,
             "Imported state root does not match expected state root"
         );
         println!("Import successful! State root: {}", new_state_root);
@@ -91,7 +113,7 @@ mod test {
         // Export data
         {
             let mut csv_writer = csv::WriterBuilder::new().from_path(&export_path)?;
-            export_from_statedb(&export_chain_statedb, export_state_root, &mut csv_writer)?;
+            export_from_statedb(&export_chain_statedb, &mut csv_writer)?;
         }
 
         //////////////////////////////////////////////////////
@@ -113,14 +135,7 @@ mod test {
         let storage = Storage::new(StorageInstance::new_db_instance(db_storage))?;
         let storage = Arc::new(storage);
         let imported_statedb = ChainStateDB::new(storage.clone(), None);
-        import_from_statedb(&imported_statedb, &export_path)?;
-
-        // Verify state root matches
-        assert_eq!(
-            imported_statedb.state_root(),
-            export_state_root,
-            "Imported state root does not match genesis state root"
-        );
+        import_from_statedb(&imported_statedb, &export_path, export_state_root)?;
 
         Ok(())
     }

cmd/resource-code-exporter/src/import.rs

@@ -42,6 +42,10 @@ enum Commands {
         #[clap(long, short = 'd', parse(from_os_str))]
         /// Output database path
         db_path: PathBuf,
+
+        #[clap(long)]
+        /// expect state root hash
+        state_root: starcoin_crypto::HashValue,
     },
 }
 
@@ -56,8 +60,12 @@ fn main() -> anyhow::Result<()> {
         } => {
             export::export(db_path.display().to_string().as_str(), &output, block_id)?;
         }
-        Commands::Import { csv_input, db_path } => {
-            import::import(&csv_input, &db_path)?;
+        Commands::Import {
+            csv_input,
+            db_path,
+            state_root,
+        } => {
+            import::import(&csv_input, &db_path, state_root)?;
         }
     }
 

cmd/resource-code-exporter/src/main.rs

@@ -13,6 +13,7 @@ starcoin-types = { workspace = true }
 starcoin-vm-types = { workspace = true }
 thiserror = { workspace = true }
 once_cell = "1.17.1"
+rayon = "1.8"
 
 [package]
 authors = { workspace = true }

state/statedb/Cargo.toml

@@ -22,7 +22,7 @@ use starcoin_types::{
     access_path::{AccessPath, DataType},
     account_address::AccountAddress,
     account_state::AccountState,
-    state_set::{AccountStateSet, ChainStateSet},
+    state_set::{AccountStateSet, ChainStateSet, StateSet},
 };
 use starcoin_vm_types::access_path::{DataPath, ModuleName};
 #[cfg(test)]
@@ -789,5 +789,6 @@ impl TableHandleStateObject {
     }
 }
 
+mod parallel_state_set;
 #[cfg(test)]
 mod tests;