add append (previously concat) with most comments resolved

Author: Johanna

src/cli.rs

@@ -161,9 +161,9 @@ pub enum Commands {
         #[arg(required = true, short)]
         output: String,
     },
-    /// Concat one sketch file (.skm and .skd pair) with new genomes (seq_files or files_list)
-    Concat {
-        /// The first .skd (sketch data) file
+    /// Append new genomes to be sketched to an existing sketch database
+    Append {
+        /// Sketching database basename (so without .skm or .skd)
         #[arg(required = true)]
         db: String,
 

src/lib.rs

@@ -397,7 +397,7 @@ pub fn main() -> Result<(), Error> {
             utils::save_sketch_data(ref_db_name1, ref_db_name2, output)
         }
 
-        Commands::Concat {
+        Commands::Append {
             db,
             seq_files,
             file_list,
@@ -409,6 +409,8 @@ pub fn main() -> Result<(), Error> {
             threads,
             level,
         } => {
+            // An extra thread is needed for the writer. This doesn't 'overuse' CPU
+            check_threads(*threads + 1);
             //get input files
             log::info!("Getting input files");
             let input_files: Vec<(String, String, Option<String>)> =
@@ -417,10 +419,9 @@ pub fn main() -> Result<(), Error> {
 
             //check if any of the new files are already existant in the db
             let db_metadata: MultiSketch = MultiSketch::load(db)
-                .unwrap_or_else(|_| panic!("Could not read sketch metadata from {}.skm", db));
-            println!("{:?}", db_metadata);
+                .expect(&format!("Could not read sketch metadata from .skm: {}", db));
 
-            if !db_metadata.concat_competibility(&input_files) {
+            if !db_metadata.append_compatibility(&input_files) {
                 panic!("Databases are not compatible for merging.")
             }
             log::info!("Passed concat check");
@@ -436,7 +437,7 @@ pub fn main() -> Result<(), Error> {
 
             log::info!(
                 "Running sketching: k:{:?}; sketch_size:{}; seq:{:?}; threads:{}",
-                &kmers,
+                kmers,
                 sketch_size * u64::BITS as u64,
                 seq_type,
                 threads,
@@ -452,7 +453,7 @@ pub fn main() -> Result<(), Error> {
                 output,
                 &input_files,
                 *concat_fasta,
-                &kmers,
+                kmers,
                 sketch_size,
                 &seq_type,
                 rc,
@@ -479,7 +480,7 @@ pub fn main() -> Result<(), Error> {
             let concat_metadata = db2_metadata.merge_sketches(&db_metadata);
             concat_metadata
                 .save_metadata(output)
-                .unwrap_or_else(|_| panic!("Couldn't save metadata to {}", output));
+                .expect(&format!("Could not save metadata to {}", output));
             Ok(())
         }
 

src/multisketch.rs

@@ -158,14 +158,21 @@ impl MultiSketch {
             && self.get_hash_type() == sketch2.get_hash_type()
     }
 
-    pub fn concat_competibility(&self, name_vec: &[(String, String, Option<String>)]) -> bool {
+    pub fn append_compatibility(&self, name_vec: &[(String, String, Option<String>)]) -> bool {
+        let mut compatibility = true;
+        let mut duplicate_list = Vec::new();
         for (id, _, _) in name_vec.iter() {
             if self.name_map.contains_key(id) {
-                println!("{} is found in both database and input fasta", id);
-                return false;
+                duplicate_list.push(id);
+                compatibility = false;
             }
         }
-        true
+
+        if !duplicate_list.is_empty() {
+            println!("Duplicates found: {:?}", duplicate_list);
+        }
+
+        compatibility
     }
 
     pub fn merge_sketches(&mut self, sketch2: &Self) -> &mut Self {

tests/concat.rs

@@ -4,48 +4,16 @@ use std::path::Path;
 
 pub mod common;
 use crate::common::*;
-
-#[path = "../src/io.rs"]
-pub mod io;
+use sketchlib::io::*;
 
 use sketchlib::multisketch::MultiSketch;
 
 #[cfg(test)]
 
 mod tests {
+    use sketchlib::io;
     use super::*;
-
-    #[test]
-    fn concat_competibility_test() {
-        let sandbox = TestSetup::setup();
-        let ref_db1 = sandbox.file_string("sketches1", TestDir::Input);
-        let ref_db2 = sandbox.file_string("sketches2", TestDir::Input);
-        let file_list_name = sandbox.file_string("fasta.txt", TestDir::Input);
-        let non_seq_files: Option<Vec<String>> = None;
-
-
-        let file_list: Option<String> = Some(file_list_name);
-        //get input files with file_list
-        log::info!("Getting input files for test");
-        let input_file_list = io::get_input_list(&file_list, &non_seq_files);
-        log::info!("Parsed {} samples in input list", input_file_list.len());
-
-        //check if any of the new files are already existant in the db
-        let db1_metadata: MultiSketch = MultiSketch::load(&ref_db1)
-            .unwrap_or_else(|_| panic!("Could not read sketch metadata from {}.skm", ref_db1));
-        println!("{:?}", db1_metadata);
-
-        //check if any of the new files are already existant in the db
-        let db2_metadata: MultiSketch = MultiSketch::load(&ref_db2)
-            .unwrap_or_else(|_| panic!("Could not read sketch metadata from {}.skm", ref_db2));
-        println!("{:?}", db2_metadata);
-
-        // Test case 1:
-        assert!(
-            !db1_metadata.concat_competibility(&input_file_list),
-            "Sketches should not be compatible"
-        );
-    }
+   
 
     #[test]
     fn test_concat_sketches() {
@@ -62,7 +30,6 @@ mod tests {
             .args(&["-o", "part1"])
             .assert()
             .success();
-        log::info!("Part 1 Sketched");
 
         Command::new(cargo_bin("sketchlib"))
             .current_dir(sandbox.get_wd())
@@ -75,7 +42,6 @@ mod tests {
             .args(&["-o", "part2"])
             .assert()
             .success();
-        log::info!("Part 2 Sketched");
 
         Command::new(cargo_bin("sketchlib"))
             .current_dir(sandbox.get_wd())
@@ -90,7 +56,6 @@ mod tests {
             .args(&["-o", "concat_ref"])
             .assert()
             .success();
-        log::info!("concat_ref Sketched");
 
         // Overlapping labels fails
         Command::new(cargo_bin("sketchlib"))
@@ -107,7 +72,7 @@ mod tests {
 
         Command::new(cargo_bin("sketchlib"))
             .current_dir(sandbox.get_wd())
-            .arg("concat")
+            .arg("append")
             .arg("part1")
             .arg("--seq-files")
             // .arg(sandbox.file_string("fasta_part2.txt", TestDir::Input))
@@ -117,7 +82,6 @@ mod tests {
             .args(&["-o", "concat_test"])
             .assert()
             .success();
-        log::info!("concat_test Sketched");
 
         // Check .skm the same
         let concat_sketch: MultiSketch =