Code clean up and more tests

Author: imotov

columnar/src/dynamic_column.rs

@@ -83,19 +83,6 @@ impl DynamicColumn {
         }
     }
 
-    pub fn num_docs(&self) -> u32 {
-        match self {
-            DynamicColumn::Bool(c) => c.num_docs(),
-            DynamicColumn::I64(c) => c.num_docs(),
-            DynamicColumn::U64(c) => c.num_docs(),
-            DynamicColumn::F64(c) => c.num_docs(),
-            DynamicColumn::IpAddr(c) => c.num_docs(),
-            DynamicColumn::DateTime(c) => c.num_docs(),
-            DynamicColumn::Bytes(c) => c.ords().num_docs(),
-            DynamicColumn::Str(c) => c.ords().num_docs(),
-        }
-    }
-
     pub fn coerce_numerical(self, target_numerical_type: NumericalType) -> Option<Self> {
         match target_numerical_type {
             NumericalType::I64 => self.coerce_to_i64(),

src/fastfield/readers.rs

@@ -354,6 +354,8 @@ impl FastFieldReaders {
 
 #[cfg(test)]
 mod tests {
+    use columnar::ColumnType;
+
     use crate::schema::{JsonObjectOptions, Schema, FAST};
     use crate::{Document, Index};
 
@@ -433,4 +435,45 @@ mod tests {
             Some("_dyna\u{1}notinschema\u{1}attr\u{1}color".to_string())
         );
     }
+
+    #[test]
+    fn test_fast_field_reader_dynamic_column_handles() {
+        let mut schema_builder = Schema::builder();
+        let id = schema_builder.add_u64_field("id", FAST);
+        let json = schema_builder.add_json_field("json", FAST);
+        let schema = schema_builder.build();
+        let index = Index::create_in_ram(schema);
+        let mut index_writer = index.writer_for_tests().unwrap();
+        index_writer
+            .add_document(doc!(id=> 1u64, json => json!({"foo": 42})))
+            .unwrap();
+        index_writer
+            .add_document(doc!(id=> 2u64, json => json!({"foo": true})))
+            .unwrap();
+        index_writer
+            .add_document(doc!(id=> 3u64, json => json!({"foo": "bar"})))
+            .unwrap();
+        index_writer.commit().unwrap();
+        let reader = index.reader().unwrap();
+        let searcher = reader.searcher();
+        let reader = searcher.segment_reader(0u32);
+        let fast_fields = reader.fast_fields();
+        let id_columns = fast_fields.dynamic_column_handles("id").unwrap();
+        assert_eq!(id_columns.len(), 1);
+        assert_eq!(id_columns.first().unwrap().column_type(), ColumnType::U64);
+
+        let foo_columns = fast_fields.dynamic_column_handles("json.foo").unwrap();
+        assert_eq!(foo_columns.len(), 3);
+        assert!(foo_columns
+            .iter()
+            .any(|column| column.column_type() == ColumnType::I64));
+        assert!(foo_columns
+            .iter()
+            .any(|column| column.column_type() == ColumnType::Bool));
+        assert!(foo_columns
+            .iter()
+            .any(|column| column.column_type() == ColumnType::Str));
+
+        println!("*** {:?}", fast_fields.columnar().list_columns());
+    }
 }

src/query/exist_query.rs

@@ -9,19 +9,19 @@ use crate::query::explanation::does_not_match;
 use crate::query::{EnableScoring, Explanation, Query, Scorer, Weight};
 use crate::{DocId, Score, TantivyError};
 
-/// Query that matches all of the documents.
+/// Query that matches all documents with a non-null value in the specified field.
 ///
-/// All of the document get the score 1.0.
+/// All of the matched documents get the score 1.0.
 #[derive(Clone, Debug)]
 pub struct ExistsQuery {
     field_name: String,
 }
 
 impl ExistsQuery {
-    /// Creates a new `RangeQuery` from bounded start and end terms.
+    #![allow(dead_code)]
+    /// Creates a new `ExistQuery` from the given field.
     ///
-    /// If the value type is not correct, something may go terribly wrong when
-    /// the `Weight` object is created.
+    /// This query mathces all documents with at least one non-null value in the specified field.
     pub fn new_exists_query(field: String) -> ExistsQuery {
         ExistsQuery { field_name: field }
     }
@@ -69,9 +69,9 @@ impl Weight for ExistsWeight {
         }
         if !non_empty_columns.is_empty() {
             let docset = ExistsDocSet::new(non_empty_columns, reader.max_doc());
-            return Ok(Box::new(ConstScorer::new(docset, boost)));
+            Ok(Box::new(ConstScorer::new(docset, boost)))
         } else {
-            return Ok(Box::new(EmptyScorer));
+            Ok(Box::new(EmptyScorer))
         }
     }
 
@@ -102,20 +102,18 @@ impl ExistsDocSet {
     }
 
     fn find_next(&mut self) -> DocId {
-        // TODO: can this be optimized?
         while self.doc < self.max_doc {
             if self
                 .columns
                 .iter()
-                .find(|col| col.column_index().has_value(self.doc))
-                .is_some()
+                .any(|col| col.column_index().has_value(self.doc))
             {
                 return self.doc;
             }
             self.doc += 1;
         }
         self.doc = TERMINATED;
-        return TERMINATED;
+        TERMINATED
     }
 }
 
@@ -126,7 +124,7 @@ impl DocSet for ExistsDocSet {
     }
 
     fn size_hint(&self) -> u32 {
-        0 // heuristic possible by checking number of hits when fetching a block
+        0
     }
 
     fn doc(&self) -> DocId {
@@ -136,10 +134,15 @@ impl DocSet for ExistsDocSet {
 
 #[cfg(test)]
 mod tests {
+    use std::net::Ipv6Addr;
+
+    use common::DateTime;
+    use time::OffsetDateTime;
+
     use crate::collector::Count;
     use crate::query::exist_query::ExistsQuery;
-    use crate::schema::{Schema, FAST, INDEXED, STRING, TEXT};
-    use crate::{doc, Index};
+    use crate::schema::{Facet, FacetOptions, Schema, FAST, INDEXED, STRING, TEXT};
+    use crate::{doc, Index, Searcher};
 
     #[test]
     fn test_exists_query_simple() -> crate::Result<()> {
@@ -170,25 +173,11 @@ mod tests {
         let reader = index.reader()?;
         let searcher = reader.searcher();
 
-        let alldocs = ExistsQuery::new_exists_query("all".to_string());
-        let count = searcher.search(&alldocs, &Count)?;
-        assert_eq!(count, 100);
-
-        let even_docs = ExistsQuery::new_exists_query("even".to_string());
-        let count = searcher.search(&even_docs, &Count)?;
-        assert_eq!(count, 50);
-
-        let odd_docs = ExistsQuery::new_exists_query("odd".to_string());
-        let count = searcher.search(&odd_docs, &Count)?;
-        assert_eq!(count, 50);
-
-        let multi_docs = ExistsQuery::new_exists_query("multi".to_string());
-        let count = searcher.search(&multi_docs, &Count)?;
-        assert_eq!(count, 10);
-
-        let never_docs = ExistsQuery::new_exists_query("never".to_string());
-        let count = searcher.search(&never_docs, &Count)?;
-        assert_eq!(count, 0);
+        assert_eq!(count_existing_fields(&searcher, "all")?, 100);
+        assert_eq!(count_existing_fields(&searcher, "odd")?, 50);
+        assert_eq!(count_existing_fields(&searcher, "even")?, 50);
+        assert_eq!(count_existing_fields(&searcher, "multi")?, 10);
+        assert_eq!(count_existing_fields(&searcher, "never")?, 0);
 
         Ok(())
     }
@@ -215,18 +204,101 @@ mod tests {
         let reader = index.reader()?;
         let searcher = reader.searcher();
 
-        let alldocs = ExistsQuery::new_exists_query("json.all".to_string());
-        let count = searcher.search(&alldocs, &Count)?;
-        assert_eq!(count, 100);
+        assert_eq!(count_existing_fields(&searcher, "json.all")?, 100);
+        assert_eq!(count_existing_fields(&searcher, "json.even")?, 50);
+        assert_eq!(count_existing_fields(&searcher, "json.odd")?, 50);
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_exists_query_misc_supported_types() -> crate::Result<()> {
+        let mut schema_builder = Schema::builder();
+        let bool = schema_builder.add_bool_field("bool", FAST);
+        let bytes = schema_builder.add_bytes_field("bytes", FAST);
+        let date = schema_builder.add_date_field("date", FAST);
+        let f64 = schema_builder.add_f64_field("f64", FAST);
+        let ip_addr = schema_builder.add_ip_addr_field("ip_addr", FAST);
+        let facet = schema_builder.add_facet_field("facet", FacetOptions::default());
+        let schema = schema_builder.build();
+
+        let index = Index::create_in_ram(schema);
+        {
+            let mut index_writer = index.writer_for_tests()?;
+            let now = OffsetDateTime::now_utc().unix_timestamp();
+            for i in 0u8..100u8 {
+                if i % 2 == 0 {
+                    let date_val = DateTime::from_utc(OffsetDateTime::from_unix_timestamp(
+                        now + i as i64 * 100,
+                    )?);
+                    index_writer.add_document(
+                        doc!(bool => i % 3 == 0, bytes => vec![i, i + 1,  i + 2], date => date_val),
+                    )?;
+                } else {
+                    let ip_addr_v6 = Ipv6Addr::new(0, 0, 0, 0, 0, 0xffff, 0xc00a, i.into());
+                    index_writer
+                        .add_document(doc!(f64 => i as f64 * 0.5, ip_addr => ip_addr_v6, facet => Facet::from("/facet/foo"), facet => Facet::from("/facet/bar")))?;
+                }
+            }
+            index_writer.commit()?;
+        }
+        let reader = index.reader()?;
+        let searcher = reader.searcher();
+
+        assert_eq!(count_existing_fields(&searcher, "bool")?, 50);
+        assert_eq!(count_existing_fields(&searcher, "bytes")?, 50);
+        assert_eq!(count_existing_fields(&searcher, "date")?, 50);
+        assert_eq!(count_existing_fields(&searcher, "f64")?, 50);
+        assert_eq!(count_existing_fields(&searcher, "ip_addr")?, 50);
+        assert_eq!(count_existing_fields(&searcher, "facet")?, 50);
+
+        Ok(())
+    }
+
+    #[test]
+    fn test_exists_query_unsupported_types() -> crate::Result<()> {
+        let mut schema_builder = Schema::builder();
+        let not_fast = schema_builder.add_text_field("not_fast", TEXT);
+        let schema = schema_builder.build();
+
+        let index = Index::create_in_ram(schema);
+        {
+            let mut index_writer = index.writer_for_tests()?;
+            index_writer.add_document(doc!(
+                not_fast => "slow",
+            ))?;
+            index_writer.commit()?;
+        }
+        let reader = index.reader()?;
+        let searcher = reader.searcher();
 
-        let even_docs = ExistsQuery::new_exists_query("json.even".to_string());
-        let count = searcher.search(&even_docs, &Count)?;
-        assert_eq!(count, 50);
+        assert_eq!(
+            searcher
+                .search(
+                    &ExistsQuery::new_exists_query("not_fast".to_string()),
+                    &Count
+                )
+                .unwrap_err()
+                .to_string(),
+            "Schema error: 'Field not_fast is not a fast field.'"
+        );
 
-        let odd_docs = ExistsQuery::new_exists_query("json.odd".to_string());
-        let count = searcher.search(&odd_docs, &Count)?;
-        assert_eq!(count, 50);
+        assert_eq!(
+            searcher
+                .search(
+                    &ExistsQuery::new_exists_query("does_not_exists".to_string()),
+                    &Count
+                )
+                .unwrap_err()
+                .to_string(),
+            "The field does not exist: 'does_not_exists'"
+        );
 
         Ok(())
     }
+
+    fn count_existing_fields(searcher: &Searcher, field: &str) -> crate::Result<usize> {
+        let query = ExistsQuery::new_exists_query(field.to_string());
+        searcher.search(&query, &Count)
+    }
 }