Expose phrase-prefix queries via the built-in query parser (#2044)

* Expose phrase-prefix queries via the built-in query parser

This proposes the less-than-imaginative syntax `field:"phrase ter"*` to
perform a phrase prefix query against `field` using `phrase` and `ter` as the
terms. The aim of this is to make this type of query more discoverable and
simplify manual testing.

I did consider exposing the `max_expansions` parameter similar to how slop is
handled, but I think that this is rather something that should be configured via
the querser parser (similar to `set_field_boost` and `set_field_fuzzy`) as
choosing it requires rather intimiate knowledge of the backing index.

* Prevent construction of zero or one term phrase-prefix queries via the query parser.

* Add example using phrase-prefix search via surface API to improve feature discoverability.

Author: adamreichold

examples/phrase_prefix_search.rs

@@ -0,0 +1,79 @@
+use tantivy::collector::TopDocs;
+use tantivy::query::QueryParser;
+use tantivy::schema::*;
+use tantivy::{doc, Index, ReloadPolicy, Result};
+use tempfile::TempDir;
+
+fn main() -> Result<()> {
+    let index_path = TempDir::new()?;
+
+    let mut schema_builder = Schema::builder();
+    schema_builder.add_text_field("title", TEXT | STORED);
+    schema_builder.add_text_field("body", TEXT);
+    let schema = schema_builder.build();
+
+    let title = schema.get_field("title").unwrap();
+    let body = schema.get_field("body").unwrap();
+
+    let index = Index::create_in_dir(&index_path, schema)?;
+
+    let mut index_writer = index.writer(50_000_000)?;
+
+    index_writer.add_document(doc!(
+    title => "The Old Man and the Sea",
+    body => "He was an old man who fished alone in a skiff in the Gulf Stream and he had gone \
+            eighty-four days now without taking a fish.",
+    ))?;
+
+    index_writer.add_document(doc!(
+    title => "Of Mice and Men",
+    body => "A few miles south of Soledad, the Salinas River drops in close to the hillside \
+            bank and runs deep and green. The water is warm too, for it has slipped twinkling \
+            over the yellow sands in the sunlight before reaching the narrow pool. On one \
+            side of the river the golden foothill slopes curve up to the strong and rocky \
+            Gabilan Mountains, but on the valley side the water is lined with trees—willows \
+            fresh and green with every spring, carrying in their lower leaf junctures the \
+            debris of the winter’s flooding; and sycamores with mottled, white, recumbent \
+            limbs and branches that arch over the pool"
+    ))?;
+
+    // Multivalued field just need to be repeated.
+    index_writer.add_document(doc!(
+    title => "Frankenstein",
+    title => "The Modern Prometheus",
+    body => "You will rejoice to hear that no disaster has accompanied the commencement of an \
+             enterprise which you have regarded with such evil forebodings.  I arrived here \
+             yesterday, and my first task is to assure my dear sister of my welfare and \
+             increasing confidence in the success of my undertaking."
+    ))?;
+
+    index_writer.commit()?;
+
+    let reader = index
+        .reader_builder()
+        .reload_policy(ReloadPolicy::OnCommit)
+        .try_into()?;
+
+    let searcher = reader.searcher();
+
+    let query_parser = QueryParser::for_index(&index, vec![title, body]);
+    // This will match documents containing the phrase "in the"
+    // followed by some word starting with "su",
+    // i.e. it will match "in the sunlight" and "in the success",
+    // but not "in the Gulf Stream".
+    let query = query_parser.parse_query("\"in the su\"*")?;
+
+    let top_docs = searcher.search(&query, &TopDocs::with_limit(10))?;
+    let mut titles = top_docs
+        .into_iter()
+        .map(|(_score, doc_address)| {
+            let doc = searcher.doc(doc_address)?;
+            let title = doc.get_first(title).unwrap().as_text().unwrap().to_owned();
+            Ok(title)
+        })
+        .collect::<Result<Vec<_>>>()?;
+    titles.sort_unstable();
+    assert_eq!(titles, ["Frankenstein", "Of Mice and Men"]);
+
+    Ok(())
+}

query-grammar/src/query_grammar.rs

@@ -162,14 +162,22 @@ fn term_val<'a>() -> impl Parser<&'a str, Output = (Delimiter, String)> {
 }
 
 fn term_query<'a>() -> impl Parser<&'a str, Output = UserInputLiteral> {
-    (field_name(), term_val(), slop_val()).map(|(field_name, (delimiter, phrase), slop)| {
-        UserInputLiteral {
+    (field_name(), term_val(), slop_or_prefix_val()).map(
+        |(field_name, (delimiter, phrase), (slop, prefix))| UserInputLiteral {
             field_name: Some(field_name),
             phrase,
             delimiter,
             slop,
-        }
-    })
+            prefix,
+        },
+    )
+}
+
+fn slop_or_prefix_val<'a>() -> impl Parser<&'a str, Output = (u32, bool)> {
+    let prefix_val = char('*').map(|_ast| (0, true));
+    let slop_val = slop_val().map(|slop| (slop, false));
+
+    prefix_val.or(slop_val)
 }
 
 fn slop_val<'a>() -> impl Parser<&'a str, Output = u32> {
@@ -186,11 +194,14 @@ fn slop_val<'a>() -> impl Parser<&'a str, Output = u32> {
 
 fn literal<'a>() -> impl Parser<&'a str, Output = UserInputLeaf> {
     let term_default_field =
-        (term_val(), slop_val()).map(|((delimiter, phrase), slop)| UserInputLiteral {
-            field_name: None,
-            phrase,
-            delimiter,
-            slop,
+        (term_val(), slop_or_prefix_val()).map(|((delimiter, phrase), (slop, prefix))| {
+            UserInputLiteral {
+                field_name: None,
+                phrase,
+                delimiter,
+                slop,
+                prefix,
+            }
         });
 
     attempt(term_query())
@@ -872,6 +883,16 @@ mod test {
         test_parse_query_to_ast_helper("\"a b\"~300^2", "(\"a b\"~300)^2");
     }
 
+    #[test]
+    fn test_phrase_prefix() {
+        test_parse_query_to_ast_helper("\"a b\"*", "\"a b\"*");
+        test_parse_query_to_ast_helper("\"a\"*", "\"a\"*");
+        test_parse_query_to_ast_helper("\"\"*", "\"\"*");
+        test_parse_query_to_ast_helper("foo:\"a b\"*", "\"foo\":\"a b\"*");
+        test_parse_query_to_ast_helper("foo:\"a\"*", "\"foo\":\"a\"*");
+        test_parse_query_to_ast_helper("foo:\"\"*", "\"foo\":\"\"*");
+    }
+
     #[test]
     fn test_not_queries_are_consistent() {
         test_parse_query_to_ast_helper("tata -toto", "(*tata -toto)");

query-grammar/src/user_input_ast.rs

@@ -66,6 +66,7 @@ pub struct UserInputLiteral {
     pub phrase: String,
     pub delimiter: Delimiter,
     pub slop: u32,
+    pub prefix: bool,
 }
 
 impl fmt::Debug for UserInputLiteral {
@@ -86,6 +87,8 @@ impl fmt::Debug for UserInputLiteral {
         }
         if self.slop > 0 {
             write!(formatter, "~{}", self.slop)?;
+        } else if self.prefix {
+            write!(formatter, "*")?;
         }
         Ok(())
     }

src/query/phrase_prefix_query/phrase_prefix_query.rs

@@ -88,9 +88,6 @@ impl PhrasePrefixQuery {
     /// a specialized type [`PhraseQueryWeight`] instead of a Boxed trait.
     /// If the query was only one term long, this returns `None` wherease [`Query::weight`]
     /// returns a boxed [`RangeWeight`]
-    ///
-    /// Returns `None`, if phrase_terms is empty, which happens if the phrase prefix query was
-    /// built with a single term.
     pub(crate) fn phrase_prefix_query_weight(
         &self,
         enable_scoring: EnableScoring<'_>,

src/query/query_parser/logical_ast.rs

@@ -8,7 +8,11 @@ use crate::Score;
 #[derive(Clone)]
 pub enum LogicalLiteral {
     Term(Term),
-    Phrase(Vec<(usize, Term)>, u32),
+    Phrase {
+        terms: Vec<(usize, Term)>,
+        slop: u32,
+        prefix: bool,
+    },
     Range {
         field: String,
         value_type: Type,
@@ -79,10 +83,16 @@ impl fmt::Debug for LogicalLiteral {
     fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> {
         match *self {
             LogicalLiteral::Term(ref term) => write!(formatter, "{term:?}"),
-            LogicalLiteral::Phrase(ref terms, slop) => {
+            LogicalLiteral::Phrase {
+                ref terms,
+                slop,
+                prefix,
+            } => {
                 write!(formatter, "\"{terms:?}\"")?;
                 if slop > 0 {
                     write!(formatter, "~{slop:?}")
+                } else if prefix {
+                    write!(formatter, "*")
                 } else {
                     Ok(())
                 }