readyset-sql-parsing: Parse with both nom-sql and sqlparser-rs

This entails adding a suite of `From`/`TryFrom` impls for the AST types
to convert from sqlparser's AST to ours. It is currently mostly `From`
with panics and todos for easy clicking during initial implementation,
but most of these should be converted to `TryFrom` later.

Gated behind `sqlparser` feature flag:

```sh
cargo test --features sqlparser
```

Change-Id: I28d23de30cbbd87d034270ab1572e8ca051fffb6
Reviewed-on: https://gerrit.readyset.name/c/readyset/+/8773
Reviewed-by: Jason Brown <[email protected]>
Tested-by: Buildkite CI

Author: mvzink

Cargo.lock

@@ -885,7 +885,8 @@ mod tests {
     fn select_statement(s: &str) -> anyhow::Result<SelectStatement> {
         match readyset_sql_parsing::parse_query(readyset_sql::Dialect::MySQL, s) {
             Ok(SqlQuery::Select(s)) => Ok(s),
-            _ => Err(anyhow::anyhow!("Invalid SELECT statement")),
+            Ok(q) => Err(anyhow::anyhow!("Not a SELECT statement: {q:?}")),
+            Err(e) => Err(anyhow::anyhow!("Parsing error: {e}")),
         }
     }
 

readyset-adapter/src/query_status_cache.rs

@@ -917,10 +917,10 @@ mod tests {
 
     #[test]
     fn test_dollar_number_parameter_column_extraction() {
-        let query = "SELECT  `votes`.* FROM `votes` WHERE `votes`.`user_id` = 1 \
-                     AND `votes`.`story_id` = $1 AND `votes`.`comment_id` IS NULL \
-                     ORDER BY `votes`.`id` ASC LIMIT 1";
-        let q = readyset_sql_parsing::parse_query(Dialect::MySQL, query).unwrap();
+        let query = r#"SELECT "votes".* FROM "votes" WHERE "votes"."user_id" = 1
+                       AND "votes"."story_id" = $1 AND "votes"."comment_id" IS NULL
+                       ORDER BY "votes"."id" ASC LIMIT 1"#;
+        let q = readyset_sql_parsing::parse_query(Dialect::PostgreSQL, query).unwrap();
 
         let pc = get_parameter_columns(&q);
 

readyset-adapter/src/utils.rs

@@ -29,10 +29,6 @@ async fn mysql_send_long_data_inner() {
         .query_drop("CREATE TABLE t (x LONGBLOB)")
         .await
         .unwrap();
-    rs_conn
-        .query_drop("SET SESSION max_execution_time=60000")
-        .await
-        .unwrap();
     rs_conn
         .exec_drop(
             "INSERT INTO t VALUES (?)",

readyset-e2e-tests/tests/mysql_long_data.rs

@@ -84,7 +84,7 @@ async fn duplicate_join_key() {
         .await
         .unwrap();
 
-    conn.query_drop("SELECT a.id FROM a JOIN b on b.a_id = a.id WHERE a.id = -1")
+    conn.query_drop("SELECT a.id FROM a JOIN b on b.a_id = a.id WHERE a.id = 42")
         .await
         .unwrap();
 

readyset-mysql/tests/integration.rs

@@ -7,6 +7,11 @@ edition = "2021"
 
 [dependencies]
 nom-sql = { path = "../nom-sql" }
+pretty_assertions = { workspace = true }
 readyset-sql = { path = "../readyset-sql" }
-sqlparser = { workspace = true }
+sqlparser = { workspace = true, optional = true }
 thiserror = { workspace = true }
+tracing = { workspace = true }
+
+[features]
+sqlparser = ["dep:sqlparser"]

readyset-sql-parsing/Cargo.toml

@@ -1,5 +1,56 @@
 use readyset_sql::{ast::SqlQuery, Dialect};
 
+#[cfg(feature = "sqlparser")]
+fn sqlparser_dialect_from_readyset_dialect(
+    dialect: Dialect,
+) -> Box<dyn sqlparser::dialect::Dialect> {
+    match dialect {
+        Dialect::PostgreSQL => Box::new(sqlparser::dialect::PostgreSqlDialect {}),
+        Dialect::MySQL => Box::new(sqlparser::dialect::MySqlDialect {}),
+    }
+}
+
+#[cfg(feature = "sqlparser")]
+pub fn parse_query(dialect: Dialect, input: impl AsRef<str>) -> Result<SqlQuery, String> {
+    let nom_result = nom_sql::parse_query(dialect, input.as_ref());
+    let sqlparser_dialect = sqlparser_dialect_from_readyset_dialect(dialect);
+    let sqlparser_result: Result<SqlQuery, _> =
+        sqlparser::parser::Parser::new(sqlparser_dialect.as_ref())
+            .try_with_sql(input.as_ref())
+            .and_then(|mut p| p.parse_statement())
+            .map_err(|e| format!("failed to parse: {e}"))
+            .and_then(|q| {
+                q.try_into()
+                    .map_err(|e| format!("failed to convert AST: {e}"))
+            });
+    match (&nom_result, sqlparser_result) {
+        (Ok(nom_ast), Ok(sqlparser_ast)) => {
+            pretty_assertions::assert_eq!(
+                nom_ast,
+                &sqlparser_ast,
+                "nom-sql AST differs from sqlparser-rs AST. input: {:?}",
+                input.as_ref()
+            );
+        }
+        (Ok(nom_ast), Err(sqlparser_error)) => {
+            panic!(
+                "nom-sql succeeded but sqlparser-rs failed: {}\ninput: {}\nnom_ast: {:?}",
+                sqlparser_error,
+                input.as_ref(),
+                nom_ast
+            )
+        }
+        (Err(nom_error), Ok(sqlparser_ast)) => {
+            tracing::warn!(%nom_error, ?sqlparser_ast, "sqlparser-rs succeeded but nom-sql failed")
+        }
+        (Err(nom_error), Err(sqlparser_error)) => {
+            tracing::warn!(%nom_error, %sqlparser_error, "both nom-sql and sqlparser-rs failed");
+        }
+    }
+    nom_result
+}
+
+#[cfg(not(feature = "sqlparser"))]
 pub fn parse_query(dialect: Dialect, input: impl AsRef<str>) -> Result<SqlQuery, String> {
     nom_sql::parse_query(dialect, input.as_ref())
 }

readyset-sql-parsing/src/lib.rs

@@ -4,7 +4,7 @@ use readyset_util::fmt::fmt_with;
 use serde::{Deserialize, Serialize};
 use test_strategy::Arbitrary;
 
-use crate::{ast::*, Dialect, DialectDisplay};
+use crate::{ast::*, AstConversionError, Dialect, DialectDisplay};
 
 #[derive(Clone, Debug, Eq, Hash, PartialEq, Serialize, Deserialize, Arbitrary)]
 pub struct Column {
@@ -33,6 +33,66 @@ impl From<&'_ str> for Column {
     }
 }
 
+impl From<sqlparser::ast::Ident> for Column {
+    fn from(value: sqlparser::ast::Ident) -> Self {
+        Self {
+            name: value.into(),
+            table: None,
+        }
+    }
+}
+
+impl From<Vec<sqlparser::ast::Ident>> for Column {
+    fn from(mut value: Vec<sqlparser::ast::Ident>) -> Self {
+        let name: SqlIdentifier = value.pop().unwrap().into();
+        let table = if let Some(table) = value.pop() {
+            if let Some(schema) = value.pop() {
+                Some(Relation {
+                    schema: Some(schema.into()),
+                    name: table.into(),
+                })
+            } else {
+                Some(Relation {
+                    schema: None,
+                    name: table.into(),
+                })
+            }
+        } else {
+            None
+        };
+        Self { name, table }
+    }
+}
+
+impl From<sqlparser::ast::ObjectName> for Column {
+    fn from(value: sqlparser::ast::ObjectName) -> Self {
+        value
+            .0
+            .into_iter()
+            .map(|sqlparser::ast::ObjectNamePart::Identifier(ident)| ident)
+            .collect::<Vec<_>>()
+            .into()
+    }
+}
+
+impl From<sqlparser::ast::ViewColumnDef> for Column {
+    fn from(value: sqlparser::ast::ViewColumnDef) -> Self {
+        Self {
+            name: value.name.into(),
+            table: None,
+        }
+    }
+}
+
+impl From<sqlparser::ast::AssignmentTarget> for Column {
+    fn from(value: sqlparser::ast::AssignmentTarget) -> Self {
+        match value {
+            sqlparser::ast::AssignmentTarget::ColumnName(object_name) => object_name.into(),
+            sqlparser::ast::AssignmentTarget::Tuple(_vec) => todo!("tuple assignment syntax"),
+        }
+    }
+}
+
 impl Ord for Column {
     fn cmp(&self, other: &Column) -> Ordering {
         match (self.table.as_ref(), other.table.as_ref()) {
@@ -120,6 +180,105 @@ pub struct ColumnSpecification {
     pub comment: Option<String>,
 }
 
+impl TryFrom<sqlparser::ast::ColumnDef> for ColumnSpecification {
+    type Error = AstConversionError;
+
+    fn try_from(value: sqlparser::ast::ColumnDef) -> Result<Self, Self::Error> {
+        use sqlparser::{
+            keywords::Keyword,
+            tokenizer::{Token, Word},
+        };
+
+        let mut comment = None;
+        let mut constraints = vec![];
+        let mut generated = None;
+        for option in value.options {
+            match option.option {
+                sqlparser::ast::ColumnOption::Null => constraints.push(ColumnConstraint::Null),
+                sqlparser::ast::ColumnOption::NotNull => {
+                    constraints.push(ColumnConstraint::NotNull)
+                }
+                sqlparser::ast::ColumnOption::Default(expr) => {
+                    constraints.push(ColumnConstraint::DefaultValue(expr.try_into()?))
+                }
+                sqlparser::ast::ColumnOption::Unique {
+                    is_primary,
+                    characteristics,
+                } => {
+                    if characteristics.is_some() {
+                        return not_yet_implemented!("constraint timing on column definitions");
+                    } else if is_primary {
+                        constraints.push(ColumnConstraint::PrimaryKey)
+                    } else {
+                        constraints.push(ColumnConstraint::Unique)
+                    }
+                }
+                sqlparser::ast::ColumnOption::ForeignKey { .. } => {
+                    return not_yet_implemented!("foreign key");
+                }
+                sqlparser::ast::ColumnOption::DialectSpecific(vec) => {
+                    if vec.iter().any(|token| {
+                        matches!(
+                            token,
+                            Token::Word(Word {
+                                keyword: Keyword::AUTO_INCREMENT,
+                                ..
+                            })
+                        )
+                    }) {
+                        constraints.push(ColumnConstraint::AutoIncrement)
+                    }
+                }
+                sqlparser::ast::ColumnOption::CharacterSet(object_name) => {
+                    constraints.push(ColumnConstraint::CharacterSet(object_name.to_string()))
+                }
+                sqlparser::ast::ColumnOption::Collation(object_name) => {
+                    constraints.push(ColumnConstraint::Collation(object_name.to_string()))
+                }
+                sqlparser::ast::ColumnOption::Comment(s) => {
+                    comment = Some(s);
+                }
+                sqlparser::ast::ColumnOption::OnUpdate(_expr) => {
+                    todo!("on update (check for current_timestamp)")
+                }
+                sqlparser::ast::ColumnOption::Generated {
+                    generated_as: _,
+                    sequence_options: _,
+                    generation_expr,
+                    generation_expr_mode,
+                    generated_keyword: _,
+                } => {
+                    generated = Some(GeneratedColumn {
+                        expr: generation_expr
+                            .map(TryInto::try_into)
+                            .expect("generated expr can't be None")?,
+                        stored: generation_expr_mode
+                            == Some(sqlparser::ast::GeneratedExpressionMode::Stored),
+                    })
+                }
+                sqlparser::ast::ColumnOption::Materialized(_)
+                | sqlparser::ast::ColumnOption::Ephemeral(_)
+                | sqlparser::ast::ColumnOption::Alias(_)
+                | sqlparser::ast::ColumnOption::Check(_)
+                | sqlparser::ast::ColumnOption::Options(_)
+                | sqlparser::ast::ColumnOption::Identity(_)
+                | sqlparser::ast::ColumnOption::OnConflict(_)
+                | sqlparser::ast::ColumnOption::Policy(_)
+                | sqlparser::ast::ColumnOption::Tags(_) => {
+                    // Don't care about these options
+                }
+            }
+        }
+        Ok(Self {
+            column: value.name.into(),
+            sql_type: value.data_type.try_into()?,
+            constraints,
+            comment,
+            generated,
+        })
+    }
+}
+
 impl ColumnSpecification {
     pub fn new(column: Column, sql_type: SqlType) -> ColumnSpecification {
         ColumnSpecification {