From 0a6a5dcf68e627415597c6426732b308740ec2b1 Mon Sep 17 00:00:00 2001 From: Bryan Reynaert Date: Sun, 16 Mar 2025 18:50:37 -0700 Subject: [PATCH 1/3] use sqlite3_column_type() for ColumnTypeScanType() --- sqlite3_type.go | 66 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 46 insertions(+), 20 deletions(-) diff --git a/sqlite3_type.go b/sqlite3_type.go index 20537a09..923e402b 100644 --- a/sqlite3_type.go +++ b/sqlite3_type.go @@ -18,6 +18,27 @@ import ( "strings" ) +const ( + SQLITE_INTEGER = iota + SQLITE_TEXT + SQLITE_BLOB + SQLITE_REAL + SQLITE_NUMERIC + SQLITE_TIME + SQLITE_BOOL + SQLITE_NULL +) + +var ( + TYPE_NULLINT = reflect.TypeOf(sql.NullInt64{}) + TYPE_NULLFLOAT = reflect.TypeOf(sql.NullFloat64{}) + TYPE_NULLSTRING = reflect.TypeOf(sql.NullString{}) + TYPE_RAWBYTES = reflect.TypeOf(sql.RawBytes{}) + TYPE_NULLBOOL = reflect.TypeOf(sql.NullBool{}) + TYPE_NULLTIME = reflect.TypeOf(sql.NullTime{}) + TYPE_ANY = reflect.TypeOf(new(any)) +) + // ColumnTypeDatabaseTypeName implement RowsColumnTypeDatabaseTypeName. func (rc *SQLiteRows) ColumnTypeDatabaseTypeName(i int) string { return C.GoString(C.sqlite3_column_decltype(rc.s.s, C.int(i))) @@ -39,42 +60,47 @@ func (rc *SQLiteRows) ColumnTypeNullable(i int) (nullable, ok bool) { } // ColumnTypeScanType implement RowsColumnTypeScanType. +// In SQLite3, this method should be called after Next() has been called, as sqlite3_column_type() +// returns the column type for a specific row. If Next() has not been called, fallback to +// sqlite3_column_decltype() func (rc *SQLiteRows) ColumnTypeScanType(i int) reflect.Type { - //ct := C.sqlite3_column_type(rc.s.s, C.int(i)) // Always returns 5 + switch C.sqlite3_column_type(rc.s.s, C.int(i)) { + case C.SQLITE_INTEGER: + return TYPE_NULLINT + case C.SQLITE_FLOAT: + return TYPE_NULLFLOAT + case C.SQLITE_TEXT: + return TYPE_NULLSTRING + case C.SQLITE_BLOB: + return TYPE_RAWBYTES + //case C.SQLITE_NULL: + // return TYPE_ANY + } + + // Fallback to schema declared to remain retro-compatible return scanType(C.GoString(C.sqlite3_column_decltype(rc.s.s, C.int(i)))) } -const ( - SQLITE_INTEGER = iota - SQLITE_TEXT - SQLITE_BLOB - SQLITE_REAL - SQLITE_NUMERIC - SQLITE_TIME - SQLITE_BOOL - SQLITE_NULL -) - func scanType(cdt string) reflect.Type { t := strings.ToUpper(cdt) i := databaseTypeConvSqlite(t) switch i { case SQLITE_INTEGER: - return reflect.TypeOf(sql.NullInt64{}) + return TYPE_NULLINT case SQLITE_TEXT: - return reflect.TypeOf(sql.NullString{}) + return TYPE_NULLSTRING case SQLITE_BLOB: - return reflect.TypeOf(sql.RawBytes{}) + return TYPE_RAWBYTES case SQLITE_REAL: - return reflect.TypeOf(sql.NullFloat64{}) + return TYPE_NULLFLOAT case SQLITE_NUMERIC: - return reflect.TypeOf(sql.NullFloat64{}) + return TYPE_NULLFLOAT case SQLITE_BOOL: - return reflect.TypeOf(sql.NullBool{}) + return TYPE_NULLBOOL case SQLITE_TIME: - return reflect.TypeOf(sql.NullTime{}) + return TYPE_NULLTIME } - return reflect.TypeOf(new(any)) + return TYPE_ANY } func databaseTypeConvSqlite(t string) int { From 27e8f68419838a42a80fdc29ff1e1e6865178582 Mon Sep 17 00:00:00 2001 From: Bryan Reynaert Date: Tue, 18 Mar 2025 19:16:00 -0700 Subject: [PATCH 2/3] add clarifying comment for skipping C.SQLITE_NULL --- sqlite3_type.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sqlite3_type.go b/sqlite3_type.go index 923e402b..cf8bc1bc 100644 --- a/sqlite3_type.go +++ b/sqlite3_type.go @@ -73,6 +73,9 @@ func (rc *SQLiteRows) ColumnTypeScanType(i int) reflect.Type { return TYPE_NULLSTRING case C.SQLITE_BLOB: return TYPE_RAWBYTES + // This case can signal that the value is NULL or that Next() has not been called yet. + // Skip it and return the fallback behaviour as a best effort. This is safe as all types + // returned are Nullable or any, which is the expected value for SQLite3. //case C.SQLITE_NULL: // return TYPE_ANY } From 855d42684f87aabb8c22d73c146918a8280be08d Mon Sep 17 00:00:00 2001 From: Bryan Reynaert Date: Sun, 23 Mar 2025 17:07:14 -0700 Subject: [PATCH 3/3] add guards to ColumnTypeScanType. Return interface{} on error or null. Add unit tests --- sqlite3.go | 9 +- sqlite3_test.go | 283 ++++++++++++++++++++++++++++++++++++++++++++++++ sqlite3_type.go | 118 ++++++-------------- 3 files changed, 323 insertions(+), 87 deletions(-) diff --git a/sqlite3.go b/sqlite3.go index 3025a500..162bd549 100644 --- a/sqlite3.go +++ b/sqlite3.go @@ -243,6 +243,7 @@ const ( columnDate string = "date" columnDatetime string = "datetime" columnTimestamp string = "timestamp" + columnBoolean string = "boolean" ) // This variable can be replaced with -ldflags like below: @@ -269,7 +270,7 @@ const ( SQLITE_INSERT = C.SQLITE_INSERT SQLITE_UPDATE = C.SQLITE_UPDATE - // used by authorzier - as return value + // used by authorizer - as return value SQLITE_OK = C.SQLITE_OK SQLITE_IGNORE = C.SQLITE_IGNORE SQLITE_DENY = C.SQLITE_DENY @@ -2105,7 +2106,7 @@ func (s *SQLiteStmt) execSync(args []driver.NamedValue) (driver.Result, error) { // // See: https://sqlite.org/c3ref/stmt_readonly.html func (s *SQLiteStmt) Readonly() bool { - return C.sqlite3_stmt_readonly(s.s) == 1 + return C.sqlite3_stmt_readonly(s.s) != 0 } // Close the rows. @@ -2233,8 +2234,8 @@ func (rc *SQLiteRows) nextSyncLocked(dest []driver.Value) error { t = t.In(rc.s.c.loc) } dest[i] = t - case "boolean": - dest[i] = val > 0 + case columnBoolean: + dest[i] = val != 0 default: dest[i] = val } diff --git a/sqlite3_test.go b/sqlite3_test.go index 94de7386..e7347656 100644 --- a/sqlite3_test.go +++ b/sqlite3_test.go @@ -14,6 +14,7 @@ import ( "database/sql/driver" "errors" "fmt" + "io" "io/ioutil" "math/rand" "net/url" @@ -1709,6 +1710,288 @@ func TestDeclTypes(t *testing.T) { } } +func TestScanTypes(t *testing.T) { + + d := SQLiteDriver{} + + conn, err := d.Open(":memory:") + if err != nil { + t.Fatal("Failed to begin transaction:", err) + } + defer conn.Close() + + sqlite3conn := conn.(*SQLiteConn) + + _, err = sqlite3conn.Exec("create table foo (id integer not null primary key, name text, price integer, length float, token blob, dob timestamp, jdays date, somedate datetime)", nil) + if err != nil { + t.Fatal("Failed to create table:", err) + } + expected := []reflect.Type{type_nullint, type_nullstring, type_nullint, type_nullfloat, type_rawbytes, type_nulltime, type_nulltime, type_nulltime} + + _, err = sqlite3conn.Exec("insert into foo(name, price, length, token, dob, jdays, somedate) values('bar', 10, 3.1415, x'0500', 100, 5.0, '2006-01-02 15:04:05')", nil) + if err != nil { + t.Fatal("Failed to insert:", err) + } + + rs, err := sqlite3conn.Query("select * from foo", nil) + if err != nil { + t.Fatal("Failed to select:", err) + } + defer rs.Close() + + cols := make([]driver.Value, len(rs.Columns())) + err = rs.Next(cols) + if err != nil { + t.Fatal("Failed to advance cursor:", err) + } + + rc, ok := rs.(driver.RowsColumnTypeScanType) + if !ok { + t.Fatal("SQLiteRows does not implement driver.RowsColumnTypeScanType") + } + + for i := range rc.Columns() { + if st := rc.ColumnTypeScanType(i); st != expected[i] { + t.Fatal("Unexpected ScanType. Expected:", expected[i], "Got:", st) + } + } +} + +func TestScanTypesBeforeNext(t *testing.T) { + + d := SQLiteDriver{} + + conn, err := d.Open(":memory:") + if err != nil { + t.Fatal("Failed to begin transaction:", err) + } + defer conn.Close() + + sqlite3conn := conn.(*SQLiteConn) + + _, err = sqlite3conn.Exec("create table foo (id integer not null primary key, name text, price integer, length float, token blob, dob timestamp, jdays date, somedate datetime)", nil) + if err != nil { + t.Fatal("Failed to create table:", err) + } + + _, err = sqlite3conn.Exec("insert into foo(name, price, length, token, dob, jdays, somedate) values('bar', 10, 3.1415, x'0500', 100, 5.0, '2006-01-02 15:04:05')", nil) + if err != nil { + t.Fatal("Failed to insert:", err) + } + + rs, err := sqlite3conn.Query("select * from foo", nil) + if err != nil { + t.Fatal("Failed to select:", err) + } + defer rs.Close() + + rc, ok := rs.(driver.RowsColumnTypeScanType) + if !ok { + t.Fatal("SQLiteRows does not implement driver.RowsColumnTypeScanType") + } + + for i := range rc.Columns() { + if st := rc.ColumnTypeScanType(i); st != type_any { + t.Fatal("Unexpected ScanType:", st) + } + } +} + +func TestScanTypesAfterClosed(t *testing.T) { + + d := SQLiteDriver{} + + conn, err := d.Open(":memory:") + if err != nil { + t.Fatal("Failed to begin transaction:", err) + } + defer conn.Close() + + sqlite3conn := conn.(*SQLiteConn) + + _, err = sqlite3conn.Exec("create table foo (id integer not null primary key, name text, price integer, length float, token blob, dob timestamp, jdays date, somedate datetime)", nil) + if err != nil { + t.Fatal("Failed to create table:", err) + } + + _, err = sqlite3conn.Exec("insert into foo(name, price, length, token, dob, jdays, somedate) values('bar', 10, 3.1415, x'0500', 100, 5.0, '2006-01-02 15:04:05')", nil) + if err != nil { + t.Fatal("Failed to insert:", err) + } + + rs, err := sqlite3conn.Query("select * from foo", nil) + if err != nil { + t.Fatal("Failed to select:", err) + } + defer rs.Close() + + cols := make([]driver.Value, len(rs.Columns())) + err = rs.Next(cols) + if err != nil { + t.Fatal("Failed to advance cursor:", err) + } + err = rs.Next(cols) + if err != io.EOF { + t.Fatal("Unexpected error when reaching end of dataset:", err) + } + + rc, ok := rs.(driver.RowsColumnTypeScanType) + if !ok { + t.Fatal("SQLiteRows does not implement driver.RowsColumnTypeScanType") + } + + for i := range rc.Columns() { + if st := rc.ColumnTypeScanType(i); st != type_any { + t.Fatal("Unexpected ScanType:", st) + } + } +} + +func TestScanTypesInvalidColumn(t *testing.T) { + + d := SQLiteDriver{} + + conn, err := d.Open(":memory:") + if err != nil { + t.Fatal("Failed to begin transaction:", err) + } + defer conn.Close() + + sqlite3conn := conn.(*SQLiteConn) + + _, err = sqlite3conn.Exec("create table foo (id integer not null primary key, name text, price integer, length float, token blob, dob timestamp, jdays date, somedate datetime)", nil) + if err != nil { + t.Fatal("Failed to create table:", err) + } + + _, err = sqlite3conn.Exec("insert into foo(name, price, length, token, dob, jdays, somedate) values('bar', 10, 3.1415, x'0500', 100, 5.0, '2006-01-02 15:04:05')", nil) + if err != nil { + t.Fatal("Failed to insert:", err) + } + + rs, err := sqlite3conn.Query("select * from foo", nil) + if err != nil { + t.Fatal("Failed to select:", err) + } + defer rs.Close() + + cols := make([]driver.Value, len(rs.Columns())) + err = rs.Next(cols) + if err != nil { + t.Fatal("Failed to advance cursor:", err) + } + + rc, ok := rs.(driver.RowsColumnTypeScanType) + if !ok { + t.Fatal("SQLiteRows does not implement driver.RowsColumnTypeScanType") + } + + if st := rc.ColumnTypeScanType(len(rc.Columns())); st != type_any { + t.Fatal("Unexpected ScanType:", st) + } + if st := rc.ColumnTypeScanType(-1); st != type_any { + t.Fatal("Unexpected ScanType:", st) + } +} + +func TestScanTypesNull(t *testing.T) { + + d := SQLiteDriver{} + + conn, err := d.Open(":memory:") + if err != nil { + t.Fatal("Failed to begin transaction:", err) + } + defer conn.Close() + + sqlite3conn := conn.(*SQLiteConn) + + _, err = sqlite3conn.Exec("create table foo (id integer not null primary key, name text, price integer, length float, token blob, dob timestamp, jdays date, somedate datetime)", nil) + if err != nil { + t.Fatal("Failed to create table:", err) + } + + _, err = sqlite3conn.Exec("insert into foo(name, price, length, token, dob, jdays, somedate) values(null, null, null, null, null, null, null)", nil) + if err != nil { + t.Fatal("Failed to insert:", err) + } + + rs, err := sqlite3conn.Query("select * from foo", nil) + if err != nil { + t.Fatal("Failed to select:", err) + } + defer rs.Close() + + cols := make([]driver.Value, len(rs.Columns())) + err = rs.Next(cols) + if err != nil { + t.Fatal("Failed to advance cursor:", err) + } + + rc, ok := rs.(driver.RowsColumnTypeScanType) + if !ok { + t.Fatal("SQLiteRows does not implement driver.RowsColumnTypeScanType") + } + + for i := 1; i < len(rc.Columns()); i++ { + if st := rc.ColumnTypeScanType(i); st != type_any { + t.Fatal("Unexpected ScanType:", i, st) + } + } +} + +func TestScanTypesAggregate(t *testing.T) { + + d := SQLiteDriver{} + + conn, err := d.Open(":memory:") + if err != nil { + t.Fatal("Failed to begin transaction:", err) + } + defer conn.Close() + + sqlite3conn := conn.(*SQLiteConn) + + _, err = sqlite3conn.Exec("create table foo (id integer not null primary key, price integer)", nil) + if err != nil { + t.Fatal("Failed to create table:", err) + } + + _, err = sqlite3conn.Exec("insert into foo(price) values(0)", nil) + if err != nil { + t.Fatal("Failed to insert:", err) + } + _, err = sqlite3conn.Exec("insert into foo(price) values(5)", nil) + if err != nil { + t.Fatal("Failed to insert:", err) + } + _, err = sqlite3conn.Exec("insert into foo(price) values(10)", nil) + if err != nil { + t.Fatal("Failed to insert:", err) + } + + rs, err := sqlite3conn.Query("select total(price) from foo", nil) + if err != nil { + t.Fatal("Failed to select:", err) + } + defer rs.Close() + + cols := make([]driver.Value, len(rs.Columns())) + err = rs.Next(cols) + if err != nil { + t.Fatal("Failed to advance cursor:", err) + } + + rc, ok := rs.(driver.RowsColumnTypeScanType) + if !ok { + t.Fatal("SQLiteRows does not implement driver.RowsColumnTypeScanType") + } + + if st := rc.ColumnTypeScanType(0); st != type_nullfloat { + t.Fatal("Unexpected ScanType:", st) + } +} + func TestPinger(t *testing.T) { db, err := sql.Open("sqlite3", ":memory:") if err != nil { diff --git a/sqlite3_type.go b/sqlite3_type.go index cf8bc1bc..4d56deac 100644 --- a/sqlite3_type.go +++ b/sqlite3_type.go @@ -15,28 +15,16 @@ import "C" import ( "database/sql" "reflect" - "strings" -) - -const ( - SQLITE_INTEGER = iota - SQLITE_TEXT - SQLITE_BLOB - SQLITE_REAL - SQLITE_NUMERIC - SQLITE_TIME - SQLITE_BOOL - SQLITE_NULL ) var ( - TYPE_NULLINT = reflect.TypeOf(sql.NullInt64{}) - TYPE_NULLFLOAT = reflect.TypeOf(sql.NullFloat64{}) - TYPE_NULLSTRING = reflect.TypeOf(sql.NullString{}) - TYPE_RAWBYTES = reflect.TypeOf(sql.RawBytes{}) - TYPE_NULLBOOL = reflect.TypeOf(sql.NullBool{}) - TYPE_NULLTIME = reflect.TypeOf(sql.NullTime{}) - TYPE_ANY = reflect.TypeOf(new(any)) + type_nullint = reflect.TypeOf(sql.NullInt64{}) + type_nullfloat = reflect.TypeOf(sql.NullFloat64{}) + type_nullstring = reflect.TypeOf(sql.NullString{}) + type_rawbytes = reflect.TypeOf(sql.RawBytes{}) + type_nullbool = reflect.TypeOf(sql.NullBool{}) + type_nulltime = reflect.TypeOf(sql.NullTime{}) + type_any = reflect.TypeOf(new(any)).Elem() ) // ColumnTypeDatabaseTypeName implement RowsColumnTypeDatabaseTypeName. @@ -64,74 +52,38 @@ func (rc *SQLiteRows) ColumnTypeNullable(i int) (nullable, ok bool) { // returns the column type for a specific row. If Next() has not been called, fallback to // sqlite3_column_decltype() func (rc *SQLiteRows) ColumnTypeScanType(i int) reflect.Type { + rc.s.mu.Lock() + defer rc.s.mu.Unlock() + + if isValidRow := C.sqlite3_stmt_busy(rc.s.s) != 0; !isValidRow { + return type_any + } + if isValidColumn := i >= 0 && i < int(rc.nc); !isValidColumn { + return type_any + } + switch C.sqlite3_column_type(rc.s.s, C.int(i)) { case C.SQLITE_INTEGER: - return TYPE_NULLINT + switch rc.decltype[i] { + case columnTimestamp, columnDatetime, columnDate: + return type_nulltime + case columnBoolean: + return type_nullbool + } + return type_nullint case C.SQLITE_FLOAT: - return TYPE_NULLFLOAT + return type_nullfloat case C.SQLITE_TEXT: - return TYPE_NULLSTRING + switch rc.decltype[i] { + case columnTimestamp, columnDatetime, columnDate: + return type_nulltime + } + return type_nullstring case C.SQLITE_BLOB: - return TYPE_RAWBYTES - // This case can signal that the value is NULL or that Next() has not been called yet. - // Skip it and return the fallback behaviour as a best effort. This is safe as all types - // returned are Nullable or any, which is the expected value for SQLite3. - //case C.SQLITE_NULL: - // return TYPE_ANY - } - - // Fallback to schema declared to remain retro-compatible - return scanType(C.GoString(C.sqlite3_column_decltype(rc.s.s, C.int(i)))) -} - -func scanType(cdt string) reflect.Type { - t := strings.ToUpper(cdt) - i := databaseTypeConvSqlite(t) - switch i { - case SQLITE_INTEGER: - return TYPE_NULLINT - case SQLITE_TEXT: - return TYPE_NULLSTRING - case SQLITE_BLOB: - return TYPE_RAWBYTES - case SQLITE_REAL: - return TYPE_NULLFLOAT - case SQLITE_NUMERIC: - return TYPE_NULLFLOAT - case SQLITE_BOOL: - return TYPE_NULLBOOL - case SQLITE_TIME: - return TYPE_NULLTIME + return type_rawbytes + case C.SQLITE_NULL: + fallthrough + default: + return type_any } - return TYPE_ANY -} - -func databaseTypeConvSqlite(t string) int { - if strings.Contains(t, "INT") { - return SQLITE_INTEGER - } - if t == "CLOB" || t == "TEXT" || - strings.Contains(t, "CHAR") { - return SQLITE_TEXT - } - if t == "BLOB" { - return SQLITE_BLOB - } - if t == "REAL" || t == "FLOAT" || - strings.Contains(t, "DOUBLE") { - return SQLITE_REAL - } - if t == "DATE" || t == "DATETIME" || - t == "TIMESTAMP" { - return SQLITE_TIME - } - if t == "NUMERIC" || - strings.Contains(t, "DECIMAL") { - return SQLITE_NUMERIC - } - if t == "BOOLEAN" { - return SQLITE_BOOL - } - - return SQLITE_NULL }