feat: Add arrow support for timestamp and bytea (#724)

This should unblock cloudquery/filetypes#87

Add supports for timestamp to decode arrow string and for byte to decode base64 (backward compat is saved)

Author: yevgenypats

schema/bytea.go

@@ -3,6 +3,7 @@ package schema
 
 import (
 	"bytes"
+	"encoding/base64"
 	"encoding/hex"
 )
 
@@ -41,7 +42,7 @@ func (dst *Bytea) Equal(src CQType) bool {
 
 func (dst *Bytea) String() string {
 	if dst.Status == Present {
-		return hex.EncodeToString(dst.Bytes)
+		return base64.StdEncoding.EncodeToString(dst.Bytes)
 	} else {
 		return ""
 	}
@@ -72,7 +73,11 @@ func (dst *Bytea) Set(src any) error {
 			b := make([]byte, hex.DecodedLen(len(value)))
 			_, err := hex.Decode(b, []byte(value))
 			if err != nil {
-				return &ValidationError{Type: TypeByteArray, Msg: "hex decode failed", Err: err, Value: value}
+				b = make([]byte, base64.StdEncoding.DecodedLen(len(value)))
+				_, err := base64.StdEncoding.Decode(b, []byte(value))
+				if err != nil {
+					return &ValidationError{Type: TypeByteArray, Msg: "base64 and hex decode failed", Err: err, Value: value}
+				}
 			}
 			*dst = Bytea{Status: Present, Bytes: b}
 		} else {

schema/resource_test.go

@@ -438,7 +438,7 @@ var calculateCQIDPrimaryKeyTestCases = []struct {
 			"MacAddressValue":      "aa:bb:cc:dd:ee:ff",
 			"MacAddressArrayValue": []string{"aa:bb:cc:dd:ee:ff", "11:22:33:44:55:66"},
 		},
-		ExpectedValue:     &UUID{Bytes: [16]uint8{0xa6, 0x76, 0x76, 0xfb, 0x4c, 0x95, 0x51, 0x2d, 0x9e, 0xcd, 0xf4, 0xc4, 0xae, 0xc1, 0x2a, 0xf5}, Status: 0x2},
+		ExpectedValue:     &UUID{Bytes: [16]uint8{0x82, 0xaa, 0xa2, 0xc7, 0x75, 0x10, 0x5c, 0x6d, 0xb8, 0xef, 0x18, 0x49, 0x33, 0x99, 0x4a, 0x9d}, Status: 0x2},
 		DeterministicCQID: true,
 	},
 }

schema/timestamptz.go

@@ -14,6 +14,9 @@ import (
 // this is the default format used by time.Time.String()
 const defaultStringFormat = "2006-01-02 15:04:05.999999999 -0700 MST"
 
+// this is used by arrow string format (time is in UTC)
+const arrowStringFormat = "2006-01-02 15:04:05.999999999"
+
 // const microsecFromUnixEpochToY2K = 946684800 * 1000000
 
 const (
@@ -170,6 +173,11 @@ func (dst *Timestamptz) DecodeText(src []byte) error {
 			*dst = Timestamptz{Time: tim.UTC(), Status: Present}
 			return nil
 		}
+		tim, err = time.Parse(arrowStringFormat, sbuf)
+		if err == nil {
+			*dst = Timestamptz{Time: tim.UTC(), Status: Present}
+			return nil
+		}
 		return &ValidationError{Type: TypeTimestamp, Msg: "cannot parse timestamp", Value: sbuf, Err: err}
 	}
 

testdata/testdata.go

@@ -183,7 +183,7 @@ func GenTestData(table *schema.Table) schema.CQTypes {
 			data[i] = uuidArrayColumn
 		case schema.TypeInet:
 			inetColumn := &schema.Inet{}
-			if err := inetColumn.Set("192.0.2.1/24"); err != nil {
+			if err := inetColumn.Set("192.0.2.0/24"); err != nil {
 				panic(err)
 			}
 			data[i] = inetColumn