From f27da6d09d51a839984528582e23cf79cc97e592 Mon Sep 17 00:00:00 2001 From: Brent Woodruff Date: Wed, 8 Jun 2022 16:03:54 -0400 Subject: [PATCH 1/2] port to jsonparser to enable tinygo/wasm --- go.mod | 5 +++- go.sum | 2 ++ main.go | 78 ++++++++++++++++++++++++++++++++++----------------------- 3 files changed, 52 insertions(+), 33 deletions(-) diff --git a/go.mod b/go.mod index 335e738..b073580 100644 --- a/go.mod +++ b/go.mod @@ -2,4 +2,7 @@ module github.com/jehiah/json2csv go 1.13 -require github.com/stretchr/testify v1.4.0 +require ( + github.com/buger/jsonparser v1.1.1 + github.com/stretchr/testify v1.4.0 +) diff --git a/go.sum b/go.sum index 8fdee58..c037162 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,5 @@ +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/main.go b/main.go index eca0ae4..7b471ed 100644 --- a/main.go +++ b/main.go @@ -3,7 +3,6 @@ package main import ( "bufio" "encoding/csv" - "encoding/json" "flag" "fmt" "io" @@ -11,8 +10,11 @@ import ( "math" "os" "runtime" + "strconv" "strings" "unicode/utf8" + + "github.com/buger/jsonparser" ) type LineReader interface { @@ -64,29 +66,6 @@ func main() { json2csv(reader, writer, keys, *printHeader) } -func get_value(data map[string]interface{}, keyparts []string) string { - if len(keyparts) > 1 { - subdata, _ := data[keyparts[0]].(map[string]interface{}) - return get_value(subdata, keyparts[1:]) - } else if v, ok := data[keyparts[0]]; ok { - switch v.(type) { - case nil: - return "" - case float64: - f, _ := v.(float64) - if math.Mod(f, 1.0) == 0.0 { - return fmt.Sprintf("%d", int(f)) - } else { - return fmt.Sprintf("%f", f) - } - default: - return fmt.Sprintf("%+v", v) - } - } - - return "" -} - func json2csv(r LineReader, w *csv.Writer, keys []string, printHeader bool) { var line []byte var err error @@ -119,19 +98,54 @@ func json2csv(r LineReader, w *csv.Writer, keys []string, printHeader bool) { printHeader = false } - var data map[string]interface{} - err = json.Unmarshal(line, &data) - if err != nil { - log.Printf("ERROR Decoding JSON at line %d: %s\n%s", line_count, err, line) - continue - } - var record []string for _, expanded_key := range expanded_keys { - record = append(record, get_value(data, expanded_key)) + //val, valuetype, offset, err + val, vt, _, err := jsonparser.Get(line, expanded_key...) + + if err != nil { + log.Printf("ERROR Retrieving JSON key %s at line %d: %s\n%s", + strings.Join(expanded_key[:], "."), line_count, err, line) + record = append(record, "") + continue + } + + value, err := convertJsonValue(val, vt) + + if err != nil { + log.Printf("ERROR Decoding JSON at line %d: %s\n%s", line_count, err, line) + continue + } + + record = append(record, value) } w.Write(record) w.Flush() } } + +func convertJsonValue(v []byte, vt jsonparser.ValueType) (string, error) { + switch vt { + case jsonparser.String: + return string(v[:]), nil + case jsonparser.Boolean: + return fmt.Sprintf("+%v", string(v[:])), nil + case jsonparser.Number: + f, _ := strconv.ParseFloat(string(v[:]), 64) + if math.Mod(f, 1.0) == 0.0 { + return fmt.Sprintf("%d", int(f)), nil + } else { + return fmt.Sprintf("%f", f), nil + } + case jsonparser.NotExist: + case jsonparser.Null: + return "", nil + case jsonparser.Object: + return "", fmt.Errorf("JSON value is an object: %s", v) + case jsonparser.Array: + return "", fmt.Errorf("JSON value is an array: %s", v) + } + + return "", fmt.Errorf("JSON value is an unknown type: %s", v) +} From 94daedcb4ade7ad82161e670525ae1e89e75bb70 Mon Sep 17 00:00:00 2001 From: Brent Woodruff Date: Wed, 8 Jun 2022 16:22:36 -0400 Subject: [PATCH 2/2] no error output on key not found --- main.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/main.go b/main.go index 7b471ed..f98f149 100644 --- a/main.go +++ b/main.go @@ -3,6 +3,7 @@ package main import ( "bufio" "encoding/csv" + "errors" "flag" "fmt" "io" @@ -104,10 +105,15 @@ func json2csv(r LineReader, w *csv.Writer, keys []string, printHeader bool) { val, vt, _, err := jsonparser.Get(line, expanded_key...) if err != nil { - log.Printf("ERROR Retrieving JSON key %s at line %d: %s\n%s", - strings.Join(expanded_key[:], "."), line_count, err, line) - record = append(record, "") - continue + if errors.Is(err, jsonparser.KeyPathNotFoundError) { + record = append(record, "") + continue + } else { + log.Printf("ERROR Retrieving JSON key %s at line %d: %s\n%s", + strings.Join(expanded_key[:], "."), line_count, err, line) + record = append(record, "") + continue + } } value, err := convertJsonValue(val, vt)