Skip to content

Commit 70b28cb

Browse files
TomWrightTomWright
committed
Add WIP CSV parser and general changes
1 parent 7e48c95 commit 70b28cb

File tree

7 files changed

+299
-8
lines changed

7 files changed

+299
-8
lines changed

‎cmd/dasel/main.go

+1
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"os"
66

77
"github.com/tomwright/dasel/v3/internal/cli"
8+
_ "github.com/tomwright/dasel/v3/parsing/csv"
89
_ "github.com/tomwright/dasel/v3/parsing/d"
910
_ "github.com/tomwright/dasel/v3/parsing/hcl"
1011
_ "github.com/tomwright/dasel/v3/parsing/json"

‎internal/cli/generic_test.go

-4
Original file line numberDiff line numberDiff line change
@@ -192,10 +192,6 @@ sliceOfNumbers = [1, 2, 3, 4, 5]
192192
newStringWithFormat(yaml.YAML, `123`),
193193
newStringWithFormat(toml.TOML, `123`),
194194
},
195-
skip: []string{
196-
// Skipped because the parser outputs as a float.
197-
"json to toml",
198-
},
199195
}.run)
200196
t.Run("float", testCases{
201197
selector: prefix + "oneTwoDotThree",

‎model/value.go

+2
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ type KeyValue struct {
3333
// Values represents a list of values.
3434
type Values []*Value
3535

36+
// ToSliceValue converts a list of values to a slice value.
3637
func (v Values) ToSliceValue() (*Value, error) {
3738
slice := NewSliceValue()
3839
for _, val := range v {
@@ -51,6 +52,7 @@ type Value struct {
5152
setFn func(*Value) error
5253
}
5354

55+
// String returns the value as a formatted string, along with type info.
5456
func (v *Value) String() string {
5557
return v.string(0)
5658
}

‎parsing/csv/csv.go

+161
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,161 @@
1+
package csv
2+
3+
import (
4+
"bytes"
5+
"encoding/csv"
6+
"errors"
7+
"fmt"
8+
"io"
9+
10+
"github.com/tomwright/dasel/v3/model"
11+
"github.com/tomwright/dasel/v3/parsing"
12+
)
13+
14+
// CSV represents the CSV file format.
15+
const CSV parsing.Format = "csv"
16+
17+
var _ parsing.Reader = (*csvReader)(nil)
18+
var _ parsing.Writer = (*csvWriter)(nil)
19+
20+
func init() {
21+
parsing.RegisterReader(CSV, newCSVReader)
22+
parsing.RegisterWriter(CSV, newCSVWriter)
23+
}
24+
25+
func newCSVReader(options parsing.ReaderOptions) (parsing.Reader, error) {
26+
return &csvReader{
27+
separator: ',',
28+
}, nil
29+
}
30+
31+
func newCSVWriter(options parsing.WriterOptions) (parsing.Writer, error) {
32+
return &csvWriter{
33+
separator: ',',
34+
}, nil
35+
}
36+
37+
type csvReader struct {
38+
separator rune
39+
}
40+
41+
// Read reads a value from a byte slice.
42+
func (j *csvReader) Read(data []byte) (*model.Value, error) {
43+
r := csv.NewReader(bytes.NewReader(data))
44+
r.Comma = j.separator
45+
46+
res := model.NewSliceValue()
47+
48+
var headers []string
49+
50+
for rowI := 0; ; rowI++ {
51+
record, err := r.Read()
52+
if err != nil {
53+
if errors.Is(err, io.EOF) {
54+
break
55+
}
56+
return nil, err
57+
}
58+
59+
if headers == nil {
60+
for _, header := range record {
61+
headers = append(headers, header)
62+
}
63+
continue
64+
}
65+
66+
row := model.NewMapValue()
67+
for colI, field := range record {
68+
if colI >= len(headers) {
69+
return nil, fmt.Errorf("row %d has more columns than headers", rowI)
70+
}
71+
headerKey := headers[colI]
72+
73+
colV, err := valueFromString(field)
74+
if err != nil {
75+
return nil, fmt.Errorf("failed reading column %q for row %d: %w", field, colI, err)
76+
}
77+
78+
if err := row.SetMapKey(headerKey, colV); err != nil {
79+
return nil, fmt.Errorf("failed to set map key %q: %w", headerKey, err)
80+
}
81+
}
82+
83+
if err := res.Append(row); err != nil {
84+
return nil, fmt.Errorf("failed to append row %d: %w", rowI, err)
85+
}
86+
}
87+
88+
return res, nil
89+
}
90+
91+
type csvWriter struct {
92+
separator rune
93+
}
94+
95+
// Write writes a value to a byte slice.
96+
func (j *csvWriter) Write(value *model.Value) ([]byte, error) {
97+
if !value.IsSlice() {
98+
return nil, fmt.Errorf("csv writer expects root output to be a slice/array, got %s", value.Type())
99+
}
100+
101+
buf := new(bytes.Buffer)
102+
w := csv.NewWriter(buf)
103+
w.Comma = j.separator
104+
105+
var headers []string
106+
107+
if err := value.RangeSlice(func(i int, row *model.Value) error {
108+
if i == 0 {
109+
var err error
110+
headers, err = row.MapKeys()
111+
if err != nil {
112+
return fmt.Errorf("error getting map keys: %w", err)
113+
}
114+
if err := w.Write(headers); err != nil {
115+
return fmt.Errorf("error writing headers: %w", err)
116+
}
117+
}
118+
119+
var values []string
120+
121+
for _, headerKey := range headers {
122+
colV, err := row.GetMapKey(headerKey)
123+
if err != nil {
124+
return fmt.Errorf("error getting map key %q: %w", headerKey, err)
125+
}
126+
127+
csvVal, err := valueToString(colV)
128+
if err != nil {
129+
return fmt.Errorf("error converting value to string: %w", err)
130+
}
131+
132+
values = append(values, csvVal)
133+
}
134+
135+
fmt.Println("headers", headers, "values", values)
136+
137+
if err := w.Write(values); err != nil {
138+
return fmt.Errorf("error writing row: %w", err)
139+
}
140+
141+
return nil
142+
}); err != nil {
143+
return nil, fmt.Errorf("error ranging slice: %w", err)
144+
}
145+
146+
w.Flush()
147+
148+
return buf.Bytes(), nil
149+
}
150+
151+
func valueFromString(s string) (*model.Value, error) {
152+
return model.NewStringValue(s), nil
153+
}
154+
155+
func valueToString(v *model.Value) (string, error) {
156+
if v.IsNull() {
157+
return "", nil
158+
}
159+
// TODO : Parse based on type. Do not assume string.
160+
return v.String(), nil
161+
}

‎parsing/json/json.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -220,7 +220,7 @@ func (j *jsonWriter) Write(value *model.Value) ([]byte, error) {
220220
return err
221221
}
222222

223-
if value.IsBranch() {
223+
if value.IsBranch() || value.IsSpread() {
224224
if err := value.RangeSlice(func(i int, v *model.Value) error {
225225
if err := j.write(buf, encoderFn, es, v); err != nil {
226226
return err

‎parsing/xml/writer.go

+133-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
package xml
22

33
import (
4+
"bytes"
5+
"encoding/xml"
6+
"fmt"
7+
"strings"
8+
49
"github.com/tomwright/dasel/v3/model"
510
"github.com/tomwright/dasel/v3/parsing"
611
)
@@ -17,5 +22,132 @@ type xmlWriter struct {
1722

1823
// Write writes a value to a byte slice.
1924
func (j *xmlWriter) Write(value *model.Value) ([]byte, error) {
20-
return nil, nil
25+
buf := new(bytes.Buffer)
26+
writer := xml.NewEncoder(buf)
27+
writer.Indent("", " ")
28+
29+
element, err := j.toElement(value)
30+
if err != nil {
31+
return nil, fmt.Errorf("failed to convert to element: %w", err)
32+
}
33+
34+
if err := writer.Encode(element); err != nil {
35+
return nil, err
36+
}
37+
38+
if err := writer.Flush(); err != nil {
39+
return nil, err
40+
}
41+
return buf.Bytes(), nil
42+
}
43+
44+
func (j *xmlWriter) toElement(value *model.Value) (*xmlElement, error) {
45+
switch value.Type() {
46+
47+
case model.TypeString:
48+
return &xmlElement{
49+
Name: "root",
50+
Content: value.String(),
51+
}, nil
52+
53+
case model.TypeMap:
54+
kvs, err := value.MapKeyValues()
55+
if err != nil {
56+
return nil, err
57+
}
58+
59+
el := &xmlElement{
60+
Name: "root",
61+
}
62+
63+
for _, kv := range kvs {
64+
if strings.HasPrefix(kv.Key, "-") {
65+
attr := xmlAttr{
66+
Name: kv.Key[1:],
67+
}
68+
attr.Value, err = valueToString(kv.Value)
69+
if err != nil {
70+
return nil, fmt.Errorf("failed to convert attribute %q to string: %w", attr.Name, err)
71+
}
72+
el.Attrs = append(el.Attrs, attr)
73+
continue
74+
}
75+
76+
if kv.Key == "#text" {
77+
el.Content, err = valueToString(kv.Value)
78+
if err != nil {
79+
return nil, fmt.Errorf("failed to convert content to string: %w", err)
80+
}
81+
continue
82+
}
83+
84+
childEl, err := j.toElement(kv.Value)
85+
if err != nil {
86+
return nil, fmt.Errorf("failed to convert child element %q to element: %w", kv.Key, err)
87+
}
88+
childEl.Name = kv.Key
89+
el.Children = append(el.Children, childEl)
90+
}
91+
92+
return el, nil
93+
case model.TypeSlice:
94+
el := &xmlElement{
95+
Name: "root",
96+
}
97+
if err := value.RangeSlice(func(i int, value *model.Value) error {
98+
childEl, err := j.toElement(value)
99+
if err != nil {
100+
return err
101+
}
102+
childEl.Name = "item"
103+
el.Children = append(el.Children, childEl)
104+
105+
return nil
106+
}); err != nil {
107+
return nil, err
108+
}
109+
return el, nil
110+
default:
111+
return nil, fmt.Errorf("xml writer does not support value type: %s", value.Type())
112+
}
113+
}
114+
115+
func valueToString(v *model.Value) (string, error) {
116+
if v.IsNull() {
117+
return "", nil
118+
}
119+
// TODO : Parse based on type. Do not assume string.
120+
return v.String(), nil
121+
}
122+
123+
func (e *xmlElement) MarshalXML(enc *xml.Encoder, start xml.StartElement) error {
124+
start.Name = xml.Name{Local: e.Name}
125+
if err := enc.EncodeToken(start); err != nil {
126+
return err
127+
}
128+
129+
if len(e.Attrs) > 0 {
130+
for _, attr := range e.Attrs {
131+
if err := enc.EncodeToken(xml.Attr{
132+
Name: xml.Name{Local: attr.Name},
133+
Value: attr.Value,
134+
}); err != nil {
135+
return err
136+
}
137+
}
138+
}
139+
140+
if len(e.Content) > 0 {
141+
if err := enc.EncodeToken(xml.CharData(e.Content)); err != nil {
142+
return err
143+
}
144+
}
145+
146+
for _, child := range e.Children {
147+
if err := enc.Encode(child); err != nil {
148+
return err
149+
}
150+
}
151+
152+
return enc.EncodeToken(start.End())
21153
}

‎parsing/xml/xml.go

+1-2
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,7 @@ var _ parsing.Writer = (*xmlWriter)(nil)
1414

1515
func init() {
1616
parsing.RegisterReader(XML, newXMLReader)
17-
// XML writer is not implemented yet
18-
//parsing.RegisterWriter(XML, newXMLWriter)
17+
parsing.RegisterWriter(XML, newXMLWriter)
1918
}
2019

2120
type xmlAttr struct {

0 commit comments

Comments
 (0)