Skip to content

Commit abb8431

Browse files
maysunfaisalmaysunfaisal
authored
Merge pull request #137 from maysunfaisal/read-POC
Add Kube YAML parser/reader
2 parents 85a4805 + 4d8ed75 commit abb8431

File tree

7 files changed

+609
-1
lines changed

7 files changed

+609
-1
lines changed

‎README.md

+8
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,14 @@ The function documentation can be accessed via [pkg.go.dev](https://pkg.go.dev/g
145145
// write to the devfile on disk
146146
err = devfile.WriteYamlDevfile()
147147
```
148+
6. To parse the outerloop Kubernetes/OpenShift component's uri or inline content, call the read and parse functions
149+
```go
150+
// Read the YAML content
151+
values, err := ReadKubernetesYaml(src, fs)
152+
153+
// Get the Kubernetes resources
154+
resources, err := ParseKubernetesYaml(values)
155+
```
148156

149157
## Projects using devfile/library
150158

‎go.mod

+1
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ require (
2020
github.com/spf13/afero v1.2.2
2121
github.com/stretchr/testify v1.7.0
2222
github.com/xeipuuv/gojsonschema v1.2.0
23+
gopkg.in/yaml.v3 v3.0.1
2324
k8s.io/api v0.21.3
2425
k8s.io/apimachinery v0.21.3
2526
k8s.io/client-go v0.21.3

‎go.sum

+2-1
Original file line numberDiff line numberDiff line change
@@ -748,8 +748,9 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
748748
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
749749
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
750750
gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
751-
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
752751
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
752+
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
753+
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
753754
gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
754755
gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
755756
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

‎pkg/devfile/generator/generators.go

+1
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package generator
22

33
import (
44
"fmt"
5+
56
v1 "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"
67
"github.com/devfile/library/pkg/devfile/parser"
78
"github.com/devfile/library/pkg/devfile/parser/data/v2/common"

‎pkg/devfile/parser/reader.go

+129
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
package parser
2+
3+
import (
4+
"bytes"
5+
"io"
6+
7+
"github.com/devfile/library/pkg/testingutil/filesystem"
8+
"github.com/devfile/library/pkg/util"
9+
"github.com/pkg/errors"
10+
"gopkg.in/yaml.v3"
11+
k8yaml "sigs.k8s.io/yaml"
12+
13+
routev1 "github.com/openshift/api/route/v1"
14+
appsv1 "k8s.io/api/apps/v1"
15+
corev1 "k8s.io/api/core/v1"
16+
extensionsv1 "k8s.io/api/extensions/v1beta1"
17+
)
18+
19+
// YamlSrc specifies the src of the yaml in either Path, URL or Data format
20+
type YamlSrc struct {
21+
// Path to the yaml file
22+
Path string
23+
// URL of the yaml file
24+
URL string
25+
// Data is the yaml content in []byte format
26+
Data []byte
27+
}
28+
29+
// KubernetesResources struct contains the Deployments, Services,
30+
// Routes and Ingresses resources
31+
type KubernetesResources struct {
32+
Deployments []appsv1.Deployment
33+
Services []corev1.Service
34+
Routes []routev1.Route
35+
Ingresses []extensionsv1.Ingress
36+
}
37+
38+
// ReadKubernetesYaml reads a yaml Kubernetes file from either the Path, URL or Data provided.
39+
// It returns all the parsed Kubernetes objects as an array of interface.
40+
// Consumers interested in the Kubernetes resources are expected to Unmarshal
41+
// it to the struct of the respective Kubernetes resource.
42+
func ReadKubernetesYaml(src YamlSrc, fs filesystem.Filesystem) ([]interface{}, error) {
43+
44+
var data []byte
45+
var err error
46+
47+
if src.URL != "" {
48+
data, err = util.DownloadFileInMemory(src.URL)
49+
if err != nil {
50+
return nil, errors.Wrapf(err, "failed to download file %q", src.URL)
51+
}
52+
} else if src.Path != "" {
53+
data, err = fs.ReadFile(src.Path)
54+
if err != nil {
55+
return nil, errors.Wrapf(err, "failed to read yaml from path %q", src.Path)
56+
}
57+
} else if len(src.Data) > 0 {
58+
data = src.Data
59+
}
60+
61+
var values []interface{}
62+
dec := yaml.NewDecoder(bytes.NewReader(data))
63+
for {
64+
var value interface{}
65+
err = dec.Decode(&value)
66+
if err != nil {
67+
if err == io.EOF {
68+
break
69+
}
70+
return nil, err
71+
}
72+
values = append(values, value)
73+
}
74+
75+
return values, nil
76+
}
77+
78+
// ParseKubernetesYaml Unmarshals the interface array of the Kubernetes resources
79+
// and returns it as a KubernetesResources struct. Only Deployment, Service, Route
80+
// and Ingress are processed. Consumers interested in other Kubernetes resources
81+
// are expected to parse the values interface array an Unmarshal it to their
82+
// desired Kuberenetes struct
83+
func ParseKubernetesYaml(values []interface{}) (KubernetesResources, error) {
84+
var deployments []appsv1.Deployment
85+
var services []corev1.Service
86+
var routes []routev1.Route
87+
var ingresses []extensionsv1.Ingress
88+
89+
for _, value := range values {
90+
var deployment appsv1.Deployment
91+
var service corev1.Service
92+
var route routev1.Route
93+
var ingress extensionsv1.Ingress
94+
95+
byteData, err := k8yaml.Marshal(value)
96+
if err != nil {
97+
return KubernetesResources{}, err
98+
}
99+
100+
kubernetesMap := value.(map[string]interface{})
101+
kind := kubernetesMap["kind"]
102+
103+
switch kind {
104+
case "Deployment":
105+
err = k8yaml.Unmarshal(byteData, &deployment)
106+
deployments = append(deployments, deployment)
107+
case "Service":
108+
err = k8yaml.Unmarshal(byteData, &service)
109+
services = append(services, service)
110+
case "Route":
111+
err = k8yaml.Unmarshal(byteData, &route)
112+
routes = append(routes, route)
113+
case "Ingress":
114+
err = k8yaml.Unmarshal(byteData, &ingress)
115+
ingresses = append(ingresses, ingress)
116+
}
117+
118+
if err != nil {
119+
return KubernetesResources{}, err
120+
}
121+
}
122+
123+
return KubernetesResources{
124+
Deployments: deployments,
125+
Services: services,
126+
Routes: routes,
127+
Ingresses: ingresses,
128+
}, nil
129+
}

‎pkg/devfile/parser/reader_test.go

+189
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
package parser
2+
3+
import (
4+
"net"
5+
"net/http"
6+
"net/http/httptest"
7+
"reflect"
8+
"testing"
9+
10+
"github.com/devfile/library/pkg/testingutil/filesystem"
11+
"github.com/devfile/library/pkg/util"
12+
"github.com/stretchr/testify/assert"
13+
)
14+
15+
func TestReadAndParseKubernetesYaml(t *testing.T) {
16+
const serverIP = "127.0.0.1:9080"
17+
var data []byte
18+
19+
fs := filesystem.DefaultFs{}
20+
absPath, err := util.GetAbsPath("../../../tests/yamls/resources.yaml")
21+
if err != nil {
22+
t.Error(err)
23+
return
24+
}
25+
26+
data, err = fs.ReadFile(absPath)
27+
if err != nil {
28+
t.Error(err)
29+
return
30+
}
31+
32+
// Mocking the YAML file endpoint on a very basic level
33+
testServer := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
34+
_, err = w.Write(data)
35+
if err != nil {
36+
t.Errorf("Unexpected error while writing data: %v", err)
37+
}
38+
}))
39+
// create a listener with the desired port.
40+
l, err := net.Listen("tcp", serverIP)
41+
if err != nil {
42+
t.Errorf("Unexpected error while creating listener: %v", err)
43+
return
44+
}
45+
46+
// NewUnstartedServer creates a listener. Close that listener and replace
47+
// with the one we created.
48+
testServer.Listener.Close()
49+
testServer.Listener = l
50+
51+
testServer.Start()
52+
defer testServer.Close()
53+
54+
badData := append(data, 59)
55+
56+
tests := []struct {
57+
name string
58+
src YamlSrc
59+
fs filesystem.Filesystem
60+
wantErr bool
61+
wantDeploymentNames []string
62+
wantServiceNames []string
63+
wantRouteNames []string
64+
wantIngressNames []string
65+
wantOtherNames []string
66+
}{
67+
{
68+
name: "Read the YAML from the URL",
69+
src: YamlSrc{
70+
URL: "http://" + serverIP,
71+
},
72+
fs: filesystem.DefaultFs{},
73+
wantDeploymentNames: []string{"deploy-sample", "deploy-sample-2"},
74+
wantServiceNames: []string{"service-sample", "service-sample-2"},
75+
wantRouteNames: []string{"route-sample", "route-sample-2"},
76+
wantIngressNames: []string{"ingress-sample", "ingress-sample-2"},
77+
wantOtherNames: []string{"pvc-sample", "pvc-sample-2"},
78+
},
79+
{
80+
name: "Read the YAML from the Path",
81+
src: YamlSrc{
82+
Path: "../../../tests/yamls/resources.yaml",
83+
},
84+
fs: filesystem.DefaultFs{},
85+
wantDeploymentNames: []string{"deploy-sample", "deploy-sample-2"},
86+
wantServiceNames: []string{"service-sample", "service-sample-2"},
87+
wantRouteNames: []string{"route-sample", "route-sample-2"},
88+
wantIngressNames: []string{"ingress-sample", "ingress-sample-2"},
89+
wantOtherNames: []string{"pvc-sample", "pvc-sample-2"},
90+
},
91+
{
92+
name: "Read the YAML from the Data",
93+
src: YamlSrc{
94+
Data: data,
95+
},
96+
fs: filesystem.DefaultFs{},
97+
wantDeploymentNames: []string{"deploy-sample", "deploy-sample-2"},
98+
wantServiceNames: []string{"service-sample", "service-sample-2"},
99+
wantRouteNames: []string{"route-sample", "route-sample-2"},
100+
wantIngressNames: []string{"ingress-sample", "ingress-sample-2"},
101+
wantOtherNames: []string{"pvc-sample", "pvc-sample-2"},
102+
},
103+
{
104+
name: "Bad URL",
105+
src: YamlSrc{
106+
URL: "http://badurl",
107+
},
108+
fs: filesystem.DefaultFs{},
109+
wantErr: true,
110+
},
111+
{
112+
name: "Bad Path",
113+
src: YamlSrc{
114+
Path: "$%^&",
115+
},
116+
fs: filesystem.DefaultFs{},
117+
wantErr: true,
118+
},
119+
{
120+
name: "Bad Data",
121+
src: YamlSrc{
122+
Data: badData,
123+
},
124+
fs: filesystem.DefaultFs{},
125+
wantErr: true,
126+
},
127+
}
128+
129+
for _, tt := range tests {
130+
t.Run(tt.name, func(t *testing.T) {
131+
values, err := ReadKubernetesYaml(tt.src, tt.fs)
132+
if (err != nil) != tt.wantErr {
133+
t.Errorf("unexpected error: %v", err)
134+
return
135+
}
136+
137+
for _, value := range values {
138+
kubernetesMap := value.(map[string]interface{})
139+
140+
kind := kubernetesMap["kind"]
141+
metadataMap := kubernetesMap["metadata"].(map[string]interface{})
142+
name := metadataMap["name"]
143+
144+
switch kind {
145+
case "Deployment":
146+
assert.Contains(t, tt.wantDeploymentNames, name)
147+
case "Service":
148+
assert.Contains(t, tt.wantServiceNames, name)
149+
case "Route":
150+
assert.Contains(t, tt.wantRouteNames, name)
151+
case "Ingress":
152+
assert.Contains(t, tt.wantIngressNames, name)
153+
default:
154+
assert.Contains(t, tt.wantOtherNames, name)
155+
}
156+
}
157+
158+
if len(values) > 0 {
159+
resources, err := ParseKubernetesYaml(values)
160+
if err != nil {
161+
t.Error(err)
162+
return
163+
}
164+
165+
if reflect.DeepEqual(resources, KubernetesResources{}) {
166+
t.Error("Kubernetes resources is empty, expected to contain some resources")
167+
} else {
168+
deployments := resources.Deployments
169+
services := resources.Services
170+
routes := resources.Routes
171+
ingresses := resources.Ingresses
172+
173+
for _, deploy := range deployments {
174+
assert.Contains(t, tt.wantDeploymentNames, deploy.Name)
175+
}
176+
for _, svc := range services {
177+
assert.Contains(t, tt.wantServiceNames, svc.Name)
178+
}
179+
for _, route := range routes {
180+
assert.Contains(t, tt.wantRouteNames, route.Name)
181+
}
182+
for _, ingress := range ingresses {
183+
assert.Contains(t, tt.wantIngressNames, ingress.Name)
184+
}
185+
}
186+
}
187+
})
188+
}
189+
}

0 commit comments

Comments
 (0)