Skip to content

Commit eb1cd3b

Browse files
yangcao77yangcao77
authoredMar 12, 2021
Merge pull request #63 from yangcao77/354-parse-parent-from-registry
parse parent devfile from devfile registry
·
v2.3.0v1.0.0-alpha.3
2 parents 3ea632c + d8423c3 commit eb1cd3b

File tree

6 files changed

+611
-85
lines changed

6 files changed

+611
-85
lines changed
 

‎devfile.yaml

+3-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@ metadata:
55
attributes:
66
alpha.build-dockerfile: /relative/path/to/Dockerfile
77
parent:
8-
uri: https://raw.githubusercontent.com/odo-devfiles/registry/master/devfiles/nodejs/devfile.yaml
8+
# uri: https://raw.githubusercontent.com/odo-devfiles/registry/master/devfiles/nodejs/devfile.yaml
9+
id: nodejs
10+
registryUrl: "https://registry.devfile.io"
911
commands:
1012
- id: install
1113
exec:

‎main.go

+12-13
Original file line numberDiff line numberDiff line change
@@ -20,28 +20,27 @@ func main() {
2020
}
2121
}
2222

23-
//ParseDevfile to parse devfile from library
24-
func ParseDevfile(devfileLocation string) (parser.DevfileObj, error) {
25-
26-
devfile, err := devfilepkg.ParseAndValidate(devfileLocation)
27-
return devfile, err
28-
}
29-
3023
func parserTest() {
31-
var devfile parser.DevfileObj
32-
var err error
24+
var args parser.ParserArgs
3325
if len(os.Args) > 1 {
3426
if strings.HasPrefix(os.Args[1], "http") {
35-
devfile, err = devfilepkg.ParseFromURLAndValidate(os.Args[1])
27+
args = parser.ParserArgs{
28+
URL: os.Args[1],
29+
}
3630
} else {
37-
devfile, err = ParseDevfile(os.Args[1])
31+
args = parser.ParserArgs{
32+
Path: os.Args[1],
33+
}
3834
}
3935
fmt.Println("parsing devfile from " + os.Args[1])
4036

4137
} else {
42-
devfile, err = ParseDevfile("devfile.yaml")
43-
fmt.Println("parsing devfile from " + devfile.Ctx.GetAbsPath())
38+
args = parser.ParserArgs{
39+
Path: "devfile.yaml",
40+
}
41+
fmt.Println("parsing devfile from ./devfile.yaml")
4442
}
43+
devfile, err := devfilepkg.ParseDevfileAndValidate(args)
4544
if err != nil {
4645
fmt.Println(err)
4746
} else {

‎pkg/devfile/parse.go

+22
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
// and validates the devfile integrity with the schema
1010
// and validates the devfile data.
1111
// Creates devfile context and runtime objects.
12+
// Deprecated, use ParseDevfileAndValidate() instead
1213
func ParseFromURLAndValidate(url string) (d parser.DevfileObj, err error) {
1314

1415
// read and parse devfile from the given URL
@@ -30,6 +31,7 @@ func ParseFromURLAndValidate(url string) (d parser.DevfileObj, err error) {
3031
// and validates the devfile integrity with the schema
3132
// and validates the devfile data.
3233
// Creates devfile context and runtime objects.
34+
// Deprecated, use ParseDevfileAndValidate() instead
3335
func ParseFromDataAndValidate(data []byte) (d parser.DevfileObj, err error) {
3436
// read and parse devfile from the given bytes
3537
d, err = parser.ParseFromData(data)
@@ -49,6 +51,7 @@ func ParseFromDataAndValidate(data []byte) (d parser.DevfileObj, err error) {
4951
// and validates the devfile integrity with the schema
5052
// and validates the devfile data.
5153
// Creates devfile context and runtime objects.
54+
// Deprecated, use ParseDevfileAndValidate() instead
5255
func ParseAndValidate(path string) (d parser.DevfileObj, err error) {
5356

5457
// read and parse devfile from given path
@@ -65,3 +68,22 @@ func ParseAndValidate(path string) (d parser.DevfileObj, err error) {
6568

6669
return d, err
6770
}
71+
72+
// ParseDevfileAndValidate func parses the devfile data
73+
// and validates the devfile integrity with the schema
74+
// and validates the devfile data.
75+
// Creates devfile context and runtime objects.
76+
func ParseDevfileAndValidate(args parser.ParserArgs) (d parser.DevfileObj, err error) {
77+
d, err = parser.ParseDevfile(args)
78+
if err != nil {
79+
return d, err
80+
}
81+
82+
// generic validation on devfile content
83+
err = validate.ValidateDevfileData(d.Data)
84+
if err != nil {
85+
return d, err
86+
}
87+
88+
return d, err
89+
}

‎pkg/devfile/parser/context/context.go

+13
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,9 @@ type DevfileCtx struct {
3535

3636
// trace of all url referenced
3737
uriMap map[string]bool
38+
39+
// registry URLs list
40+
registryURLs []string
3841
}
3942

4043
// NewDevfileCtx returns a new DevfileCtx type object
@@ -149,3 +152,13 @@ func (d *DevfileCtx) GetURIMap() map[string]bool {
149152
func (d *DevfileCtx) SetURIMap(uriMap map[string]bool) {
150153
d.uriMap = uriMap
151154
}
155+
156+
// GetRegistryURLs func returns current devfile registry URLs
157+
func (d *DevfileCtx) GetRegistryURLs() []string {
158+
return d.registryURLs
159+
}
160+
161+
// SetRegistryURLs set registry URLs in the devfile ctx
162+
func (d *DevfileCtx) SetRegistryURLs(registryURLs []string) {
163+
d.registryURLs = registryURLs
164+
}

‎pkg/devfile/parser/parse.go

+114-42
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package parser
33
import (
44
"encoding/json"
55
"fmt"
6+
"github.com/devfile/library/pkg/util"
67
"net/url"
78
"path"
89
"strings"
@@ -53,60 +54,108 @@ func parseDevfile(d DevfileObj, flattenedDevfile bool) (DevfileObj, error) {
5354
return d, nil
5455
}
5556

56-
// Parse func populates the flattened devfile data, parses and validates the devfile integrity.
57+
// ParserArgs is the struct to pass into parser functions which contains required info for parsing devfile.
58+
// It accepts devfile path, devfile URL or devfile content in []byte format.
59+
type ParserArgs struct {
60+
// Path is a relative or absolute devfile path.
61+
Path string
62+
// URL is the URL address of the specific devfile.
63+
URL string
64+
// Data is the devfile content in []byte format.
65+
Data []byte
66+
// FlattenedDevfile defines if the returned devfileObj is flattened content (true) or raw content (false).
67+
// The value is default to be true.
68+
FlattenedDevfile *bool
69+
// RegistryURLs is a list of registry hosts which parser should pull parent devfile from.
70+
// If registryUrl is defined in devfile, this list will be ignored.
71+
RegistryURLs []string
72+
}
73+
74+
// ParseDevfile func populates the devfile data, parses and validates the devfile integrity.
5775
// Creates devfile context and runtime objects
58-
func Parse(path string) (d DevfileObj, err error) {
76+
func ParseDevfile(args ParserArgs) (d DevfileObj, err error) {
77+
if args.Data != nil {
78+
d.Ctx = devfileCtx.DevfileCtx{}
79+
err = d.Ctx.SetDevfileContentFromBytes(args.Data)
80+
if err != nil {
81+
return d, errors.Wrap(err, "failed to set devfile content from bytes")
82+
}
83+
} else if args.Path != "" {
84+
d.Ctx = devfileCtx.NewDevfileCtx(args.Path)
85+
} else if args.URL != "" {
86+
d.Ctx = devfileCtx.NewURLDevfileCtx(args.URL)
87+
} else {
88+
return d, errors.Wrap(err, "the devfile source is not provided")
89+
}
5990

60-
// NewDevfileCtx
61-
d.Ctx = devfileCtx.NewDevfileCtx(path)
91+
if args.RegistryURLs != nil {
92+
d.Ctx.SetRegistryURLs(args.RegistryURLs)
93+
}
94+
95+
flattenedDevfile := true
96+
if args.FlattenedDevfile != nil {
97+
flattenedDevfile = *args.FlattenedDevfile
98+
}
99+
100+
return populateAndParseDevfile(d, flattenedDevfile)
101+
}
102+
103+
func populateAndParseDevfile(d DevfileObj, flattenedDevfile bool) (DevfileObj, error) {
104+
var err error
62105

63106
// Fill the fields of DevfileCtx struct
64-
err = d.Ctx.Populate()
107+
if d.Ctx.GetURL() != "" {
108+
err = d.Ctx.PopulateFromURL()
109+
} else if d.Ctx.GetDevfileContent() != nil {
110+
err = d.Ctx.PopulateFromRaw()
111+
} else {
112+
err = d.Ctx.Populate()
113+
}
65114
if err != nil {
66115
return d, err
67116
}
68-
return parseDevfile(d, true)
117+
118+
return parseDevfile(d, flattenedDevfile)
119+
}
120+
121+
// Parse func populates the flattened devfile data, parses and validates the devfile integrity.
122+
// Creates devfile context and runtime objects
123+
// Deprecated, use ParseDevfile() instead
124+
func Parse(path string) (d DevfileObj, err error) {
125+
126+
// NewDevfileCtx
127+
d.Ctx = devfileCtx.NewDevfileCtx(path)
128+
129+
return populateAndParseDevfile(d, true)
69130
}
70131

71132
// ParseRawDevfile populates the raw devfile data without overriding and merging
133+
// Deprecated, use ParseDevfile() instead
72134
func ParseRawDevfile(path string) (d DevfileObj, err error) {
73135
// NewDevfileCtx
74136
d.Ctx = devfileCtx.NewDevfileCtx(path)
75137

76-
// Fill the fields of DevfileCtx struct
77-
err = d.Ctx.Populate()
78-
if err != nil {
79-
return d, err
80-
}
81-
return parseDevfile(d, false)
138+
return populateAndParseDevfile(d, false)
82139
}
83140

84141
// ParseFromURL func parses and validates the devfile integrity.
85142
// Creates devfile context and runtime objects
143+
// Deprecated, use ParseDevfile() instead
86144
func ParseFromURL(url string) (d DevfileObj, err error) {
87145
d.Ctx = devfileCtx.NewURLDevfileCtx(url)
88-
// Fill the fields of DevfileCtx struct
89-
err = d.Ctx.PopulateFromURL()
90-
if err != nil {
91-
return d, err
92-
}
93-
return parseDevfile(d, true)
146+
return populateAndParseDevfile(d, true)
94147
}
95148

96149
// ParseFromData func parses and validates the devfile integrity.
97150
// Creates devfile context and runtime objects
151+
// Deprecated, use ParseDevfile() instead
98152
func ParseFromData(data []byte) (d DevfileObj, err error) {
99153
d.Ctx = devfileCtx.DevfileCtx{}
100154
err = d.Ctx.SetDevfileContentFromBytes(data)
101155
if err != nil {
102156
return d, errors.Wrap(err, "failed to set devfile content from bytes")
103157
}
104-
err = d.Ctx.PopulateFromRaw()
105-
if err != nil {
106-
return d, err
107-
}
108-
109-
return parseDevfile(d, true)
158+
return populateAndParseDevfile(d, true)
110159
}
111160

112161
func parseParentAndPlugin(d DevfileObj) (err error) {
@@ -121,8 +170,13 @@ func parseParentAndPlugin(d DevfileObj) (err error) {
121170
if err != nil {
122171
return err
123172
}
173+
} else if parent.Id != "" {
174+
parentDevfileObj, err = parseFromRegistry(parent.Id, parent.RegistryUrl, d.Ctx)
175+
if err != nil {
176+
return err
177+
}
124178
} else {
125-
return fmt.Errorf("parent URI undefined, currently only URI is suppported")
179+
return fmt.Errorf("parent URI or parent Id undefined, currently only URI and Id are suppported")
126180
}
127181

128182
parentWorkspaceContent := parentDevfileObj.Data.GetDevfileWorkspace()
@@ -138,6 +192,7 @@ func parseParentAndPlugin(d DevfileObj) (err error) {
138192
klog.V(4).Infof("adding data of devfile with URI: %v", parent.Uri)
139193
}
140194
}
195+
141196
flattenedPlugins := []*v1.DevWorkspaceTemplateSpecContent{}
142197
components, err := d.Data.GetComponents(common.DevfileOptions{})
143198
if err != nil {
@@ -166,6 +221,7 @@ func parseParentAndPlugin(d DevfileObj) (err error) {
166221
flattenedPlugins = append(flattenedPlugins, flattenedPlugin)
167222
}
168223
}
224+
169225
mergedContent, err := apiOverride.MergeDevWorkspaceTemplateSpec(d.Data.GetDevfileWorkspace(), flattenedParent, flattenedPlugins...)
170226
if err != nil {
171227
return err
@@ -190,20 +246,11 @@ func parseFromURI(uri string, curDevfileCtx devfileCtx.DevfileCtx) (DevfileObj,
190246
// relative path on disk
191247
if !absoluteURL && curDevfileCtx.GetAbsPath() != "" {
192248
d.Ctx = devfileCtx.NewDevfileCtx(path.Join(path.Dir(curDevfileCtx.GetAbsPath()), uri))
193-
d.Ctx.SetURIMap(curDevfileCtx.GetURIMap())
194-
195-
// Fill the fields of DevfileCtx struct
196-
err = d.Ctx.Populate()
197-
if err != nil {
198-
return DevfileObj{}, err
199-
}
200-
return parseDevfile(d, true)
201-
}
202-
203-
// absolute URL address
204-
if absoluteURL {
249+
} else if absoluteURL {
250+
// absolute URL address
205251
d.Ctx = devfileCtx.NewURLDevfileCtx(uri)
206252
} else if curDevfileCtx.GetURL() != "" {
253+
// relative path to a URL
207254
u, err := url.Parse(curDevfileCtx.GetURL())
208255
if err != nil {
209256
return DevfileObj{}, err
@@ -212,11 +259,36 @@ func parseFromURI(uri string, curDevfileCtx devfileCtx.DevfileCtx) (DevfileObj,
212259
d.Ctx = devfileCtx.NewURLDevfileCtx(u.String())
213260
}
214261
d.Ctx.SetURIMap(curDevfileCtx.GetURIMap())
215-
// Fill the fields of DevfileCtx struct
216-
err = d.Ctx.PopulateFromURL()
217-
if err != nil {
218-
return DevfileObj{}, err
262+
return populateAndParseDevfile(d, true)
263+
}
264+
265+
func parseFromRegistry(parentId, registryURL string, curDevfileCtx devfileCtx.DevfileCtx) (DevfileObj, error) {
266+
if registryURL != "" {
267+
devfileContent, err := getDevfileFromRegistry(parentId, registryURL)
268+
if err != nil {
269+
return DevfileObj{}, err
270+
}
271+
return ParseDevfile(ParserArgs{Data: devfileContent, RegistryURLs: curDevfileCtx.GetRegistryURLs()})
272+
} else if curDevfileCtx.GetRegistryURLs() != nil {
273+
for _, registry := range curDevfileCtx.GetRegistryURLs() {
274+
devfileContent, err := getDevfileFromRegistry(parentId, registry)
275+
if devfileContent != nil && err == nil {
276+
return ParseDevfile(ParserArgs{Data: devfileContent, RegistryURLs: curDevfileCtx.GetRegistryURLs()})
277+
}
278+
}
279+
} else {
280+
return DevfileObj{}, fmt.Errorf("failed to fetch from registry, registry URL is not provided")
219281
}
220-
return parseDevfile(d, true)
221282

283+
return DevfileObj{}, fmt.Errorf("failed to get parent Id: %s from registry URLs provided", parentId)
284+
}
285+
286+
func getDevfileFromRegistry(parentId, registryURL string) ([]byte, error) {
287+
if !strings.HasPrefix(registryURL, "http://") && !strings.HasPrefix(registryURL, "https://") {
288+
registryURL = fmt.Sprintf("http://%s", registryURL)
289+
}
290+
param := util.HTTPRequestParams{
291+
URL: fmt.Sprintf("%s/devfiles/%s", registryURL, parentId),
292+
}
293+
return util.HTTPGetRequest(param, 0)
222294
}

‎pkg/devfile/parser/parse_test.go

+447-29
Large diffs are not rendered by default.

0 commit comments

Comments
 (0)
Please sign in to comment.