parse parent devfile from devfile registry

devfile.yaml

@@ -5,7 +5,9 @@ metadata:
   attributes:
     alpha.build-dockerfile: /relative/path/to/Dockerfile
 parent:
-  uri: https://raw.githubusercontent.com/odo-devfiles/registry/master/devfiles/nodejs/devfile.yaml
+  # uri: https://raw.githubusercontent.com/odo-devfiles/registry/master/devfiles/nodejs/devfile.yaml
+  id: nodejs
+  registryUrl: "https://registry.devfile.io"
   commands:
   - id: install
     exec:

main.go

@@ -20,28 +20,27 @@ func main() {
 	}
 }
 
-//ParseDevfile to parse devfile from library
-func ParseDevfile(devfileLocation string) (parser.DevfileObj, error) {
-
-	devfile, err := devfilepkg.ParseAndValidate(devfileLocation)
-	return devfile, err
-}
-
 func parserTest() {
-	var devfile parser.DevfileObj
-	var err error
+	var args parser.ParserArgs
 	if len(os.Args) > 1 {
 		if strings.HasPrefix(os.Args[1], "http") {
-			devfile, err = devfilepkg.ParseFromURLAndValidate(os.Args[1])
+			args = parser.ParserArgs{
+				URL: os.Args[1],
+			}
 		} else {
-			devfile, err = ParseDevfile(os.Args[1])
+			args = parser.ParserArgs{
+				Path: os.Args[1],
+			}
 		}
 		fmt.Println("parsing devfile from " + os.Args[1])
 
 	} else {
-		devfile, err = ParseDevfile("devfile.yaml")
-		fmt.Println("parsing devfile from " + devfile.Ctx.GetAbsPath())
+		args = parser.ParserArgs{
+			Path: "devfile.yaml",
+		}
+		fmt.Println("parsing devfile from ./devfile.yaml")
 	}
+	devfile, err := devfilepkg.ParseDevfileAndValidate(args)
 	if err != nil {
 		fmt.Println(err)
 	} else {

pkg/devfile/parse.go

@@ -9,6 +9,7 @@ import (
 // and validates the devfile integrity with the schema
 // and validates the devfile data.
 // Creates devfile context and runtime objects.
+// Deprecated, use ParseDevfileAndValidate() instead
 func ParseFromURLAndValidate(url string) (d parser.DevfileObj, err error) {
 
 	// read and parse devfile from the given URL
@@ -30,6 +31,7 @@ func ParseFromURLAndValidate(url string) (d parser.DevfileObj, err error) {
 // and validates the devfile integrity with the schema
 // and validates the devfile data.
 // Creates devfile context and runtime objects.
+// Deprecated, use ParseDevfileAndValidate() instead
 func ParseFromDataAndValidate(data []byte) (d parser.DevfileObj, err error) {
 	// read and parse devfile from the given bytes
 	d, err = parser.ParseFromData(data)
@@ -49,6 +51,7 @@ func ParseFromDataAndValidate(data []byte) (d parser.DevfileObj, err error) {
 // and validates the devfile integrity with the schema
 // and validates the devfile data.
 // Creates devfile context and runtime objects.
+// Deprecated, use ParseDevfileAndValidate() instead
 func ParseAndValidate(path string) (d parser.DevfileObj, err error) {
 
 	// read and parse devfile from given path
@@ -65,3 +68,22 @@ func ParseAndValidate(path string) (d parser.DevfileObj, err error) {
 
 	return d, err
 }
+
+// ParseDevfileAndValidate func parses the devfile data
+// and validates the devfile integrity with the schema
+// and validates the devfile data.
+// Creates devfile context and runtime objects.
+func ParseDevfileAndValidate(args parser.ParserArgs) (d parser.DevfileObj, err error) {
+	d, err = parser.ParseDevfile(args)
+	if err != nil {
+		return d, err
+	}
+
+	// generic validation on devfile content
+	err = validate.ValidateDevfileData(d.Data)
+	if err != nil {
+		return d, err
+	}
+
+	return d, err
+}

pkg/devfile/parser/context/context.go

@@ -35,6 +35,9 @@ type DevfileCtx struct {
 
 	// trace of all url referenced
 	uriMap map[string]bool
+
+	// registry URLs list
+	registryURLs []string
 }
 
 // NewDevfileCtx returns a new DevfileCtx type object
@@ -149,3 +152,13 @@ func (d *DevfileCtx) GetURIMap() map[string]bool {
 func (d *DevfileCtx) SetURIMap(uriMap map[string]bool) {
 	d.uriMap = uriMap
 }
+
+// GetRegistryURLs func returns current devfile registry URLs
+func (d *DevfileCtx) GetRegistryURLs() []string {
+	return d.registryURLs
+}
+
+// SetRegistryURLs set registry URLs in the devfile ctx
+func (d *DevfileCtx) SetRegistryURLs(registryURLs []string) {
+	d.registryURLs = registryURLs
+}

pkg/devfile/parser/parse.go

@@ -3,6 +3,7 @@ package parser
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/devfile/library/pkg/util"
 	"net/url"
 	"path"
 	"strings"
@@ -53,60 +54,108 @@ func parseDevfile(d DevfileObj, flattenedDevfile bool) (DevfileObj, error) {
 	return d, nil
 }
 
-// Parse func populates the flattened devfile data, parses and validates the devfile integrity.
+// ParserArgs is the struct to pass into parser functions which contains required info for parsing devfile.
+// It accepts devfile path, devfile URL or devfile content in []byte format.
+type ParserArgs struct {
+	// Path is a relative or absolute devfile path.
+	Path string
+	// URL is the URL address of the specific devfile.
+	URL string
+	// Data is the devfile content in []byte format.
+	Data []byte
+	// FlattenedDevfile defines if the returned devfileObj is flattened content (true) or raw content (false).
+	// The value is default to be true.
+	FlattenedDevfile *bool
+	// RegistryURLs is a list of registry hosts which parser should pull parent devfile from.
+	// If registryUrl is defined in devfile, this list will be ignored.
+	RegistryURLs []string
+}
+
+// ParseDevfile func populates the devfile data, parses and validates the devfile integrity.
 // Creates devfile context and runtime objects
-func Parse(path string) (d DevfileObj, err error) {
+func ParseDevfile(args ParserArgs) (d DevfileObj, err error) {
+	if args.Data != nil {
+		d.Ctx = devfileCtx.DevfileCtx{}
+		err = d.Ctx.SetDevfileContentFromBytes(args.Data)
+		if err != nil {
+			return d, errors.Wrap(err, "failed to set devfile content from bytes")
+		}
+	} else if args.Path != "" {
+		d.Ctx = devfileCtx.NewDevfileCtx(args.Path)
+	} else if args.URL != "" {
+		d.Ctx = devfileCtx.NewURLDevfileCtx(args.URL)
+	} else {
+		return d, errors.Wrap(err, "the devfile source is not provided")
+	}
 
-	// NewDevfileCtx
-	d.Ctx = devfileCtx.NewDevfileCtx(path)
+	if args.RegistryURLs != nil {
+		d.Ctx.SetRegistryURLs(args.RegistryURLs)
+	}
+
+	flattenedDevfile := true
+	if args.FlattenedDevfile != nil {
+		flattenedDevfile = *args.FlattenedDevfile
+	}
+
+	return populateAndParseDevfile(d, flattenedDevfile)
+}
+
+func populateAndParseDevfile(d DevfileObj, flattenedDevfile bool) (DevfileObj, error) {
+	var err error
 
 	// Fill the fields of DevfileCtx struct
-	err = d.Ctx.Populate()
+	if d.Ctx.GetURL() != "" {
+		err = d.Ctx.PopulateFromURL()
+	} else if d.Ctx.GetDevfileContent() != nil {
+		err = d.Ctx.PopulateFromRaw()
+	} else {
+		err = d.Ctx.Populate()
+	}
 	if err != nil {
 		return d, err
 	}
-	return parseDevfile(d, true)
+
+	return parseDevfile(d, flattenedDevfile)
+}
+
+// Parse func populates the flattened devfile data, parses and validates the devfile integrity.
+// Creates devfile context and runtime objects
+// Deprecated, use ParseDevfile() instead
+func Parse(path string) (d DevfileObj, err error) {
+
+	// NewDevfileCtx
+	d.Ctx = devfileCtx.NewDevfileCtx(path)
+
+	return populateAndParseDevfile(d, true)
 }
 
 // ParseRawDevfile populates the raw devfile data without overriding and merging
+// Deprecated, use ParseDevfile() instead
 func ParseRawDevfile(path string) (d DevfileObj, err error) {
 	// NewDevfileCtx
 	d.Ctx = devfileCtx.NewDevfileCtx(path)
 
-	// Fill the fields of DevfileCtx struct
-	err = d.Ctx.Populate()
-	if err != nil {
-		return d, err
-	}
-	return parseDevfile(d, false)
+	return populateAndParseDevfile(d, false)
 }
 
 // ParseFromURL func parses and validates the devfile integrity.
 // Creates devfile context and runtime objects
+// Deprecated, use ParseDevfile() instead
 func ParseFromURL(url string) (d DevfileObj, err error) {
 	d.Ctx = devfileCtx.NewURLDevfileCtx(url)
-	// Fill the fields of DevfileCtx struct
-	err = d.Ctx.PopulateFromURL()
-	if err != nil {
-		return d, err
-	}
-	return parseDevfile(d, true)
+	return populateAndParseDevfile(d, true)
 }
 
 // ParseFromData func parses and validates the devfile integrity.
 // Creates devfile context and runtime objects
+// Deprecated, use ParseDevfile() instead
 func ParseFromData(data []byte) (d DevfileObj, err error) {
 	d.Ctx = devfileCtx.DevfileCtx{}
 	err = d.Ctx.SetDevfileContentFromBytes(data)
 	if err != nil {
 		return d, errors.Wrap(err, "failed to set devfile content from bytes")
 	}
-	err = d.Ctx.PopulateFromRaw()
-	if err != nil {
-		return d, err
-	}
-
-	return parseDevfile(d, true)
+	return populateAndParseDevfile(d, true)
 }
 
 func parseParentAndPlugin(d DevfileObj) (err error) {
@@ -121,8 +170,13 @@ func parseParentAndPlugin(d DevfileObj) (err error) {
 				if err != nil {
 					return err
 				}
+			} else if parent.Id != "" {
+				parentDevfileObj, err = parseFromRegistry(parent.Id, parent.RegistryUrl, d.Ctx)
+				if err != nil {
+					return err
+				}
 			} else {
-				return fmt.Errorf("parent URI undefined, currently only URI is suppported")
+				return fmt.Errorf("parent URI or parent Id undefined, currently only URI and Id are suppported")
 			}
 
 			parentWorkspaceContent := parentDevfileObj.Data.GetDevfileWorkspace()
@@ -138,6 +192,7 @@ func parseParentAndPlugin(d DevfileObj) (err error) {
 			klog.V(4).Infof("adding data of devfile with URI: %v", parent.Uri)
 		}
 	}
+
 	flattenedPlugins := []*v1.DevWorkspaceTemplateSpecContent{}
 	components, err := d.Data.GetComponents(common.DevfileOptions{})
 	if err != nil {
@@ -166,6 +221,7 @@ func parseParentAndPlugin(d DevfileObj) (err error) {
 			flattenedPlugins = append(flattenedPlugins, flattenedPlugin)
 		}
 	}
+
 	mergedContent, err := apiOverride.MergeDevWorkspaceTemplateSpec(d.Data.GetDevfileWorkspace(), flattenedParent, flattenedPlugins...)
 	if err != nil {
 		return err
@@ -190,20 +246,11 @@ func parseFromURI(uri string, curDevfileCtx devfileCtx.DevfileCtx) (DevfileObj,
 	// relative path on disk
 	if !absoluteURL && curDevfileCtx.GetAbsPath() != "" {
 		d.Ctx = devfileCtx.NewDevfileCtx(path.Join(path.Dir(curDevfileCtx.GetAbsPath()), uri))
-		d.Ctx.SetURIMap(curDevfileCtx.GetURIMap())
-
-		// Fill the fields of DevfileCtx struct
-		err = d.Ctx.Populate()
-		if err != nil {
-			return DevfileObj{}, err
-		}
-		return parseDevfile(d, true)
-	}
-
-	// absolute URL address
-	if absoluteURL {
+	} else if absoluteURL {
+		// absolute URL address
 		d.Ctx = devfileCtx.NewURLDevfileCtx(uri)
 	} else if curDevfileCtx.GetURL() != "" {
+		// relative path to a URL
 		u, err := url.Parse(curDevfileCtx.GetURL())
 		if err != nil {
 			return DevfileObj{}, err
@@ -212,11 +259,36 @@ func parseFromURI(uri string, curDevfileCtx devfileCtx.DevfileCtx) (DevfileObj,
 		d.Ctx = devfileCtx.NewURLDevfileCtx(u.String())
 	}
 	d.Ctx.SetURIMap(curDevfileCtx.GetURIMap())
-	// Fill the fields of DevfileCtx struct
-	err = d.Ctx.PopulateFromURL()
-	if err != nil {
-		return DevfileObj{}, err
+	return populateAndParseDevfile(d, true)
+}
+
+func parseFromRegistry(parentId, registryURL string, curDevfileCtx devfileCtx.DevfileCtx) (DevfileObj, error) {
+	if registryURL != "" {
+		devfileContent, err := getDevfileFromRegistry(parentId, registryURL)
+		if err != nil {
+			return DevfileObj{}, err
+		}
+		return ParseDevfile(ParserArgs{Data: devfileContent, RegistryURLs: curDevfileCtx.GetRegistryURLs()})
+	} else if curDevfileCtx.GetRegistryURLs() != nil {
+		for _, registry := range curDevfileCtx.GetRegistryURLs() {
+			devfileContent, err := getDevfileFromRegistry(parentId, registry)
+			if devfileContent != nil && err == nil {
+				return ParseDevfile(ParserArgs{Data: devfileContent, RegistryURLs: curDevfileCtx.GetRegistryURLs()})
+			}
+		}
+	} else {
+		return DevfileObj{}, fmt.Errorf("failed to fetch from registry, registry URL is not provided")
 	}
-	return parseDevfile(d, true)
 
+	return DevfileObj{}, fmt.Errorf("failed to get parent Id: %s from registry URLs provided", parentId)
+}
+
+func getDevfileFromRegistry(parentId, registryURL string) ([]byte, error) {
+	if !strings.HasPrefix(registryURL, "http://") && !strings.HasPrefix(registryURL, "https://") {
+		registryURL = fmt.Sprintf("http://%s", registryURL)
+	}
+	param := util.HTTPRequestParams{
+		URL: fmt.Sprintf("%s/devfiles/%s", registryURL, parentId),
+	}
+	return util.HTTPGetRequest(param, 0)
 }