Skip to content

Commit 8ae84fb

Browse files
mmulhollammulholla
committed
add new tests shared with library
1 parent 283b0c5 commit 8ae84fb

8 files changed

+1031
-2
lines changed

‎.gitignore

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# Temporary Build Files
22
build/_output
33
build/_test
4-
test/v200/schemaTest/tmp
4+
test/**/tmp
55
test/go/pkg
66
# Created by https://www.gitignore.io/api/go,vim,emacs,visualstudiocode
77
### Emacs ###

‎test/README.md

+25-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# API Tests
1+
# schemsTest
22

33
The API tests are intended to provide a comprehensive verification of the devfile schemas. This includes:
44
- Ensuring every possible attribute is valid.
@@ -44,3 +44,27 @@ The test will read each of the test-xxxxxx.json files and run the tests defined
4444
1. Modify the copied tests as needed for the new version as decsribed above.
4545
1. Add `test/v201/schemaTest/tmp` to the .gitignore file.
4646
1. Run the test
47+
48+
49+
# apiTest
50+
51+
A new test approach, shared with the library repository for testing valid devfiles. Basically the test creates lots of valid devfiles whith different content. The attributes which are set and the values to which they are set are randomized. These tests are a work in progress and the intent is to eventually replace schemaTest.
52+
53+
## Test structure
54+
55+
- `test/v200/apiTest/api-test.go`: The go unit test program
56+
- `test/v200/utils/api/test-utils.go` : utilites, used by the test, which contain functions uniqiue to the api tests.
57+
- `test/v200/utils/common/*-utils.go` : utilites, used by the test, which are also used by the library tests. Mostly contain the code to generate valid devfile content.
58+
59+
60+
## Running tests
61+
62+
from the `test/v200/apiTest/` directory run
63+
- `go test -v`
64+
65+
* The test will generate a set of valid devfile.yaml files in `test/v200/apiTest/tmp/api-test/
66+
* The test will generate a log file: `test/v200/apiTest/tmp/test.log`
67+
* Each run of the test removes the `test/v200/apiTest/tmp` directory from the previous run.
68+
69+
70+

‎test/v200/apiTest/api_test.go

+80
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
package apiTest
2+
3+
import (
4+
"testing"
5+
6+
schema "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"
7+
api "github.com/devfile/api/v2/test/v200/utils/api"
8+
common "github.com/devfile/api/v2/test/v200/utils/common"
9+
)
10+
11+
func Test_ExecCommand(t *testing.T) {
12+
testContent := common.TestContent{}
13+
testContent.CommandTypes = []schema.CommandType{schema.ExecCommandType}
14+
testContent.EditContent = false
15+
testContent.FileName = common.GetDevFileName()
16+
api.RunTest(testContent, t)
17+
}
18+
19+
func Test_ApplyCommand(t *testing.T) {
20+
testContent := common.TestContent{}
21+
testContent.CommandTypes = []schema.CommandType{schema.ApplyCommandType}
22+
testContent.EditContent = false
23+
testContent.FileName = common.GetDevFileName()
24+
api.RunTest(testContent, t)
25+
}
26+
27+
func Test_CompositeCommand(t *testing.T) {
28+
testContent := common.TestContent{}
29+
testContent.CommandTypes = []schema.CommandType{schema.CompositeCommandType}
30+
testContent.EditContent = false
31+
testContent.FileName = common.GetDevFileName()
32+
api.RunTest(testContent, t)
33+
}
34+
35+
func Test_MultiCommand(t *testing.T) {
36+
testContent := common.TestContent{}
37+
testContent.CommandTypes = []schema.CommandType{schema.ExecCommandType,
38+
schema.CompositeCommandType,
39+
schema.ApplyCommandType}
40+
testContent.EditContent = true
41+
testContent.FileName = common.GetDevFileName()
42+
api.RunTest(testContent, t)
43+
}
44+
45+
func Test_ContainerComponent(t *testing.T) {
46+
testContent := common.TestContent{}
47+
testContent.ComponentTypes = []schema.ComponentType{schema.ContainerComponentType}
48+
testContent.EditContent = false
49+
testContent.FileName = common.GetDevFileName()
50+
api.RunTest(testContent, t)
51+
}
52+
53+
func Test_VolumeComponent(t *testing.T) {
54+
testContent := common.TestContent{}
55+
testContent.ComponentTypes = []schema.ComponentType{schema.VolumeComponentType}
56+
testContent.FileName = common.GetDevFileName()
57+
api.RunTest(testContent, t)
58+
}
59+
60+
func Test_MultiComponent(t *testing.T) {
61+
testContent := common.TestContent{}
62+
testContent.ComponentTypes = []schema.ComponentType{
63+
schema.ContainerComponentType,
64+
schema.VolumeComponentType}
65+
testContent.FileName = common.GetDevFileName()
66+
api.RunTest(testContent, t)
67+
}
68+
69+
func Test_Everything(t *testing.T) {
70+
testContent := common.TestContent{}
71+
testContent.CommandTypes = []schema.CommandType{
72+
schema.ExecCommandType,
73+
schema.CompositeCommandType,
74+
schema.ApplyCommandType}
75+
testContent.ComponentTypes = []schema.ComponentType{
76+
schema.ContainerComponentType,
77+
schema.VolumeComponentType}
78+
testContent.FileName = common.GetDevFileName()
79+
api.RunTest(testContent, t)
80+
}

‎test/v200/utils/api/test_utils.go

+173
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
package api
2+
3+
import (
4+
"bytes"
5+
"errors"
6+
"fmt"
7+
"io/ioutil"
8+
"strconv"
9+
"strings"
10+
"testing"
11+
12+
common "github.com/devfile/api/v2/test/v200/utils/common"
13+
"github.com/santhosh-tekuri/jsonschema"
14+
"sigs.k8s.io/yaml"
15+
)
16+
17+
const (
18+
// numDevfiles : the number of devfiles to create for each test
19+
numDevfiles = 5
20+
21+
schemaFileName = "../../../schemas/latest/ide-targeted/devfile.json"
22+
)
23+
24+
var schemas = make(map[string]SchemaFile)
25+
26+
type SchemaFile struct {
27+
Schema *jsonschema.Schema
28+
}
29+
30+
type DevfileValidator struct{}
31+
32+
// WriteAndVerify implements Saved.DevfileValidator interface.
33+
// writes to disk and validates the specified devfile
34+
func (devfileValidator DevfileValidator) WriteAndValidate(devfile *common.TestDevfile) error {
35+
err := writeDevfile(devfile)
36+
if err != nil {
37+
common.LogErrorMessage(fmt.Sprintf("Error writing file : %s : %v", devfile.FileName, err))
38+
} else {
39+
err = validateDevfile(devfile)
40+
if err != nil {
41+
common.LogErrorMessage(fmt.Sprintf("Error vaidating file : %s : %v", devfile.FileName, err))
42+
}
43+
}
44+
return err
45+
}
46+
47+
// CheckWithSchema checks the validity of aa devfile against the schema.
48+
func (schemaFile *SchemaFile) CheckWithSchema(devfile string, expectedMessage string) error {
49+
50+
// Read the created yaml file, ready for converison to json
51+
devfileData, err := ioutil.ReadFile(devfile)
52+
if err != nil {
53+
common.LogErrorMessage(fmt.Sprintf(" FAIL: schema : unable to read %s: %v", devfile, err))
54+
return err
55+
}
56+
57+
// Convert the yaml file to json
58+
devfileDataAsJSON, err := yaml.YAMLToJSON(devfileData)
59+
if err != nil {
60+
common.LogErrorMessage(fmt.Sprintf(" FAIL : %s : schema : failed to convert to json : %v", devfile, err))
61+
return err
62+
}
63+
64+
validationErr := schemaFile.Schema.Validate(bytes.NewReader(devfileDataAsJSON))
65+
if validationErr != nil {
66+
if len(expectedMessage) > 0 {
67+
if !strings.Contains(validationErr.Error(), expectedMessage) {
68+
err = errors.New(common.LogErrorMessage(fmt.Sprintf(" FAIL : schema : %s : Did not fail as expected : %s got : %v", devfile, expectedMessage, validationErr)))
69+
} else {
70+
common.LogInfoMessage(fmt.Sprintf("PASS: schema : Expected Error received : %s", expectedMessage))
71+
}
72+
} else {
73+
err = errors.New(common.LogErrorMessage(fmt.Sprintf(" FAIL : schema : %s : Did not pass as expected, got : %v", devfile, validationErr)))
74+
}
75+
} else {
76+
if len(expectedMessage) > 0 {
77+
err = errors.New(common.LogErrorMessage(fmt.Sprintf(" FAIL : schema : %s : was valid - Expected Error not found : %v", devfile, validationErr)))
78+
} else {
79+
common.LogInfoMessage(fmt.Sprintf(" PASS : schema : %s : devfile was valid.", devfile))
80+
}
81+
}
82+
return err
83+
}
84+
85+
// GetSchema downloads and saves a schema from the provided url
86+
func GetSchema(schemafile string) (SchemaFile, error) {
87+
88+
var err error
89+
schemaFile, found := schemas[schemafile]
90+
if !found {
91+
92+
schemaFile = SchemaFile{}
93+
94+
// Prepare the schema file
95+
compiler := jsonschema.NewCompiler()
96+
compiler.Draft = jsonschema.Draft7
97+
schemaFile.Schema, err = compiler.Compile(schemafile)
98+
if err != nil {
99+
//t.Fatalf(" FAIL : Schema compile failed : %s: %v", testJsonContent.SchemaFile, err)
100+
common.LogErrorMessage(fmt.Sprintf("FAIL : Failed to compile schema %v", err))
101+
} else {
102+
common.LogInfoMessage(fmt.Sprintf("Schema compiled from file: %s)", schemafile))
103+
schemas[schemafile] = schemaFile
104+
}
105+
}
106+
return schemaFile, err
107+
}
108+
109+
// WriteDevfile creates a devfile on disk for use in a test.
110+
func writeDevfile(devfile *common.TestDevfile) error {
111+
var err error
112+
113+
fileName := devfile.FileName
114+
if !strings.HasSuffix(fileName, ".yaml") {
115+
fileName += ".yaml"
116+
}
117+
118+
common.LogInfoMessage(fmt.Sprintf("Marshall and write devfile %s", devfile.FileName))
119+
120+
c, marshallErr := yaml.Marshal(&(devfile.SchemaDevFile))
121+
122+
if marshallErr != nil {
123+
err = errors.New(common.LogErrorMessage(fmt.Sprintf("Marshall devfile %s : %v", devfile.FileName, marshallErr)))
124+
} else {
125+
err = ioutil.WriteFile(fileName, c, 0644)
126+
if err != nil {
127+
common.LogErrorMessage(fmt.Sprintf("Write devfile %s : %v", devfile.FileName, err))
128+
}
129+
}
130+
return err
131+
}
132+
133+
// validateDevfile check the provided defile against the schema
134+
func validateDevfile(devfile *common.TestDevfile) error {
135+
136+
var err error
137+
var schemaFile SchemaFile
138+
139+
schemaFile, err = GetSchema(schemaFileName)
140+
if err != nil {
141+
common.LogErrorMessage(fmt.Sprintf("Failed to get devfile schema : %v", err))
142+
} else {
143+
err = schemaFile.CheckWithSchema(devfile.FileName, "")
144+
if err != nil {
145+
common.LogErrorMessage(fmt.Sprintf("Verification with devfile schema failed : %v", err))
146+
} else {
147+
common.LogInfoMessage(fmt.Sprintf("Devfile validated using JSONSchema schema : %s", devfile.FileName))
148+
}
149+
}
150+
151+
return err
152+
}
153+
154+
// RunTest : Runs a test to create and verify a devfile based on the content of the specified TestContent
155+
func RunTest(testContent common.TestContent, t *testing.T) {
156+
157+
common.LogMessage(fmt.Sprintf("Start test for %s", testContent.FileName))
158+
159+
validator := DevfileValidator{}
160+
161+
devfileName := testContent.FileName
162+
for i := 1; i <= numDevfiles; i++ {
163+
164+
testContent.FileName = common.AddSuffixToFileName(devfileName, strconv.Itoa(i))
165+
166+
testDevfile, err := common.GetDevfile(testContent.FileName, nil, validator)
167+
if err != nil {
168+
t.Fatalf(common.LogMessage(fmt.Sprintf("Error creating devfile : %v", err)))
169+
}
170+
171+
testDevfile.RunTest(testContent, t)
172+
}
173+
}

0 commit comments

Comments
 (0)