Skip to content

Commit 81859ea

Browse files
mmulhollammulholla
authored
First stage of new tests which share logic with library tests (#348)
* add new tests shared with library * Code review comments part 2
1 parent 585f5fe commit 81859ea

8 files changed

+1028
-2
lines changed

‎.gitignore

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
# Temporary Build Files
22
build/_output
33
build/_test
4-
test/v200/schemaTest/tmp
4+
test/**/tmp
55
test/go/pkg
66
# Created by https://www.gitignore.io/api/go,vim,emacs,visualstudiocode
77
### Emacs ###

‎test/README.md

+25-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# API Tests
1+
# schemaTest
22

33
The API tests are intended to provide a comprehensive verification of the devfile schemas. This includes:
44
- Ensuring every possible attribute is valid.
@@ -44,3 +44,27 @@ The test will read each of the test-xxxxxx.json files and run the tests defined
4444
1. Modify the copied tests as needed for the new version as decsribed above.
4545
1. Add `test/v201/schemaTest/tmp` to the .gitignore file.
4646
1. Run the test
47+
48+
49+
# apiTest
50+
51+
A new test approach, shared with the library repository for testing valid devfiles. Basically the test creates lots of valid devfiles whith different content. The attributes which are set and the values to which they are set are randomized. These tests are a work in progress and the intent is to eventually replace schemaTest.
52+
53+
## Test structure
54+
55+
- `test/v200/apiTest/api-test.go`: The go unit test program
56+
- `test/v200/utils/api/test-utils.go` : utilites, used by the test, which contain functions uniqiue to the api tests.
57+
- `test/v200/utils/common/*-utils.go` : utilites, used by the test, which are also used by the library tests. Mostly contain the code to generate valid devfile content.
58+
59+
60+
## Running tests
61+
62+
from the `test/v200/apiTest/` directory run
63+
- `go test -v`
64+
65+
* The test will generate a set of valid devfile.yaml files in `test/v200/apiTest/tmp/api-test/
66+
* The test will generate a log file: `test/v200/apiTest/tmp/test.log`
67+
* Each run of the test removes the `test/v200/apiTest/tmp` directory from the previous run.
68+
69+
70+

‎test/v200/apiTest/api_test.go

+80
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
package apiTest
2+
3+
import (
4+
"testing"
5+
6+
schema "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"
7+
apiUtils "github.com/devfile/api/v2/test/v200/utils/api"
8+
commonUtils "github.com/devfile/api/v2/test/v200/utils/common"
9+
)
10+
11+
func Test_ExecCommand(t *testing.T) {
12+
testContent := commonUtils.TestContent{}
13+
testContent.CommandTypes = []schema.CommandType{schema.ExecCommandType}
14+
testContent.EditContent = false
15+
testContent.FileName = commonUtils.GetDevFileName()
16+
apiUtils.RunTest(testContent, t)
17+
}
18+
19+
func Test_ApplyCommand(t *testing.T) {
20+
testContent := commonUtils.TestContent{}
21+
testContent.CommandTypes = []schema.CommandType{schema.ApplyCommandType}
22+
testContent.EditContent = false
23+
testContent.FileName = commonUtils.GetDevFileName()
24+
apiUtils.RunTest(testContent, t)
25+
}
26+
27+
func Test_CompositeCommand(t *testing.T) {
28+
testContent := commonUtils.TestContent{}
29+
testContent.CommandTypes = []schema.CommandType{schema.CompositeCommandType}
30+
testContent.EditContent = false
31+
testContent.FileName = commonUtils.GetDevFileName()
32+
apiUtils.RunTest(testContent, t)
33+
}
34+
35+
func Test_MultiCommand(t *testing.T) {
36+
testContent := commonUtils.TestContent{}
37+
testContent.CommandTypes = []schema.CommandType{schema.ExecCommandType,
38+
schema.CompositeCommandType,
39+
schema.ApplyCommandType}
40+
testContent.EditContent = true
41+
testContent.FileName = commonUtils.GetDevFileName()
42+
apiUtils.RunTest(testContent, t)
43+
}
44+
45+
func Test_ContainerComponent(t *testing.T) {
46+
testContent := commonUtils.TestContent{}
47+
testContent.ComponentTypes = []schema.ComponentType{schema.ContainerComponentType}
48+
testContent.EditContent = false
49+
testContent.FileName = commonUtils.GetDevFileName()
50+
apiUtils.RunTest(testContent, t)
51+
}
52+
53+
func Test_VolumeComponent(t *testing.T) {
54+
testContent := commonUtils.TestContent{}
55+
testContent.ComponentTypes = []schema.ComponentType{schema.VolumeComponentType}
56+
testContent.FileName = commonUtils.GetDevFileName()
57+
apiUtils.RunTest(testContent, t)
58+
}
59+
60+
func Test_MultiComponent(t *testing.T) {
61+
testContent := commonUtils.TestContent{}
62+
testContent.ComponentTypes = []schema.ComponentType{
63+
schema.ContainerComponentType,
64+
schema.VolumeComponentType}
65+
testContent.FileName = commonUtils.GetDevFileName()
66+
apiUtils.RunTest(testContent, t)
67+
}
68+
69+
func Test_Everything(t *testing.T) {
70+
testContent := commonUtils.TestContent{}
71+
testContent.CommandTypes = []schema.CommandType{
72+
schema.ExecCommandType,
73+
schema.CompositeCommandType,
74+
schema.ApplyCommandType}
75+
testContent.ComponentTypes = []schema.ComponentType{
76+
schema.ContainerComponentType,
77+
schema.VolumeComponentType}
78+
testContent.FileName = commonUtils.GetDevFileName()
79+
apiUtils.RunTest(testContent, t)
80+
}

‎test/v200/utils/api/test_utils.go

+175
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
package api
2+
3+
import (
4+
"bytes"
5+
"errors"
6+
"fmt"
7+
"io/ioutil"
8+
"strconv"
9+
"strings"
10+
"testing"
11+
12+
commonUtils "github.com/devfile/api/v2/test/v200/utils/common"
13+
"github.com/santhosh-tekuri/jsonschema"
14+
"sigs.k8s.io/yaml"
15+
)
16+
17+
const (
18+
// numDevfiles : the number of devfiles to create for each test
19+
numDevfiles = 5
20+
21+
schemaFileName = "../../../schemas/latest/ide-targeted/devfile.json"
22+
)
23+
24+
var schemas = make(map[string]SchemaFile)
25+
26+
// SchemaFile - represents the schema stucture
27+
type SchemaFile struct {
28+
Schema *jsonschema.Schema
29+
}
30+
31+
// DevfileValidator struct for DevfileValidator interface defined in common utils.
32+
type DevfileValidator struct{}
33+
34+
// WriteAndValidate implements Saved.DevfileValidator interface.
35+
// writes to disk and validates the specified devfile
36+
func (devfileValidator DevfileValidator) WriteAndValidate(devfile *commonUtils.TestDevfile) error {
37+
err := writeDevfile(devfile)
38+
if err != nil {
39+
commonUtils.LogErrorMessage(fmt.Sprintf("Error writing file : %s : %v", devfile.FileName, err))
40+
} else {
41+
err = validateDevfile(devfile)
42+
if err != nil {
43+
commonUtils.LogErrorMessage(fmt.Sprintf("Error vaidating file : %s : %v", devfile.FileName, err))
44+
}
45+
}
46+
return err
47+
}
48+
49+
// checkWithSchema checks the validity of a devfile against the schema.
50+
func (schemaFile *SchemaFile) checkWithSchema(devfile string, expectedMessage string) error {
51+
52+
// Read the created yaml file, ready for converison to json
53+
devfileData, err := ioutil.ReadFile(devfile)
54+
if err != nil {
55+
commonUtils.LogErrorMessage(fmt.Sprintf(" FAIL: schema : unable to read %s: %v", devfile, err))
56+
return err
57+
}
58+
59+
// Convert the yaml file to json
60+
devfileDataAsJSON, err := yaml.YAMLToJSON(devfileData)
61+
if err != nil {
62+
commonUtils.LogErrorMessage(fmt.Sprintf(" FAIL : %s : schema : failed to convert to json : %v", devfile, err))
63+
return err
64+
}
65+
66+
validationErr := schemaFile.Schema.Validate(bytes.NewReader(devfileDataAsJSON))
67+
if validationErr != nil {
68+
if len(expectedMessage) > 0 {
69+
if !strings.Contains(validationErr.Error(), expectedMessage) {
70+
err = errors.New(commonUtils.LogErrorMessage(fmt.Sprintf(" FAIL : schema : %s : Did not fail as expected : %s got : %v", devfile, expectedMessage, validationErr)))
71+
} else {
72+
commonUtils.LogInfoMessage(fmt.Sprintf("PASS: schema : Expected Error received : %s", expectedMessage))
73+
}
74+
} else {
75+
err = errors.New(commonUtils.LogErrorMessage(fmt.Sprintf(" FAIL : schema : %s : Did not pass as expected, got : %v", devfile, validationErr)))
76+
}
77+
} else {
78+
if len(expectedMessage) > 0 {
79+
err = errors.New(commonUtils.LogErrorMessage(fmt.Sprintf(" FAIL : schema : %s : was valid - Expected Error not found : %v", devfile, validationErr)))
80+
} else {
81+
commonUtils.LogInfoMessage(fmt.Sprintf(" PASS : schema : %s : devfile was valid.", devfile))
82+
}
83+
}
84+
return err
85+
}
86+
87+
// getSchema downloads and saves a schema from the provided url
88+
func getSchema(schemaFileName string) (SchemaFile, error) {
89+
90+
var err error
91+
schemaFile, found := schemas[schemaFileName]
92+
if !found {
93+
94+
schemaFile = SchemaFile{}
95+
96+
// Prepare the schema file
97+
compiler := jsonschema.NewCompiler()
98+
// Use Draft 7, github.com/santhosh-tekuri/jsonschema provides 4,6 an 7 so use the latest
99+
compiler.Draft = jsonschema.Draft7
100+
schemaFile.Schema, err = compiler.Compile(schemaFileName)
101+
if err != nil {
102+
commonUtils.LogErrorMessage(fmt.Sprintf("FAIL : Failed to compile schema %v", err))
103+
} else {
104+
commonUtils.LogInfoMessage(fmt.Sprintf("Schema compiled from file: %s)", schemaFileName))
105+
schemas[schemaFileName] = schemaFile
106+
}
107+
}
108+
return schemaFile, err
109+
}
110+
111+
// writeDevfile creates a devfile on disk for use in a test.
112+
func writeDevfile(devfile *commonUtils.TestDevfile) error {
113+
var err error
114+
115+
fileName := devfile.FileName
116+
if !strings.HasSuffix(fileName, ".yaml") {
117+
fileName += ".yaml"
118+
}
119+
120+
commonUtils.LogInfoMessage(fmt.Sprintf("Marshall and write devfile %s", devfile.FileName))
121+
122+
c, marshallErr := yaml.Marshal(&(devfile.SchemaDevFile))
123+
124+
if marshallErr != nil {
125+
err = errors.New(commonUtils.LogErrorMessage(fmt.Sprintf("Marshall devfile %s : %v", devfile.FileName, marshallErr)))
126+
} else {
127+
err = ioutil.WriteFile(fileName, c, 0644)
128+
if err != nil {
129+
commonUtils.LogErrorMessage(fmt.Sprintf("Write devfile %s : %v", devfile.FileName, err))
130+
}
131+
}
132+
return err
133+
}
134+
135+
// validateDevfile check the provided defile against the schema
136+
func validateDevfile(devfile *commonUtils.TestDevfile) error {
137+
138+
var err error
139+
var schemaFile SchemaFile
140+
141+
schemaFile, err = getSchema(schemaFileName)
142+
if err != nil {
143+
commonUtils.LogErrorMessage(fmt.Sprintf("Failed to get devfile schema : %v", err))
144+
} else {
145+
err = schemaFile.checkWithSchema(devfile.FileName, "")
146+
if err != nil {
147+
commonUtils.LogErrorMessage(fmt.Sprintf("Verification with devfile schema failed : %v", err))
148+
} else {
149+
commonUtils.LogInfoMessage(fmt.Sprintf("Devfile validated using JSONSchema schema : %s", devfile.FileName))
150+
}
151+
}
152+
153+
return err
154+
}
155+
156+
// RunTest : Runs a test to create and verify a devfile based on the content of the specified TestContent
157+
func RunTest(testContent commonUtils.TestContent, t *testing.T) {
158+
159+
commonUtils.LogMessage(fmt.Sprintf("Start test for %s", testContent.FileName))
160+
161+
validator := DevfileValidator{}
162+
163+
devfileName := testContent.FileName
164+
for i := 1; i <= numDevfiles; i++ {
165+
166+
testContent.FileName = commonUtils.AddSuffixToFileName(devfileName, strconv.Itoa(i))
167+
168+
testDevfile, err := commonUtils.GetDevfile(testContent.FileName, nil, validator)
169+
if err != nil {
170+
t.Fatalf(commonUtils.LogMessage(fmt.Sprintf("Error creating devfile : %v", err)))
171+
}
172+
173+
testDevfile.RunTest(testContent, t)
174+
}
175+
}

0 commit comments

Comments
 (0)