ict test command with a fuzzy match target proposal

Author: nikolay-komarevskiy

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/mattn/go-colorable v0.1.11 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/schollz/closestmatch v2.1.0+incompatible // indirect
 	github.com/spf13/cobra v1.6.1 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/testify v1.8.1 // indirect

go.sum

@@ -34,6 +34,8 @@ github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27k
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/schollz/closestmatch v2.1.0+incompatible h1:Uel2GXEpJqOWBrlyI+oY9LTiyyjYS17cCYRqP13/SHk=
+github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g=
 github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
 github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=

go_deps.bzl

@@ -421,6 +421,12 @@ def go_dependencies():
         sum = "h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=",
         version = "v2.1.0",
     )
+    go_repository(
+        name = "com_github_schollz_closestmatch",
+        importpath = "github.com/schollz/closestmatch",
+        sum = "h1:Uel2GXEpJqOWBrlyI+oY9LTiyyjYS17cCYRqP13/SHk=",
+        version = "v2.1.0+incompatible",
+    )
 
     go_repository(
         name = "com_github_sergi_go_diff",

rs/tests/ict/cmd/BUILD.bazel

@@ -3,13 +3,15 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 go_library(
     name = "cmd",
     srcs = [
+        "helpers.go",
         "root.go",
         "testCmd.go",
     ],
     importpath = "github.com/dfinity/ic/rs/tests/ict/cmd",
     visibility = ["//visibility:public"],
     deps = [
         "@com_github_fatih_color//:color",
+        "@com_github_schollz_closestmatch//:closestmatch",
         "@com_github_spf13_cobra//:cobra",
     ],
 )

rs/tests/ict/cmd/cmd_test.go

@@ -58,16 +58,3 @@ func Test_TestCmdWithHelpArg(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Contains(t, actual.String(), expected)
 }
-
-func Test_TestCmdWithTargetAndDryRunArgs(t *testing.T) {
-	expected := "bazel test my_target --config=systest --cache_test_results=yes"
-	actual := new(bytes.Buffer)
-	var command = cmd.NewTestCmd()
-	command.SetArgs([]string{"my_target", "-c", "--dry-run"})
-	command.SetOut(actual)
-
-	err := command.Execute()
-
-	assert.Nil(t, err)
-	assert.Contains(t, actual.String(), expected)
-}

rs/tests/ict/cmd/helpers.go

@@ -0,0 +1,66 @@
+package cmd
+
+import (
+	"bytes"
+	"fmt"
+	"os/exec"
+	"strings"
+
+	"github.com/schollz/closestmatch"
+)
+
+// see https://github.com/schollz/closestmatch
+var FUZZY_SEARCH_BAG_SIZES = []int{2, 3, 4}
+
+func Filter(vs []string, f func(string) bool) []string {
+	filtered := make([]string, 0)
+	for _, v := range vs {
+		if f(v) {
+			filtered = append(filtered, v)
+		}
+	}
+	return filtered
+}
+
+func get_all_system_test_targets() ([]string, error) {
+	command := []string{"bazel", "query", "tests(//rs/tests:*)"}
+	queryCmd := exec.Command(command[0], command[1:]...)
+	outputBuffer := &bytes.Buffer{}
+	stdErrBuffer := &bytes.Buffer{}
+	queryCmd.Stdout = outputBuffer
+	queryCmd.Stderr = stdErrBuffer
+	if err := queryCmd.Run(); err != nil {
+		return []string{}, fmt.Errorf("Bazel command: [%s] failed: %s", strings.Join(command, " "), stdErrBuffer.String())
+	}
+	cmdOutput := strings.Split(outputBuffer.String(), "\n")
+	all_targets := Filter(cmdOutput, func(s string) bool {
+		return len(s) > 0 && strings.Contains(s, "//rs/tests:")
+	})
+	return all_targets, nil
+}
+
+func get_closest_target_matches(target string) ([]string, error) {
+	all_targets, err := get_all_system_test_targets()
+	if err != nil {
+		return []string{}, err
+	}
+	closest_matches := closestmatch.New(all_targets, FUZZY_SEARCH_BAG_SIZES).ClosestN(target, FUZZY_MATCHES_COUNT)
+	return Filter(closest_matches, func(s string) bool {
+		return len(s) > 0
+	}), nil
+}
+
+func check_target_exists(target string) (bool, error) {
+	command := []string{"bazel", "query", target}
+	queryCmd := exec.Command(command[0], command[1:]...)
+	stdErrBuffer := &bytes.Buffer{}
+	queryCmd.Stderr = stdErrBuffer
+	if err := queryCmd.Run(); err != nil {
+		if strings.Contains(stdErrBuffer.String(), "no such target") {
+			return false, nil
+		} else {
+			return false, fmt.Errorf("Bazel command: [%s] failed: %s", strings.Join(command, " "), stdErrBuffer.String())
+		}
+	}
+	return true, nil
+}

rs/tests/ict/cmd/testCmd.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"fmt"
 	"os"
 	"os/exec"
 	"strings"
@@ -12,18 +13,33 @@ var RED = "\033[1;31m"
 var GREEN = "\033[1;32m"
 var CYAN = "\033[0;36m"
 var NC = "\033[0m"
+var FUZZY_MATCHES_COUNT = 7
 
 type Config struct {
-	isCacheTestResult bool
-	testTmpDir        string
-	isDryRun          bool
+	useCachedTestResult   bool
+	testTmpDir            string
+	isDryRun              bool
+	useFuzzyMatchedTarget bool
 }
 
 func TestCommandWithConfig(cfg *Config) func(cmd *cobra.Command, args []string) error {
 	return func(cmd *cobra.Command, args []string) error {
 		target := args[0]
+		if res, err_target := check_target_exists(target); !res {
+			if err_target != nil {
+				return err_target
+			} else if closest_matches, err_match := get_closest_target_matches(target); err_match != nil {
+				return err_match
+			} else if len(closest_matches) == 0 {
+				return fmt.Errorf("No test target `%s` was found", target)
+			} else if cfg.useFuzzyMatchedTarget {
+				target = closest_matches[0]
+			} else {
+				return fmt.Errorf("No test target `%s` was found: \nDid you mean any of:\n%s", target, strings.Join(closest_matches, "\n"))
+			}
+		}
 		cache_test_results := "--cache_test_results="
-		if cfg.isCacheTestResult {
+		if cfg.useCachedTestResult {
 			cache_test_results += "yes"
 		} else {
 			cache_test_results += "no"
@@ -49,15 +65,16 @@ func TestCommandWithConfig(cfg *Config) func(cmd *cobra.Command, args []string)
 func NewTestCmd() *cobra.Command {
 	var cfg = Config{}
 	var testCmd = &cobra.Command{
-		Use:     "test",
+		Use:     "test <system_test_target_arg>",
 		Aliases: []string{"system_test", "t"},
 		Short:   "Run system_test target with Bazel",
 		Example: "ict test //rs/tests:basic_health_test",
 		Args:    cobra.ExactArgs(1),
 		RunE:    TestCommandWithConfig(&cfg),
 	}
+	testCmd.Flags().BoolVarP(&cfg.useFuzzyMatchedTarget, "use-fuzzy-match", "f", false, "If test target is not found, use the closest fuzzy matched one.")
 	testCmd.Flags().BoolVarP(&cfg.isDryRun, "dry-run", "n", false, "Print raw Bazel command to be invoked.")
-	testCmd.Flags().BoolVarP(&cfg.isCacheTestResult, "cache_test_results", "c", false, "Bazel's cache_test_results, see --cache_test_results tag in Bazel docs.")
+	testCmd.Flags().BoolVarP(&cfg.useCachedTestResult, "cache_test_results", "c", false, "Bazel's cache_test_results, see --cache_test_results tag in Bazel docs.")
 	testCmd.PersistentFlags().StringVarP(&cfg.testTmpDir, "test_tmpdir", "t", "", "Dir for storing test results, see --test-tmpdir tag in Bazel docs.")
 	return testCmd
 }