fmtsort: sort interfaces deterministically

Previously, the result of sorting a map[interface{}] containing
multiple concrete types was non-deterministic. To ensure consistent
results, sort first by the machine address of the value's type, then
by the value itself.

Sorting based on machine address results in unpredictable output, but
will at least be deterministic across runs of the same program.

Fixes golang#30398

Author: lukechampine

src/internal/fmtsort/sort.go

@@ -167,7 +167,7 @@ func compare(aVal, bVal reflect.Value) int {
 		if c, ok := nilCompare(aVal, bVal); ok {
 			return c
 		}
-		c := compare(reflect.ValueOf(aType), reflect.ValueOf(bType))
+		c := compare(reflect.ValueOf(aVal.Elem().Type()), reflect.ValueOf(bVal.Elem().Type()))
 		if c != 0 {
 			return c
 		}

src/internal/fmtsort/sort_test.go

@@ -126,10 +126,6 @@ var sortTests = []sortTest{
 		map[[2]int]string{{7, 2}: "72", {7, 1}: "71", {3, 4}: "34"},
 		"[3 4]:34 [7 1]:71 [7 2]:72",
 	},
-	{
-		map[interface{}]string{7: "7", 4: "4", 3: "3", nil: "nil"},
-		"<nil>:nil 3:3 4:4 7:7",
-	},
 }
 
 func sprint(data interface{}) string {
@@ -210,3 +206,41 @@ func TestOrder(t *testing.T) {
 		}
 	}
 }
+
+func TestInterface(t *testing.T) {
+	// A map containing multiple concrete types should be sorted by type,
+	// then value. However, the relative ordering of types is unspecified,
+	// so test this by checking the presence of sorted subgroups.
+	m := map[interface{}]string{
+		[2]int{1, 0}:             "",
+		[2]int{0, 1}:             "",
+		true:                     "",
+		false:                    "",
+		3.1:                      "",
+		2.1:                      "",
+		1.1:                      "",
+		math.NaN():               "",
+		3:                        "",
+		2:                        "",
+		1:                        "",
+		"c":                      "",
+		"b":                      "",
+		"a":                      "",
+		struct{ x, y int }{1, 0}: "",
+		struct{ x, y int }{0, 1}: "",
+	}
+	got := sprint(m)
+	typeGroups := []string{
+		"NaN: 1.1: 2.1: 3.1:", // float64
+		"false: true:",        // bool
+		"1: 2: 3:",            // int
+		"a: b: c:",            // string
+		"[0 1]: [1 0]:",       // [2]int
+		"{0 1}: {1 0}:",       // struct{ x int; y int }
+	}
+	for _, g := range typeGroups {
+		if !strings.Contains(got, g) {
+			t.Errorf("sorted map should contain %q", g)
+		}
+	}
+}