improve handling of TypeConstructor Types in the generic function cache. fixes #11840

when TypeConstructors showed up as parameters of Type objects in the generic function cache,
the type intersection result would trick inference into caching it in the wrong place
and using it when it didn't apply. fix that by checking explicitly for arguments declared
with ::TypeConstructor and ensuring they are matched and cached correctly

also fix a similar issue for inserting/matching Type{T} in the gf cache

Author: vtjnash

src/gf.c

@@ -78,9 +78,7 @@ static int cache_match_by_type(jl_value_t **types, size_t n, jl_tupletype_t *sig
             }
         }
         jl_value_t *a = types[i];
-        if (jl_is_datatype(a) && jl_is_datatype(decl) &&
-                 ((jl_datatype_t*)decl)->name == jl_type_type->name &&
-                 ((jl_datatype_t*)a   )->name == jl_type_type->name) {
+        if (jl_is_type_type(a) && jl_is_type_type(decl)) {
             jl_value_t *tp0 = jl_tparam0(decl);
             if (tp0 == (jl_value_t*)jl_typetype_tvar) {
                 // in the case of Type{T}, the types don't have
@@ -522,6 +520,22 @@ static jl_function_t *cache_method(jl_methtable_t *mt, jl_tupletype_t *type,
         }
         if (set_to_any || isstaged) {
         }
+        else if (jl_is_type_type(elt) && jl_is_typector(jl_tparam0(elt)) &&
+                 decl_i == (jl_value_t*)jl_typector_type) {
+            // TypeConstructors are problematic because they can be alternate
+            // representations of any type. If we matched this method because
+            // it matched the leaf type TypeConstructor, then don't
+            // cache something different since that doesn't necessarily actually apply
+            jl_svecset(newparams, i, jl_typector_type);
+        }
+        else if (jl_is_type_type(elt) && decl_i == (jl_value_t*)jl_datatype_type) {
+            // similarly, if we matched Type{T<:Any}::DataType,
+            // then we don't want to cache it that way
+            // since lookup will think we matched ::Type{T}
+            // and that is quite a different thing
+            jl_svecset(newparams, i, jl_datatype_type);
+            need_guard_entries = 1; // DataType has a UID so its precedence in the cache may be too high
+        }
         else if (jl_is_type_type(elt) && jl_is_type_type(jl_tparam0(elt)) &&
                  // give up on specializing static parameters for Type{Type{Type{...}}}
                  (jl_is_type_type(jl_tparam0(jl_tparam0(elt))) ||

test/core.jl

@@ -174,6 +174,25 @@ let T = TypeVar(:T, Tuple{Vararg{RangeIndex}}, true)
     @test  args_morespecific(t2, t1)
 end
 
+# issue #11840
+f11840(::Type) = "Type"
+f11840(::DataType) = "DataType"
+@test f11840(Type) == "DataType"
+@test f11840(AbstractVector) == "Type"
+
+g11840(::DataType) = 1
+g11840(::Type) = 2
+@test g11840(Vector.body) == 1
+@test g11840(Vector) == 2
+@test g11840(Vector.body) == 1
+
+h11840(::DataType) = '1'
+h11840(::Type) = '2'
+h11840(::TypeConstructor) = '3'
+@test h11840(Vector) == '3'
+@test h11840(Vector.body) == '1'
+@test h11840(Vector) == '3'
+
 # join
 @test typejoin(Int8,Int16) === Signed
 @test typejoin(Int,AbstractString) === Any