layout: support pointers inlining into parents [NFCI] (#33886)

Includes codegen support for immutable objects that contain pointers
appearing on stack (well, in registers, since LLVM support of
non-integral addrspace pointers inside aggregates in memory is poor),
and includes layout support, so that most (non-self-referential)
immutable objects can be stored inline inside their parent allocation.

Currently fully disabled, aside from some optimizations and improvements
to object_id / isa tests.

Co-Authored-By: Oscar Blumberg <[email protected]>

Author: 2 people

base/array.jl

@@ -158,7 +158,7 @@ size(a::Array{<:Any,N}) where {N} = (@_inline_meta; ntuple(M -> size(a, M), Val(
 
 asize_from(a::Array, n) = n > ndims(a) ? () : (arraysize(a,n), asize_from(a, n+1)...)
 
-allocatedinline(::Type{T}) where {T} = (@_pure_meta; ccall(:jl_array_store_unboxed, Cint, (Any,), T) != Cint(0))
+allocatedinline(::Type{T}) where {T} = (@_pure_meta; ccall(:jl_stored_inline, Cint, (Any,), T) != Cint(0))
 
 """
     Base.isbitsunion(::Type{T})
@@ -177,14 +177,20 @@ false
 isbitsunion(u::Union) = allocatedinline(u)
 isbitsunion(x) = false
 
-function _unsetindex!(A::Array{T}, i::Int) where {T}
+@inbounds function _unsetindex!(A::Array{T}, i::Int) where {T}
     @boundscheck checkbounds(A, i)
+    t = @_gc_preserve_begin A
+    p = Ptr{Ptr{Cvoid}}(pointer(A, i))
     if !allocatedinline(T)
-        t = @_gc_preserve_begin A
-        p = Ptr{Ptr{Cvoid}}(pointer(A))
-        unsafe_store!(p, C_NULL, i)
-        @_gc_preserve_end t
+        unsafe_store!(p, C_NULL)
+    elseif T isa DataType
+        if !datatype_pointerfree(T)
+            for j = 1:(Core.sizeof(T) ÷ Core.sizeof(Ptr{Cvoid}))
+                unsafe_store!(p, C_NULL, j)
+            end
+        end
     end
+    @_gc_preserve_end t
     return A
 end
 
@@ -255,19 +261,41 @@ the same manner as C.
 function unsafe_copyto!(dest::Array{T}, doffs, src::Array{T}, soffs, n) where T
     t1 = @_gc_preserve_begin dest
     t2 = @_gc_preserve_begin src
-    if isbitsunion(T)
+    destp = pointer(dest, doffs)
+    srcp = pointer(src, soffs)
+    if !allocatedinline(T)
+        ccall(:jl_array_ptr_copy, Cvoid, (Any, Ptr{Cvoid}, Any, Ptr{Cvoid}, Int),
+              dest, destp, src, srcp, n)
+    elseif isbitstype(T)
         ccall(:memmove, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, UInt),
-              pointer(dest, doffs), pointer(src, soffs), n * aligned_sizeof(T))
+              destp, srcp, n * aligned_sizeof(T))
+    elseif isbitsunion(T)
+        ccall(:memmove, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, UInt),
+              destp, srcp, n * aligned_sizeof(T))
         # copy selector bytes
         ccall(:memmove, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, UInt),
               ccall(:jl_array_typetagdata, Ptr{UInt8}, (Any,), dest) + doffs - 1,
               ccall(:jl_array_typetagdata, Ptr{UInt8}, (Any,), src) + soffs - 1,
               n)
-    elseif allocatedinline(T)
-        unsafe_copyto!(pointer(dest, doffs), pointer(src, soffs), n)
     else
-        ccall(:jl_array_ptr_copy, Cvoid, (Any, Ptr{Cvoid}, Any, Ptr{Cvoid}, Int),
-              dest, pointer(dest, doffs), src, pointer(src, soffs), n)
+        # handle base-case: everything else above was just optimizations
+        @inbounds if destp < srcp || destp > srcp + n
+            for i = 1:n
+                if isassigned(src, soffs + i - 1)
+                    dest[doffs + i - 1] = src[soffs + i - 1]
+                else
+                    _unsetindex!(dest, doffs + i - 1)
+                end
+            end
+        else
+            for i = n:-1:1
+                if isassigned(src, soffs + i - 1)
+                    dest[doffs + i - 1] = src[soffs + i - 1]
+                else
+                    _unsetindex!(dest, doffs + i - 1)
+                end
+            end
+        end
     end
     @_gc_preserve_end t2
     @_gc_preserve_end t1
@@ -1566,32 +1594,13 @@ function vcat(arrays::Vector{T}...) where T
         n += length(a)
     end
     arr = Vector{T}(undef, n)
-    ptr = pointer(arr)
-    if isbitsunion(T)
-        selptr = ccall(:jl_array_typetagdata, Ptr{UInt8}, (Any,), arr)
-    end
-    elsz = aligned_sizeof(T)
-    t = @_gc_preserve_begin arr
+    nd = 1
     for a in arrays
         na = length(a)
-        nba = na * elsz
-        if isbitsunion(T)
-            ccall(:memcpy, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, UInt),
-                  ptr, a, nba)
-            # copy selector bytes
-            ccall(:memcpy, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, UInt),
-                  selptr, ccall(:jl_array_typetagdata, Ptr{UInt8}, (Any,), a), na)
-            selptr += na
-        elseif allocatedinline(T)
-            ccall(:memcpy, Ptr{Cvoid}, (Ptr{Cvoid}, Ptr{Cvoid}, UInt),
-                  ptr, a, nba)
-        else
-            ccall(:jl_array_ptr_copy, Cvoid, (Any, Ptr{Cvoid}, Any, Ptr{Cvoid}, Int),
-                  arr, ptr, a, pointer(a), na)
-        end
-        ptr += nba
+        @assert nd + na <= 1 + length(arr) # Concurrent modification of arrays?
+        unsafe_copyto!(arr, nd, a, 1, na)
+        nd += na
     end
-    @_gc_preserve_end t
     return arr
 end
 

base/reflection.jl

@@ -318,6 +318,7 @@ datatype_fieldtypes(x::DataType) = ccall(:jl_get_fieldtypes, Any, (Any,), x)
 struct DataTypeLayout
     nfields::UInt32
     npointers::UInt32
+    firstptr::Int32
     alignment::UInt32
     # alignment : 9;
     # haspadding : 1;

base/refpointer.jl

@@ -74,7 +74,7 @@ RefArray(x::AbstractArray{T}, i::Int=1, roots::Nothing=nothing) where {T} = RefA
 convert(::Type{Ref{T}}, x::AbstractArray{T}) where {T} = RefArray(x, 1)
 
 function unsafe_convert(P::Type{Ptr{T}}, b::RefArray{T}) where T
-    if datatype_pointerfree(RefValue{T})
+    if allocatedinline(T)
         p = pointer(b.x, b.i)
     elseif isconcretetype(T) && T.mutable
         p = pointer_from_objref(b.x[b.i])

base/refvalue.jl

@@ -11,7 +11,7 @@ RefValue(x::T) where {T} = RefValue{T}(x)
 isassigned(x::RefValue) = isdefined(x, :x)
 
 function unsafe_convert(P::Type{Ptr{T}}, b::RefValue{T}) where T
-    if datatype_pointerfree(RefValue{T})
+    if allocatedinline(T)
         p = pointer_from_objref(b)
     elseif isconcretetype(T) && T.mutable
         p = pointer_from_objref(b.x)

src/array.c

@@ -27,12 +27,6 @@ char *jl_array_typetagdata(jl_array_t *a) JL_NOTSAFEPOINT
     return ((char*)jl_array_data(a)) + ((jl_array_ndims(a) == 1 ? (a->maxsize - a->offset) : jl_array_len(a)) * a->elsize) + a->offset;
 }
 
-JL_DLLEXPORT int jl_array_store_unboxed(jl_value_t *eltype) JL_NOTSAFEPOINT
-{
-    size_t fsz = 0, al = 0;
-    return jl_islayout_inline(eltype, &fsz, &al);
-}
-
 STATIC_INLINE jl_value_t *jl_array_owner(jl_array_t *a JL_PROPAGATES_ROOT) JL_NOTSAFEPOINT
 {
     if (a->flags.how == 3) {
@@ -53,7 +47,7 @@ size_t jl_arr_xtralloc_limit = 0;
 #define MAXINTVAL (((size_t)-1)>>1)
 
 static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims,
-                               int isunboxed, int isunion, int elsz)
+                               int isunboxed, int hasptr, int isunion, int elsz)
 {
     jl_ptls_t ptls = jl_get_ptls_states();
     size_t i, tot, nel=1;
@@ -101,7 +95,7 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims,
         // No allocation or safepoint allowed after this
         a->flags.how = 0;
         data = (char*)a + doffs;
-        if ((tot > 0 && !isunboxed) || isunion)
+        if (tot > 0 && (!isunboxed || hasptr || isunion)) // TODO: check for zeroinit
             memset(data, 0, tot);
     }
     else {
@@ -113,7 +107,7 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims,
         // No allocation or safepoint allowed after this
         a->flags.how = 2;
         jl_gc_track_malloced_array(ptls, a);
-        if (!isunboxed || isunion)
+        if (tot > 0 && (!isunboxed || hasptr || isunion)) // TODO: check for zeroinit
             // need to zero out isbits union array selector bytes to ensure a valid type index
             memset(data, 0, tot);
     }
@@ -127,6 +121,7 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims,
 #endif
     a->flags.ndims = ndims;
     a->flags.ptrarray = !isunboxed;
+    a->flags.hasptr = hasptr;
     a->elsize = elsz;
     a->flags.isshared = 0;
     a->flags.isaligned = 1;
@@ -135,9 +130,12 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims,
         a->nrows = nel;
         a->maxsize = nel;
     }
+    else if (a->flags.ndims != ndims) {
+        jl_exceptionf(jl_argumenterror_type, "invalid Array dimensions");
+    }
     else {
         size_t *adims = &a->nrows;
-        for(i=0; i < ndims; i++)
+        for (i = 0; i < ndims; i++)
             adims[i] = dims[i];
     }
 
@@ -152,6 +150,7 @@ static inline jl_array_t *_new_array(jl_value_t *atype, uint32_t ndims, size_t *
         jl_type_error_rt("Array", "element type", (jl_value_t*)jl_type_type, eltype);
     int isunboxed = jl_islayout_inline(eltype, &elsz, &al);
     int isunion = jl_is_uniontype(eltype);
+    int hasptr = isunboxed && (jl_is_datatype(eltype) && ((jl_datatype_t*)eltype)->layout->npointers > 0);
     if (!isunboxed) {
         elsz = sizeof(void*);
         al = elsz;
@@ -160,13 +159,13 @@ static inline jl_array_t *_new_array(jl_value_t *atype, uint32_t ndims, size_t *
         elsz = LLT_ALIGN(elsz, al);
     }
 
-    return _new_array_(atype, ndims, dims, isunboxed, isunion, elsz);
+    return _new_array_(atype, ndims, dims, isunboxed, hasptr, isunion, elsz);
 }
 
 jl_array_t *jl_new_array_for_deserialization(jl_value_t *atype, uint32_t ndims, size_t *dims,
-                                             int isunboxed, int isunion, int elsz)
+                                             int isunboxed, int hasptr, int isunion, int elsz)
 {
-    return _new_array_(atype, ndims, dims, isunboxed, isunion, elsz);
+    return _new_array_(atype, ndims, dims, isunboxed, hasptr, isunion, elsz);
 }
 
 #ifndef JL_NDEBUG
@@ -224,10 +223,12 @@ JL_DLLEXPORT jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data,
                           "reinterpret from alignment %d bytes to alignment %d bytes not allowed",
                           (int) oldalign, (int) align);
         a->flags.ptrarray = 0;
+        a->flags.hasptr = data->flags.hasptr;
     }
     else {
         a->elsize = sizeof(void*);
         a->flags.ptrarray = 1;
+        a->flags.hasptr = 0;
     }
 
     // if data is itself a shared wrapper,
@@ -247,6 +248,9 @@ JL_DLLEXPORT jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data,
         a->nrows = l;
         a->maxsize = l;
     }
+    else if (a->flags.ndims != ndims) {
+        jl_exceptionf(jl_argumenterror_type, "invalid Array dimensions");
+    }
     else {
         size_t *adims = &a->nrows;
         size_t l = 1;
@@ -281,6 +285,7 @@ JL_DLLEXPORT jl_array_t *jl_string_to_array(jl_value_t *str)
     a->flags.isaligned = 0;
     a->elsize = 1;
     a->flags.ptrarray = 0;
+    a->flags.hasptr = 0;
     jl_array_data_owner(a) = str;
     a->flags.how = 3;
     a->flags.isshared = 1;
@@ -300,12 +305,12 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data,
     jl_array_t *a;
     jl_value_t *eltype = jl_tparam0(atype);
 
-    int isunboxed = jl_array_store_unboxed(eltype);
-    size_t elsz;
-    unsigned align;
+    int isunboxed = jl_stored_inline(eltype);
     if (isunboxed && jl_is_uniontype(eltype))
         jl_exceptionf(jl_argumenterror_type,
                       "unsafe_wrap: unspecified layout for union element type");
+    size_t elsz;
+    unsigned align;
     if (isunboxed) {
         elsz = jl_datatype_size(eltype);
         align = jl_datatype_align(eltype);
@@ -328,6 +333,7 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data,
 #endif
     a->elsize = LLT_ALIGN(elsz, align);
     a->flags.ptrarray = !isunboxed;
+    a->flags.hasptr = isunboxed && (jl_is_datatype(eltype) && ((jl_datatype_t*)eltype)->layout->npointers > 0);
     a->flags.ndims = 1;
     a->flags.isshared = 1;
     a->flags.isaligned = 0;  // TODO: allow passing memalign'd buffers
@@ -366,12 +372,12 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data,
         return jl_ptr_to_array_1d(atype, data, nel, own_buffer);
     jl_value_t *eltype = jl_tparam0(atype);
 
-    int isunboxed = jl_array_store_unboxed(eltype);
-    size_t elsz;
-    unsigned align;
+    int isunboxed = jl_stored_inline(eltype);
     if (isunboxed && jl_is_uniontype(eltype))
         jl_exceptionf(jl_argumenterror_type,
                       "unsafe_wrap: unspecified layout for union element type");
+    size_t elsz;
+    unsigned align;
     if (isunboxed) {
         elsz = jl_datatype_size(eltype);
         align = jl_datatype_align(eltype);
@@ -394,6 +400,7 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data,
 #endif
     a->elsize = LLT_ALIGN(elsz, align);
     a->flags.ptrarray = !isunboxed;
+    a->flags.hasptr = isunboxed && (jl_is_datatype(eltype) && ((jl_datatype_t*)eltype)->layout->npointers > 0);
     a->flags.ndims = ndims;
     a->offset = 0;
     a->flags.isshared = 1;
@@ -408,6 +415,8 @@ JL_DLLEXPORT jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data,
     }
 
     assert(ndims != 1); // handled above
+    if (a->flags.ndims != ndims)
+        jl_exceptionf(jl_argumenterror_type, "invalid Array dimensions");
     memcpy(&a->nrows, dims, ndims * sizeof(size_t));
     return a;
 }
@@ -559,8 +568,16 @@ JL_DLLEXPORT jl_value_t *jl_arrayref(jl_array_t *a, size_t i)
 
 JL_DLLEXPORT int jl_array_isassigned(jl_array_t *a, size_t i)
 {
-    if (a->flags.ptrarray)
+    if (a->flags.ptrarray) {
         return ((jl_value_t**)jl_array_data(a))[i] != NULL;
+    }
+    else if (a->flags.hasptr) {
+         jl_datatype_t *eltype = (jl_datatype_t*)jl_tparam0(jl_typeof(a));
+         assert(eltype->layout->first_ptr >= 0);
+         jl_value_t **slot =
+             (jl_value_t**)(&((char*)a->data)[i*a->elsize] + eltype->layout->first_ptr);
+         return *slot != NULL;
+    }
     return 1;
 }
 
@@ -585,6 +602,8 @@ JL_DLLEXPORT void jl_arrayset(jl_array_t *a JL_ROOTING_ARGUMENT, jl_value_t *rhs
                 return;
         }
         jl_assign_bits(&((char*)a->data)[i * a->elsize], rhs);
+        if (a->flags.hasptr)
+            jl_gc_multi_wb(jl_array_owner(a), rhs);
     }
     else {
         ((jl_value_t**)a->data)[i] = rhs;
@@ -598,6 +617,11 @@ JL_DLLEXPORT void jl_arrayunset(jl_array_t *a, size_t i)
         jl_bounds_error_int((jl_value_t*)a, i + 1);
     if (a->flags.ptrarray)
         ((jl_value_t**)a->data)[i] = NULL;
+    else if (a->flags.hasptr) {
+        size_t elsize = a->elsize;
+        jl_assume(elsize >= sizeof(void*) && elsize % sizeof(void*) == 0);
+        memset(&((jl_value_t**)a->data)[i], 0, elsize);
+    }
 }
 
 // at this size and bigger, allocate resized array data with malloc directly
@@ -809,7 +833,7 @@ STATIC_INLINE void jl_array_grow_at_beg(jl_array_t *a, size_t idx, size_t inc,
 #endif
     a->nrows = newnrows;
     a->data = newdata;
-    if (a->flags.ptrarray) {
+    if (a->flags.ptrarray || a->flags.hasptr) { // TODO: check for zeroinit
         memset(newdata + idx * elsz, 0, nbinc);
     }
     else if (isbitsunion) {
@@ -890,7 +914,7 @@ STATIC_INLINE void jl_array_grow_at_end(jl_array_t *a, size_t idx,
     a->length = newnrows;
 #endif
     a->nrows = newnrows;
-    if (a->flags.ptrarray) {
+    if (a->flags.ptrarray || a->flags.hasptr) { // TODO: check for zeroinit
         memset(data + idx * elsz, 0, inc * elsz);
     }
 }
@@ -1129,7 +1153,7 @@ JL_DLLEXPORT jl_array_t *jl_array_copy(jl_array_t *ary)
     int isunion = jl_is_uniontype(jl_tparam0(jl_typeof(ary)));
     jl_array_t *new_ary = _new_array_(jl_typeof(ary), jl_array_ndims(ary),
                                       &ary->nrows, !ary->flags.ptrarray,
-                                      isunion, elsz);
+                                      ary->flags.hasptr, isunion, elsz);
     memcpy(new_ary->data, ary->data, len * elsz);
     // ensure isbits union arrays copy their selector bytes correctly
     if (jl_array_isbitsunion(ary))