Use LLVM alignment for tuples that become LLVM vectors.

But take care when that alignment exceeds the GC maximum alignment.

Author: Arch D. Robison

src/alloc.c

@@ -643,6 +643,13 @@ void jl_compute_field_offsets(jl_datatype_t *st)
             jl_throw(jl_overflow_exception);
         sz += fsz;
     }
+    if (jl_is_tuple_type(st)) {
+        // Some tuples become LLVM vectors with stronger alignment than what was calculated above.
+        unsigned al = jl_llvm_special_tuple_alignment(st);
+        assert(al % alignm == 0);
+        if (al)
+            alignm = al;
+    }
     st->alignment = alignm;
     st->size = LLT_ALIGN(sz, alignm);
     if (st->size > sz)

src/cgutils.cpp

@@ -348,70 +348,103 @@ JL_DLLEXPORT Type *julia_type_to_llvm(jl_value_t *jt, bool *isboxed)
 }
 }
 
+// Set jst->struct_decl to the LLVM type corresponding to jst.
+// jst must be a tuple or structtype.
+static void set_struct_decl(jl_datatype_t *jst) {
+    assert(jst->struct_decl == nullptr);
+    bool isTuple = jl_is_tuple_type(jst);
+    bool isVecElement = is_vecelement_type((jl_value_t*)jst);
+    StructType *structdecl;
+    if (!isTuple) {
+        if (!isVecElement) {
+            structdecl = StructType::create(jl_LLVMContext, jl_symbol_name(jst->name->name));
+            jst->struct_decl = structdecl;
+        }
+    }
+    size_t ntypes = jl_datatype_nfields(jst);
+    assert(ntypes > 0);
+    std::vector<Type*> latypes(0);
+    bool isvector = true;
+    Type *lasttype = NULL;
+    for(size_t i = 0; i < ntypes; ++i) {
+        Type *lty;
+        if (jl_field_isptr(jst, i))
+            lty = T_pjlvalue;
+        else {
+            jl_value_t *ty = jl_svecref(jst->types, i);
+            lty = ty==(jl_value_t*)jl_bool_type ? T_int8 : julia_type_to_llvm(ty);
+        }
+        if (lasttype != NULL && lasttype != lty)
+            isvector = false;
+        lasttype = lty;
+        if (type_is_ghost(lty))
+            lty = NoopType;
+        latypes.push_back(lty);
+    }
+    if (!isTuple) {
+        if (!isVecElement)
+            structdecl->setBody(latypes);
+        else
+            // VecElement type is unwrapped in LLVM
+            jst->struct_decl = latypes[0];
+    }
+    else {
+        if (isvector && lasttype != T_int1 && !type_is_ghost(lasttype)) {
+            // Homogeneous tuple
+            // TODO: currently we get LLVM assertion failures for other vector sizes
+            bool validVectorSize = (ntypes == 2 || ntypes == 4 || ntypes == 8 || ntypes == 16);
+            if (lasttype->isSingleValueType() && !lasttype->isVectorTy() && validVectorSize && is_vecelement_type(jl_svecref(jst->types,0)))
+                jst->struct_decl = VectorType::get(lasttype, ntypes);
+            else
+                jst->struct_decl = ArrayType::get(lasttype, ntypes);
+        }
+        else {
+            // Heterogeneous tuple
+            jst->struct_decl = StructType::get(jl_LLVMContext,ArrayRef<Type*>(&latypes[0],ntypes));
+        }
+    }
+}
+
 static Type *julia_struct_to_llvm(jl_value_t *jt, bool *isboxed)
 {
     // this function converts a Julia Type into the equivalent LLVM struct
     // use this where C-compatible (unboxed) structs are desired
     // use julia_type_to_llvm directly when you want to preserve Julia's type semantics
-    bool isTuple = jl_is_tuple_type(jt);
     if (isboxed) *isboxed = false;
-    if ((isTuple || jl_is_structtype(jt)) && !jl_is_array_type(jt)) {
+    if ((jl_is_tuple_type(jt) || jl_is_structtype(jt)) && !jl_is_array_type(jt)) {
         if (!jl_is_leaf_type(jt))
-            return NULL;
+            return nullptr;
         jl_datatype_t *jst = (jl_datatype_t*)jt;
-        if (jst->struct_decl == NULL) {
-            size_t ntypes = jl_datatype_nfields(jst);
-            if (ntypes == 0 || jst->size == 0)
+        if (jst->struct_decl == nullptr) {
+            assert(jl_datatype_nfields(jst) != 0 || jst->size == 0);
+            if (jst->size == 0)
                 return T_void;
-            StructType *structdecl;
-            if (!isTuple) {
-                structdecl = StructType::create(jl_LLVMContext, jl_symbol_name(jst->name->name));
-                jst->struct_decl = structdecl;
-            }
-            std::vector<Type*> latypes(0);
-            size_t i;
-            bool isvector = true;
-            Type *lasttype = NULL;
-            for(i = 0; i < ntypes; i++) {
-                jl_value_t *ty = jl_svecref(jst->types, i);
-                Type *lty;
-                if (jl_field_isptr(jst, i))
-                    lty = T_pjlvalue;
-                else
-                    lty = ty==(jl_value_t*)jl_bool_type ? T_int8 : julia_type_to_llvm(ty);
-                if (lasttype != NULL && lasttype != lty)
-                    isvector = false;
-                lasttype = lty;
-                if (type_is_ghost(lty))
-                    lty = NoopType;
-                latypes.push_back(lty);
-            }
-            if (!isTuple) {
-                if (is_vecelement_type(jt))
-                    // VecElement type is unwrapped in LLVM
-                    jst->struct_decl = latypes[0];
-                else
-                    structdecl->setBody(latypes);
-            }
-            else {
-                if (isvector && lasttype != T_int1 && !type_is_ghost(lasttype)) {
-                    // TODO: currently we get LLVM assertion failures for other vector sizes
-                    bool validVectorSize = (ntypes == 2 || ntypes == 4 || ntypes == 8 || ntypes == 16);
-                    if (lasttype->isSingleValueType() && !lasttype->isVectorTy() && validVectorSize && is_vecelement_type(jl_svecref(jst->types,0)))
-                        jst->struct_decl = VectorType::get(lasttype, ntypes);
-                    else
-                        jst->struct_decl = ArrayType::get(lasttype, ntypes);
-                }
-                else {
-                    jst->struct_decl = StructType::get(jl_LLVMContext,ArrayRef<Type*>(&latypes[0],ntypes));
-                }
-            }
+            set_struct_decl(jst);
         }
         return (Type*)jst->struct_decl;
     }
     return julia_type_to_llvm(jt, isboxed);
 }
 
+// Returns 0 if no special rule applies
+unsigned jl_llvm_special_tuple_alignment(jl_tupletype_t *st)
+{
+    size_t ntypes = jl_datatype_nfields(st);
+    // Run fast rejection tests
+    if (ntypes==0)
+        return 0;
+    for(size_t i = 0; i < ntypes; i++) {
+        jl_value_t *ty = jl_svecref(st->types, i);
+        if (!is_vecelement_type(ty))
+            return 0;
+    }
+    // Not rejected.  Get definitive answer from corresponding LLVM type.
+    jl_datatype_t *jst = (jl_datatype_t*)st;
+    if (!jst->struct_decl)
+        set_struct_decl(jst);
+    return jl_ExecutionEngine->getDataLayout().getABITypeAlignment((Type*)jst->struct_decl);
+}
+
 static bool is_datatype_all_pointers(jl_datatype_t *dt)
 {
     size_t i, l = jl_datatype_nfields(dt);
@@ -766,9 +799,9 @@ static Value *emit_bounds_check(const jl_cgval_t &ainfo, jl_value_t *ty, Value *
 // Parameter ptr should be the pointer argument for the LoadInst or StoreInst.
 // It is currently unused, but might be used in the future for a more precise answer.
 static unsigned julia_alignment(Value* /*ptr*/, jl_value_t *jltype, unsigned alignment) {
-    if (!alignment && ((jl_datatype_t*)jltype)->size >= 32) {
-        // Type might contain a 32-byte vector.  Use Julia alignment to prevent llvm from assuming default alignment.
-        return ((jl_datatype_t*)jltype)->alignment;
+    if (!alignment && ((jl_datatype_t*)jltype)->alignment > MAX_ALIGN) {
+        // Type's natural alignment exceeds strictess alignment promised in heap, so return the heap alignment.
+        return MAX_ALIGN;
     }
     return alignment;
 }

src/julia_internal.h

@@ -213,6 +213,8 @@ int jl_has_intrinsics(jl_lambda_info_t *li, jl_expr_t *e, jl_module_t *m);
 
 jl_value_t *jl_nth_slot_type(jl_tupletype_t *sig, size_t i);
 void jl_compute_field_offsets(jl_datatype_t *st);
+unsigned jl_llvm_special_tuple_alignment(jl_datatype_t *st);
+
 jl_array_t *jl_new_array_for_deserialization(jl_value_t *atype, uint32_t ndims, size_t *dims,
                                              int isunboxed, int elsz);
 jl_lambda_info_t *jl_copy_lambda_info(jl_lambda_info_t *linfo);