store tuple and vector types to the stack eagerly [ci skip]

fix #11187, fix #11450, fix #11026, ref #10525, fix #11003
TODO: confirm all of those numbers were fixed
TODO: ensure the lazy-loaded objects have gc-roots
TODO: re-enable VectorType objects, so small objects still end up in
registers in the calling convention
TODO: allow moving pointers sometimes rather than copying
TODO: teach the GC how it can re-use an existing pointer as a box

this also changes the julia specSig calling convention to pass
non-primitive types by pointer instead of by-value

this additionally fixes a bug in gen_cfunction that could be exposed by
turning off specSig

this additionally moves the alloca calls in ccall (and other places) to
the entry BasicBlock in the function, ensuring that llvm detects them as
static allocations and moves them into the function prologue

Author: vtjnash

base/inference.jl

@@ -3038,6 +3038,7 @@ end
 function remove_redundant_temp_vars(ast, sa)
     varinfo = ast.args[2][2]
     gensym_types = ast.args[2][4]
+    body = ast.args[3]
     for (v,init) in sa
         if ((isa(init,Symbol) || isa(init,SymbolNode)) &&
             any(vi->symequal(vi[1],init), varinfo) &&
@@ -3046,7 +3047,7 @@ function remove_redundant_temp_vars(ast, sa)
             # this transformation is not valid for vars used before def.
             # we need to preserve the point of assignment to know where to
             # throw errors (issue #4645).
-            if !occurs_undef(v, ast.args[3], varinfo)
+            if !occurs_undef(v, body, varinfo)
 
                 # the transformation is not ideal if the assignment
                 # is present for the auto-unbox functionality
@@ -3055,7 +3056,7 @@ function remove_redundant_temp_vars(ast, sa)
                 # everywhere later in the function
                 if (isa(init,SymbolNode) ? (init.typ <: (isa(v,GenSym)?gensym_types[(v::GenSym).id+1]:local_typeof(v, varinfo))) : true)
                     delete_var!(ast, v)
-                    sym_replace(ast.args[3], [v], [], [init], [])
+                    sym_replace(body, Any[v], Void[], Any[init], Void[])
                 end
             end
         end

src/builtins.c

@@ -273,7 +273,7 @@ static int NOINLINE compare_fields(jl_value_t *a, jl_value_t *b,
     return 1;
 }
 
-int jl_egal(jl_value_t *a, jl_value_t *b)
+int jl_egal(jl_value_t *a, jl_value_t *b) // warning: a,b may NOT have been gc-rooted by the caller
 {
     if (a == b)
         return 1;

src/ccall.cpp

@@ -246,17 +246,51 @@ static Value *runtime_sym_lookup(PointerType *funcptype, char *f_lib, char *f_na
 #  include "abi_llvm.cpp"
 #endif
 
-Value *llvm_type_rewrite(Value *v, Type *target_type, jl_value_t *ty, bool isret)
+Value *llvm_type_rewrite(Value *v, Type *from_type, Type *target_type, bool tojulia, bool byref)
 {
-    if (preferred_llvm_type(ty,isret) == NULL || target_type == NULL || target_type == v->getType())
+    Type *ptarget_type = PointerType::get(target_type, 0);
+
+    if (tojulia) {
+        if (byref) {
+            if (v->getType() != ptarget_type) {
+                v = builder.CreatePointerCast(v, ptarget_type);
+            }
+            return builder.CreateLoad(v);
+        }
+    }
+    else {
+        if (byref) { // client is supposed to have already done the alloca and store
+            if (v->getType() != target_type) {
+                v = builder.CreatePointerCast(v, target_type);
+            }
+            return v;
+        }
+
+        if (v->getType() != from_type) { // this is already be a pointer in the codegen
+            if (v->getType() != ptarget_type) {
+                v = builder.CreatePointerCast(v, ptarget_type);
+            }
+            return builder.CreateLoad(v);
+        }
+    }
+    assert(v->getType() == from_type);
+
+    if (target_type == from_type) {
         return v;
+    }
 
-    assert(!v->getType()->isPointerTy());
+    if ((target_type->isIntegerTy() && from_type->isIntegerTy()) ||
+        (target_type->isFloatingPointTy() && from_type->isFloatingPointTy()) ||
+        (target_type->isPointerTy() && from_type->isPointerTy())) {
+        assert(target_type->getPrimitiveSizeInBits() == from_type->getPrimitiveSizeInBits());
+        return builder.CreateBitCast(v, target_type);
+    }
 
+    // Vector or non-Aggregate types
     // LLVM doesn't allow us to cast values directly, so
     // we need to use this alloca trick
-    Value *mem = builder.CreateAlloca(target_type);
-    builder.CreateStore(v,builder.CreatePointerCast(mem,v->getType()->getPointerTo()));
+    Value *mem = builder.CreateAlloca(target_type); // XXX: don't frob the stack
+    builder.CreateStore(v, builder.CreatePointerCast(mem, from_type->getPointerTo()));
     return builder.CreateLoad(mem);
 }
 
@@ -265,27 +299,36 @@ Value *llvm_type_rewrite(Value *v, Type *target_type, jl_value_t *ty, bool isret
 static Value *julia_to_native(Type *ty, jl_value_t *jt, Value *jv,
                               jl_value_t *aty, bool addressOf,
                               bool byRef, bool inReg,
-                              bool needCopy,
+                              bool needCopy, bool tojulia,
                               int argn, jl_codectx_t *ctx,
                               bool *needStackRestore)
 {
     Type *vt = jv->getType();
 
-    // We're passing any
+    // We're passing Any
     if (ty == jl_pvalue_llvmt) {
         return boxed(jv,ctx);
     }
+
+    if (!tojulia && julia_type_to_llvm(aty)->isAggregateType()) {
+        // this value is expected to be a pointer in the julia codegen,
+        // so it needs to be extracted first if not tojulia
+        vt = vt->getContainedType(0);
+    }
+
     if (ty == vt && !addressOf && !byRef) {
         return jv;
     }
+
     if (vt != jl_pvalue_llvmt) {
         // argument value is unboxed
+        if (vt != jv->getType())
+            jv = builder.CreateLoad(jv);
         if (addressOf || (byRef && inReg)) {
-            if (ty->isPointerTy() && ty->getContainedType(0)==vt) {
+            if (ty->isPointerTy() && ty->getContainedType(0) == vt) {
                 // pass the address of an alloca'd thing, not a box
                 // since those are immutable.
-                *needStackRestore = true;
-                Value *slot = builder.CreateAlloca(vt);
+                Value *slot = emit_static_alloca(vt, ctx);
                 builder.CreateStore(jv, slot);
                 return builder.CreateBitCast(slot, ty);
             }
@@ -299,29 +342,30 @@ static Value *julia_to_native(Type *ty, jl_value_t *jt, Value *jv,
                     return builder.CreateBitCast(jv, ty);
                 }
                 else {
-                    *needStackRestore = true;
-                    Value *mem = builder.CreateAlloca(ty);
+                    Value *mem = emit_static_alloca(ty, ctx);
                     builder.CreateStore(jv,builder.CreateBitCast(mem,vt->getPointerTo()));
                     return mem;
                 }
             }
         }
         else if (vt->isStructTy()) {
-            if (!byRef) {
-                return jv;
+            if (byRef) {
+                Value *mem = emit_static_alloca(vt, ctx);
+                builder.CreateStore(jv, mem);
+                return mem;
             }
             else {
-                *needStackRestore = true;
-                Value *mem = builder.CreateAlloca(vt);
-                builder.CreateStore(jv,mem);
-                return mem;
+                return jv;
             }
         }
 
         emit_error("ccall: argument type did not match declaration", ctx);
     }
+
+    // argument value is boxed
     if (jl_is_tuple(jt)) {
-        return emit_unbox(ty,jv,jt);
+        emit_error("ccall: unimplemented: boxed tuple argument type", ctx);
+        return jv; // TODO: this is wrong
     }
     if (jl_is_cpointer_type(jt) && addressOf) {
         assert(ty->isPointerTy());
@@ -349,7 +393,7 @@ static Value *julia_to_native(Type *ty, jl_value_t *jt, Value *jv,
             *needStackRestore = true;
             AllocaInst *ai = builder.CreateAlloca(T_int8, nbytes);
             ai->setAlignment(16);
-            builder.CreateMemCpy(ai, builder.CreateBitCast(jv, T_pint8), nbytes, 1);
+            builder.CreateMemCpy(ai, builder.CreateBitCast(jv, T_pint8), nbytes, 0);
             return builder.CreateBitCast(ai, ty);
         }
         // emit maybe copy
@@ -375,7 +419,7 @@ static Value *julia_to_native(Type *ty, jl_value_t *jt, Value *jv,
                     false));
         AllocaInst *ai = builder.CreateAlloca(T_int8, nbytes);
         ai->setAlignment(16);
-        builder.CreateMemCpy(ai, builder.CreatePointerCast(jv, T_pint8), nbytes, 1);
+        builder.CreateMemCpy(ai, builder.CreatePointerCast(jv, T_pint8), nbytes, 0);
         Value *p2 = builder.CreatePointerCast(ai, ty);
         builder.CreateBr(afterBB);
         builder.SetInsertPoint(afterBB);
@@ -393,21 +437,19 @@ static Value *julia_to_native(Type *ty, jl_value_t *jt, Value *jv,
         msg << argn;
         emit_typecheck(jv, jt, msg.str(), ctx);
     }
-    Value *p = data_pointer(jv);
-    Value *pjv = builder.CreatePointerCast(p, PointerType::get(ty,0));
+    Value *pjv = builder.CreatePointerCast(jv, PointerType::get(ty,0));
     if (byRef) {
         if (!needCopy) {
             return pjv;
         }
         else {
-            *needStackRestore = true;
-            Value *mem = builder.CreateAlloca(ty);
-            builder.CreateMemCpy(mem,pjv,(uint64_t)jl_datatype_size(jt),(uint64_t)((jl_datatype_t*)jt)->alignment);
+            Value *mem = emit_static_alloca(ty, ctx);
+            builder.CreateMemCpy(mem, pjv, (uint64_t)jl_datatype_size(jt), (uint64_t)((jl_datatype_t*)jt)->alignment);
             return mem;
         }
     }
     else {
-        return builder.CreateLoad(pjv,false);
+        return pjv; // lazy load by llvm_type_rewrite
     }
 }
 
@@ -661,7 +703,8 @@ static Value *emit_llvmcall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx)
             make_gcroot(arg, ctx);
         }
 #endif
-        argvals[i] = julia_to_native(t, tti, arg, expr_type(argi, ctx), false, false, false, false, i, ctx, NULL);
+        Value *v = julia_to_native(t, tti, arg, expr_type(argi, ctx), false, false, false, false, false, i, ctx, NULL);
+        argvals[i] = llvm_type_rewrite(v, t, t, false, false);
     }
 
     Function *f;
@@ -786,6 +829,7 @@ typedef AttributeSet attr_type;
 
 static std::string generate_func_sig(Type **lrt, Type **prt, int &sret,
         std::vector<Type *> &fargt, std::vector<Type *> &fargt_sig,
+        Type *&fargt_vasig,
         std::vector<bool> &inRegList,
         std::vector<bool> &byRefList, attr_type &attributes,
         jl_value_t *rt, jl_svec_t *tt)
@@ -805,7 +849,7 @@ static std::string generate_func_sig(Type **lrt, Type **prt, int &sret,
         *prt = *lrt = T_void;
     }
     else {
-        *prt = preferred_llvm_type(rt,true);
+        *prt = preferred_llvm_type(rt, true);
         if (*prt == NULL)
             *prt = *lrt;
 
@@ -892,7 +936,7 @@ static std::string generate_func_sig(Type **lrt, Type **prt, int &sret,
 
         fargt.push_back(t);
 
-        Type *pat = preferred_llvm_type(tti,false);
+        Type *pat = preferred_llvm_type(tti, false);
         if (pat != NULL)
             t = pat;
         else if (byRef)
@@ -901,6 +945,9 @@ static std::string generate_func_sig(Type **lrt, Type **prt, int &sret,
         if (!current_isVa) {
             fargt_sig.push_back(t);
         }
+        else {
+            fargt_vasig = t;
+        }
     }
 
     if (retattrs.hasAttributes())
@@ -1128,12 +1175,13 @@ static Value *emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx)
 
     std::vector<Type*> fargt(0);
     std::vector<Type*> fargt_sig(0);
+    Type *fargt_vasig = NULL;
     std::vector<bool> inRegList(0);
     std::vector<bool> byRefList(0);
     attr_type attrs;
     Type *prt = NULL;
     int sret = 0;
-    std::string err_msg = generate_func_sig(&lrt, &prt, sret, fargt, fargt_sig, inRegList, byRefList, attrs, rt, tt);
+    std::string err_msg = generate_func_sig(&lrt, &prt, sret, fargt, fargt_sig, fargt_vasig, inRegList, byRefList, attrs, rt, tt);
     if (!err_msg.empty()) {
         JL_GC_POP();
         emit_error(err_msg,ctx);
@@ -1143,19 +1191,21 @@ static Value *emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx)
     // emit arguments
     Value **argvals = (Value**) alloca(((nargs-3)/2 + sret)*sizeof(Value*));
     Value *result = NULL;
+    bool needStackRestore = false;
 
     // First, if the ABI requires us to provide the space for the return
     // argument, allocate the box and store that as the first argument type
     if (sret) {
-        result = emit_new_struct(rt,1,NULL,ctx);
+        result = emit_new_struct(rt,1,NULL,ctx); // TODO: is it valid to be creating an incomplete type this way?
         assert(result != NULL && "Type was not concrete");
         if (!result->getType()->isPointerTy()) {
-            Value *mem = builder.CreateAlloca(lrt);
+            Value *mem = emit_static_alloca(lrt, ctx);
             builder.CreateStore(result, mem);
             result = mem;
             argvals[0] = result;
         }
         else {
+            // XXX: result needs a GC root here if result->getType() == jl_pvalue_llvmt
             argvals[0] = builder.CreateBitCast(result, fargt_sig[0]);
         }
     }
@@ -1164,7 +1214,6 @@ static Value *emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx)
     int last_depth = ctx->argDepth;
 
     // number of parameters to the c function
-    bool needStackRestore = false;
     for(i=4; i < nargs+1; i+=2) {
         // Current C function parameter
         size_t ai = (i-4)/2;
@@ -1240,9 +1289,9 @@ static Value *emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx)
         bool nSR=false;
         argvals[ai + sret] = llvm_type_rewrite(
                 julia_to_native(largty, jargty, arg, expr_type(argi, ctx), addressOf, byRefList[ai], inRegList[ai],
-                    need_private_copy(jargty, byRefList[ai]), ai + 1, ctx, &nSR),
-                fargt_sig.size() > ai + sret ? fargt_sig[ai + sret] : preferred_llvm_type(jargty, false),
-                jargty, false);
+                    need_private_copy(jargty, byRefList[ai]), false, ai + 1, ctx, &nSR),
+                largty, ai + sret < fargt_sig.size() ? fargt_sig[ai + sret] : fargt_vasig,
+                false, byRefList[ai]);
         needStackRestore |= nSR;
     }
 
@@ -1330,40 +1379,38 @@ static Value *emit_ccall(jl_value_t **args, size_t nargs, jl_codectx_t *ctx)
     JL_GC_POP();
     // Finally we need to box the result into julia type
     // However, if we have already created a box for the return
-    // type because we the ABI required us to pass a pointer (sret),
+    // type because the ABI required us to pass a pointer (sret),
     // then we do not need to do this.
     if (!sret) {
         if (lrt == T_void)
             result = literal_pointer_val((jl_value_t*)jl_nothing);
         else if (lrt->isStructTy()) {
             //fprintf(stderr, "ccall rt: %s -> %s\n", f_name, ((jl_tag_type_t*)rt)->name->name->name);
             assert(jl_is_structtype(rt));
-
             Value *newst = emit_new_struct(rt,1,NULL,ctx);
             assert(newst != NULL && "Type was not concrete");
-            if (newst->getType()->isPointerTy()) {
-                builder.CreateStore(result,builder.CreateBitCast(newst, prt->getPointerTo()));
-                result = newst;
-            }
-            else if (lrt != prt) {
-                result = llvm_type_rewrite(result,lrt,rt,true);
-            }
-            // otherwise it's fine to pass this by value. Technically we could do alloca/store/load,
-            // but why should we?
+            assert(newst->getType()->isPointerTy());
+            builder.CreateStore(result, builder.CreateBitCast(newst, prt->getPointerTo()));
+            result = newst;
         }
         else {
             if (prt->getPrimitiveSizeInBits() == lrt->getPrimitiveSizeInBits()) {
                 result = builder.CreateBitCast(result,lrt);
             }
             else {
-                Value *rloc = builder.CreateAlloca(prt);
-                builder.CreateStore(result, rloc);
-                result = builder.CreateLoad(builder.CreatePointerCast(rloc, PointerType::get(lrt,0)));
+                Value *rloc = emit_static_alloca(lrt, ctx);
+                builder.CreateStore(result, builder.CreatePointerCast(rloc, PointerType::get(prt,0)));
+                if (lrt->isAggregateType()) {
+                    result = rloc;
+                }
+                else {
+                    result = builder.CreateLoad(rloc);
+                }
             }
         }
     }
     else {
-        if (result->getType() != jl_pvalue_llvmt)
+        if (result->getType() != jl_pvalue_llvmt && !lrt->isAggregateType())
             result = builder.CreateLoad(result);
     }