annotate pointer loads and stores atomically

This lets us say that anytime you observe the store of a pointer, it's
valid to assume consume ordering on that read (on all processors except
the Alpha). It's also already the default behavior on the x86 hardware,
this just enforces the ordering on the compiler too.

Author: vtjnash

src/array.c

@@ -20,6 +20,32 @@ extern "C" {
 
 #define JL_ARRAY_ALIGN(jl_value, nbytes) LLT_ALIGN(jl_value, nbytes)
 
+// this is a version of memcpy that preserves atomic memory ordering
+// which makes it safe to use for objects that can contain memory references
+// without risk of creating pointers out of thin air
+void memmove_refs(void **dstp, void *const *srcp, size_t n) JL_NOTSAFEPOINT
+{
+    size_t i;
+    if (dstp < srcp || dstp > srcp + n) {
+        for (i = 0; i < n; i++) {
+            jl_atomic_store_relaxed(dstp + i, jl_atomic_load_relaxed(srcp + i));
+        }
+    }
+    else {
+        for (i = 0; i < n; i++) {
+            jl_atomic_store_relaxed(dstp + n - i - 1, jl_atomic_load_relaxed(srcp + n - i - 1));
+        }
+    }
+}
+
+void memmove_safe(int hasptr, char *dst, const char *src, size_t nb) JL_NOTSAFEPOINT
+{
+    if (hasptr)
+        memmove_refs((void**)dst, (void**)src, nb / sizeof(void*));
+    else
+        memmove(dst, src, nb);
+}
+
 // array constructors ---------------------------------------------------------
 char *jl_array_typetagdata(jl_array_t *a) JL_NOTSAFEPOINT
 {
@@ -542,10 +568,9 @@ JL_DLLEXPORT jl_value_t *jl_ptrarrayref(jl_array_t *a JL_PROPAGATES_ROOT, size_t
 {
     assert(i < jl_array_len(a));
     assert(a->flags.ptrarray);
-    jl_value_t *elt = ((jl_value_t**)a->data)[i];
-    if (elt == NULL) {
+    jl_value_t *elt = jl_atomic_load_relaxed(((jl_value_t**)a->data) + i);
+    if (elt == NULL)
         jl_throw(jl_undefref_exception);
-    }
     return elt;
 }
 
@@ -569,7 +594,7 @@ JL_DLLEXPORT jl_value_t *jl_arrayref(jl_array_t *a, size_t i)
 JL_DLLEXPORT int jl_array_isassigned(jl_array_t *a, size_t i)
 {
     if (a->flags.ptrarray) {
-        return ((jl_value_t**)jl_array_data(a))[i] != NULL;
+        return jl_atomic_load_relaxed(((jl_value_t**)jl_array_data(a)) + i) != NULL;
     }
     else if (a->flags.hasptr) {
          jl_datatype_t *eltype = (jl_datatype_t*)jl_tparam0(jl_typeof(a));
@@ -600,12 +625,18 @@ JL_DLLEXPORT void jl_arrayset(jl_array_t *a JL_ROOTING_ARGUMENT, jl_value_t *rhs
             if (jl_is_datatype_singleton((jl_datatype_t*)jl_typeof(rhs)))
                 return;
         }
-        jl_assign_bits(&((char*)a->data)[i * a->elsize], rhs);
+        if (a->flags.hasptr) {
+            jl_fence_release();
+            memmove_refs((void**)&((char*)a->data)[i * a->elsize], (void**)rhs, a->elsize / sizeof(void*));
+        }
+        else {
+            jl_assign_bits(&((char*)a->data)[i * a->elsize], rhs);
+        }
         if (a->flags.hasptr)
             jl_gc_multi_wb(jl_array_owner(a), rhs);
     }
     else {
-        ((jl_value_t**)a->data)[i] = rhs;
+        jl_atomic_store_release(((jl_value_t**)a->data) + i, rhs);
         jl_gc_wb(jl_array_owner(a), rhs);
     }
 }
@@ -615,7 +646,7 @@ JL_DLLEXPORT void jl_arrayunset(jl_array_t *a, size_t i)
     if (i >= jl_array_len(a))
         jl_bounds_error_int((jl_value_t*)a, i + 1);
     if (a->flags.ptrarray)
-        ((jl_value_t**)a->data)[i] = NULL;
+        jl_atomic_store_release(((jl_value_t**)a->data) + i, NULL);
     else if (a->flags.hasptr) {
         size_t elsize = a->elsize;
         jl_assume(elsize >= sizeof(void*) && elsize % sizeof(void*) == 0);
@@ -762,7 +793,7 @@ STATIC_INLINE void jl_array_grow_at_beg(jl_array_t *a, size_t idx, size_t inc,
         if (isbitsunion) newtypetagdata = typetagdata - inc;
         if (idx > 0) {
             // inserting new elements after 1st element
-            memmove(newdata, data, idx * elsz);
+            memmove_safe(a->flags.hasptr, newdata, data, idx * elsz);
             if (isbitsunion) {
                 memmove(newtypetagdata, typetagdata, idx);
                 memset(newtypetagdata + idx, 0, inc);
@@ -796,11 +827,11 @@ STATIC_INLINE void jl_array_grow_at_beg(jl_array_t *a, size_t idx, size_t inc,
             // We could use memcpy if resizing allocates a new buffer,
             // hopefully it's not a particularly important optimization.
             if (idx > 0 && newdata < data) {
-                memmove(newdata, data, nb1);
+                memmove_safe(a->flags.hasptr, newdata, data, nb1);
             }
-            memmove(newdata + nbinc + nb1, data + nb1, n * elsz - nb1);
+            memmove_safe(a->flags.hasptr, newdata + nbinc + nb1, data + nb1, n * elsz - nb1);
             if (idx > 0 && newdata > data) {
-                memmove(newdata, data, nb1);
+                memmove_safe(a->flags.hasptr, newdata, data, nb1);
             }
             a->offset = newoffset;
         }
@@ -810,16 +841,16 @@ STATIC_INLINE void jl_array_grow_at_beg(jl_array_t *a, size_t idx, size_t inc,
             newdata = data - oldoffsnb + a->offset * elsz;
             if (isbitsunion) newtypetagdata = newdata + (a->maxsize - a->offset) * elsz + a->offset;
             if (idx > 0 && newdata < data) {
-                memmove(newdata, data, nb1);
+                memmove_safe(a->flags.hasptr, newdata, data, nb1);
                 if (isbitsunion) {
                     memmove(newtypetagdata, typetagdata, idx);
                     memset(newtypetagdata + idx, 0, inc);
                 }
             }
-            memmove(newdata + nbinc + nb1, data + nb1, n * elsz - nb1);
+            memmove_safe(a->flags.hasptr, newdata + nbinc + nb1, data + nb1, n * elsz - nb1);
             if (isbitsunion) memmove(newtypetagdata + idx + inc, typetagdata + idx, n - idx);
             if (idx > 0 && newdata > data) {
-                memmove(newdata, data, nb1);
+                memmove_safe(a->flags.hasptr, newdata, data, nb1);
                 if (isbitsunion) {
                     memmove(newtypetagdata, typetagdata, idx);
                     memset(newtypetagdata + idx, 0, inc);
@@ -891,7 +922,7 @@ STATIC_INLINE void jl_array_grow_at_end(jl_array_t *a, size_t idx,
                 memmove(newtypetagdata, typetagdata, idx);
                 memset(newtypetagdata + idx, 0, inc);
             }
-            if (has_gap) memmove(newdata + nb1 + nbinc, newdata + nb1, n * elsz - nb1);
+            if (has_gap) memmove_safe(a->flags.hasptr, newdata + nb1 + nbinc, newdata + nb1, n * elsz - nb1);
         }
         a->data = data = newdata;
     }
@@ -901,7 +932,7 @@ STATIC_INLINE void jl_array_grow_at_end(jl_array_t *a, size_t idx,
             memset(typetagdata + idx, 0, inc);
         }
         size_t nb1 = idx * elsz;
-        memmove(data + nb1 + inc * elsz, data + nb1, n * elsz - nb1);
+        memmove_safe(a->flags.hasptr, data + nb1 + inc * elsz, data + nb1, n * elsz - nb1);
     }
     else {
         // there was enough room for requested growth already in a->maxsize
@@ -1036,12 +1067,12 @@ STATIC_INLINE void jl_array_del_at_beg(jl_array_t *a, size_t idx, size_t dec,
         if (elsz == 1 && !isbitsunion)
             nbtotal++;
         if (idx > 0) {
-            memmove(newdata, olddata, nb1);
+            memmove_safe(a->flags.hasptr, newdata, olddata, nb1);
             if (isbitsunion) memmove(newtypetagdata, typetagdata, idx);
         }
         // Move the rest of the data if the offset changed
         if (newoffs != offset) {
-            memmove(newdata + nb1, olddata + nb1 + nbdec, nbtotal - nb1);
+            memmove_safe(a->flags.hasptr, newdata + nb1, olddata + nb1 + nbdec, nbtotal - nb1);
             if (isbitsunion) memmove(newtypetagdata + idx, typetagdata + idx + dec, n - idx);
         }
         a->data = newdata;
@@ -1063,7 +1094,7 @@ STATIC_INLINE void jl_array_del_at_end(jl_array_t *a, size_t idx, size_t dec,
     int isbitsunion = jl_array_isbitsunion(a);
     size_t last = idx + dec;
     if (n > last) {
-        memmove(data + idx * elsz, data + last * elsz, (n - last) * elsz);
+        memmove_safe(a->flags.hasptr, data + idx * elsz, data + last * elsz, (n - last) * elsz);
         if (isbitsunion) {
             char *typetagdata = jl_array_typetagdata(a);
             memmove(typetagdata + idx, typetagdata + last, n - last);
@@ -1161,14 +1192,14 @@ JL_DLLEXPORT jl_array_t *jl_array_copy(jl_array_t *ary)
 }
 
 // Copy element by element until we hit a young object, at which point
-// we can continue using `memmove`.
+// we can finish by using `memmove`.
 static NOINLINE ssize_t jl_array_ptr_copy_forward(jl_value_t *owner,
                                                   void **src_p, void **dest_p,
                                                   ssize_t n)
 {
     for (ssize_t i = 0; i < n; i++) {
-        void *val = src_p[i];
-        dest_p[i] = val;
+        void *val = jl_atomic_load_relaxed(src_p + i);
+        jl_atomic_store_relaxed(dest_p + i, val);
         // `val` is young or old-unmarked
         if (val && !(jl_astaggedvalue(val)->bits.gc & GC_MARKED)) {
             jl_gc_queue_root(owner);
@@ -1183,8 +1214,8 @@ static NOINLINE ssize_t jl_array_ptr_copy_backward(jl_value_t *owner,
                                                    ssize_t n)
 {
     for (ssize_t i = 0; i < n; i++) {
-        void *val = src_p[n - i - 1];
-        dest_p[n - i - 1] = val;
+        void *val = jl_atomic_load_relaxed(src_p + n - i - 1);
+        jl_atomic_store_relaxed(dest_p + n - i - 1, val);
         // `val` is young or old-unmarked
         if (val && !(jl_astaggedvalue(val)->bits.gc & GC_MARKED)) {
             jl_gc_queue_root(owner);
@@ -1200,6 +1231,7 @@ JL_DLLEXPORT void jl_array_ptr_copy(jl_array_t *dest, void **dest_p,
 {
     assert(dest->flags.ptrarray && src->flags.ptrarray);
     jl_value_t *owner = jl_array_owner(dest);
+    jl_fence_release(); // ensure contents of src are visible on other processors
     // Destination is old and doesn't refer to any young object
     if (__unlikely(jl_astaggedvalue(owner)->bits.gc == GC_OLD_MARKED)) {
         jl_value_t *src_owner = jl_array_owner(src);
@@ -1218,7 +1250,7 @@ JL_DLLEXPORT void jl_array_ptr_copy(jl_array_t *dest, void **dest_p,
             n -= done;
         }
     }
-    memmove(dest_p, src_p, n * sizeof(void*));
+    memmove_refs(dest_p, src_p, n);
 }
 
 JL_DLLEXPORT void jl_array_ptr_1d_push(jl_array_t *a, jl_value_t *item)

src/atomics.h

@@ -44,6 +44,8 @@
  *        specified.
  */
 #if defined(__GNUC__)
+#  define jl_fence() __atomic_thread_fence(__ATOMIC_SEQ_CST)
+#  define jl_fence_release() __atomic_thread_fence(__ATOMIC_RELEASE)
 #  define jl_signal_fence() __atomic_signal_fence(__ATOMIC_SEQ_CST)
 #  define jl_atomic_fetch_add_relaxed(obj, arg)         \
     __atomic_fetch_add(obj, arg, __ATOMIC_RELAXED)
@@ -96,6 +98,9 @@
 #  define jl_atomic_load_relaxed(obj)           \
     __atomic_load_n(obj, __ATOMIC_RELAXED)
 #elif defined(_COMPILER_MICROSOFT_)
+// TODO: these only define compiler barriers, and aren't correct outside of x86
+#  define jl_fence() _ReadWriteBarrier()
+#  define jl_fence_release() _WriteBarrier()
 #  define jl_signal_fence() _ReadWriteBarrier()
 
 // add

src/cgutils.cpp

@@ -1383,26 +1383,34 @@ static void typed_store(jl_codectx_t &ctx,
     if (type_is_ghost(elty))
         return;
     Value *r;
-    if (!isboxed) {
+    if (!isboxed)
         r = emit_unbox(ctx, elty, rhs, jltype);
-        if (parent != NULL)
-            emit_write_multibarrier(ctx, parent, r);
-    }
-    else {
+    else
         r = maybe_decay_untracked(boxed(ctx, rhs));
-        if (parent != NULL)
-            emit_write_barrier(ctx, parent, r);
-    }
     Type *ptrty = PointerType::get(elty, ptr->getType()->getPointerAddressSpace());
     if (ptr->getType() != ptrty)
         ptr = ctx.builder.CreateBitCast(ptr, ptrty);
     if (idx_0based)
         ptr = ctx.builder.CreateInBoundsGEP(r->getType(), ptr, idx_0based);
-    Instruction *store = ctx.builder.CreateAlignedStore(r, ptr, isboxed || alignment ? alignment : julia_alignment(jltype));
+    if (!isboxed && CountTrackedPointers(elty).count)
+        ctx.builder.CreateFence(AtomicOrdering::Release);
+    if (isboxed)
+        alignment = sizeof(void*);
+    else if (!alignment)
+        alignment = julia_alignment(jltype);
+    StoreInst *store = ctx.builder.CreateAlignedStore(r, ptr, alignment);
+    if (isboxed)
+        store->setOrdering(AtomicOrdering::Release);
     if (aliasscope)
         store->setMetadata("noalias", aliasscope);
     if (tbaa)
         tbaa_decorate(tbaa, store);
+    if (parent != NULL) {
+        if (!isboxed)
+            emit_write_multibarrier(ctx, parent, r);
+        else
+            emit_write_barrier(ctx, parent, r);
+    }
 }
 
 // --- convert boolean value to julia ---
@@ -2617,8 +2625,10 @@ static void emit_setfield(jl_codectx_t &ctx,
         jl_value_t *jfty = jl_svecref(sty->types, idx0);
         if (jl_field_isptr(sty, idx0)) {
             Value *r = maybe_decay_untracked(boxed(ctx, rhs)); // don't need a temporary gcroot since it'll be rooted by strct
-            tbaa_decorate(strct.tbaa, ctx.builder.CreateStore(r,
-                emit_bitcast(ctx, addr, T_pprjlvalue)));
+            cast<StoreInst>(tbaa_decorate(strct.tbaa, ctx.builder.CreateAlignedStore(r,
+                        emit_bitcast(ctx, addr, T_pprjlvalue),
+                        sizeof(jl_value_t*))))
+                    ->setOrdering(AtomicOrdering::Release);
             if (wb && strct.isboxed)
                 emit_write_barrier(ctx, boxed(ctx, strct), r);
         }
@@ -2712,7 +2722,9 @@ static jl_cgval_t emit_new_struct(jl_codectx_t &ctx, jl_value_t *ty, size_t narg
                 if (jl_field_isptr(sty, i)) {
                     fval = boxed(ctx, fval_info);
                     if (!init_as_value)
-                        tbaa_decorate(tbaa_stack, ctx.builder.CreateAlignedStore(fval, dest, jl_field_align(sty, i)));
+                        cast<StoreInst>(tbaa_decorate(tbaa_stack,
+                                    ctx.builder.CreateAlignedStore(fval, dest, jl_field_align(sty, i))))
+                                ->setOrdering(AtomicOrdering::Release);
                 }
                 else if (jl_is_uniontype(jtype)) {
                     // compute tindex from rhs

src/datatype.c

@@ -1011,7 +1011,7 @@ JL_DLLEXPORT jl_value_t *jl_get_nth_field(jl_value_t *v, size_t i)
     assert(i < jl_datatype_nfields(st));
     size_t offs = jl_field_offset(st, i);
     if (jl_field_isptr(st, i)) {
-        return *(jl_value_t**)((char*)v + offs);
+        return jl_atomic_load_relaxed((jl_value_t**)((char*)v + offs));
     }
     jl_value_t *ty = jl_field_type(st, i);
     if (jl_is_uniontype(ty)) {
@@ -1029,7 +1029,7 @@ JL_DLLEXPORT jl_value_t *jl_get_nth_field_noalloc(jl_value_t *v JL_PROPAGATES_RO
     assert(i < jl_datatype_nfields(st));
     size_t offs = jl_field_offset(st,i);
     assert(jl_field_isptr(st,i));
-    return *(jl_value_t**)((char*)v + offs);
+    return jl_atomic_load_relaxed((jl_value_t**)((char*)v + offs));
 }
 
 JL_DLLEXPORT jl_value_t *jl_get_nth_field_checked(jl_value_t *v, size_t i)
@@ -1039,7 +1039,7 @@ JL_DLLEXPORT jl_value_t *jl_get_nth_field_checked(jl_value_t *v, size_t i)
         jl_bounds_error_int(v, i + 1);
     size_t offs = jl_field_offset(st, i);
     if (jl_field_isptr(st, i)) {
-        jl_value_t *fval = *(jl_value_t**)((char*)v + offs);
+        jl_value_t *fval = jl_atomic_load_relaxed((jl_value_t**)((char*)v + offs));
         if (__unlikely(fval == NULL))
             jl_throw(jl_undefref_exception);
         return fval;
@@ -1057,11 +1057,11 @@ JL_DLLEXPORT jl_value_t *jl_get_nth_field_checked(jl_value_t *v, size_t i)
 
 void set_nth_field(jl_datatype_t *st, void *v, size_t i, jl_value_t *rhs) JL_NOTSAFEPOINT
 {
+    assert(rhs != NULL);
     size_t offs = jl_field_offset(st, i);
     if (jl_field_isptr(st, i)) {
-        *(jl_value_t**)((char*)v + offs) = rhs;
-        if (rhs != NULL)
-            jl_gc_wb(v, rhs);
+        jl_atomic_store_release((jl_value_t**)((char*)v + offs), rhs);
+        jl_gc_wb(v, rhs);
     }
     else {
         jl_value_t *ty = jl_field_type_concrete(st, i);
@@ -1085,7 +1085,8 @@ JL_DLLEXPORT int jl_field_isdefined(jl_value_t *v, size_t i)
     size_t offs = jl_field_offset(st, i);
     char *fld = (char*)v + offs;
     if (jl_field_isptr(st, i)) {
-        return *(jl_value_t**)fld != NULL;
+        jl_value_t *fval = jl_atomic_load_relaxed((jl_value_t**)fld);
+        return fval != NULL;
     }
     jl_datatype_t *ft = (jl_datatype_t*)jl_field_type(st, i);
     if (jl_is_datatype(ft) && ft->layout->first_ptr >= 0) {

src/dump.c

@@ -1923,7 +1923,7 @@ static void jl_insert_backedges(jl_array_t *list, jl_array_t *targets)
             while (codeinst) {
                 if (codeinst->min_world > 0)
                     codeinst->max_world = ~(size_t)0;
-                codeinst = codeinst->next;
+                codeinst = jl_atomic_load_relaxed(&codeinst->next);
             }
         }
         else {