Add a flat address space to librustc_codegen_llvm

src/librustc_codegen_llvm/abi.rs

@@ -11,7 +11,7 @@ use rustc_target::abi::call::ArgType;
 
 use rustc_codegen_ssa::traits::*;
 
-use rustc_target::abi::{HasDataLayout, LayoutOf};
+use rustc_target::abi::LayoutOf;
 use rustc::ty::{Ty};
 use rustc::ty::layout::{self};
 
@@ -302,7 +302,6 @@ impl ArgTypeMethods<'tcx> for Builder<'a, 'll, 'tcx> {
 
 pub trait FnTypeLlvmExt<'tcx> {
     fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type;
-    fn ptr_to_llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type;
     fn llvm_cconv(&self) -> llvm::CallConv;
     fn apply_attrs_llfn(&self, cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value);
     fn apply_attrs_callsite(&self, bx: &mut Builder<'a, 'll, 'tcx>, callsite: &'ll Value);
@@ -323,11 +322,11 @@ impl<'tcx> FnTypeLlvmExt<'tcx> for FnType<'tcx, Ty<'tcx>> {
             PassMode::Ignore(IgnoreMode::CVarArgs) =>
                 bug!("`va_list` should never be a return type"),
             PassMode::Direct(_) | PassMode::Pair(..) => {
-                self.ret.layout.immediate_llvm_type(cx)
+                self.ret.layout.immediate_llvm_type(cx).copy_addr_space(cx.flat_addr_space())
             }
             PassMode::Cast(cast) => cast.llvm_type(cx),
             PassMode::Indirect(..) => {
-                llargument_tys.push(cx.type_ptr_to(self.ret.memory_ty(cx)));
+                llargument_tys.push(cx.type_ptr_to_alloca(self.ret.memory_ty(cx)));
                 cx.type_void()
             }
         };
@@ -340,7 +339,8 @@ impl<'tcx> FnTypeLlvmExt<'tcx> for FnType<'tcx, Ty<'tcx>> {
 
             let llarg_ty = match arg.mode {
                 PassMode::Ignore(_) => continue,
-                PassMode::Direct(_) => arg.layout.immediate_llvm_type(cx),
+                PassMode::Direct(_) => arg.layout.immediate_llvm_type(cx)
+                    .copy_addr_space(cx.flat_addr_space()),
                 PassMode::Pair(..) => {
                     llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(cx, 0, true));
                     llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(cx, 1, true));
@@ -354,7 +354,7 @@ impl<'tcx> FnTypeLlvmExt<'tcx> for FnType<'tcx, Ty<'tcx>> {
                     continue;
                 }
                 PassMode::Cast(cast) => cast.llvm_type(cx),
-                PassMode::Indirect(_, None) => cx.type_ptr_to(arg.memory_ty(cx)),
+                PassMode::Indirect(_, None) => cx.type_ptr_to_alloca(arg.memory_ty(cx)),
             };
             llargument_tys.push(llarg_ty);
         }
@@ -366,13 +366,6 @@ impl<'tcx> FnTypeLlvmExt<'tcx> for FnType<'tcx, Ty<'tcx>> {
         }
     }
 
-    fn ptr_to_llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type {
-        unsafe {
-            llvm::LLVMPointerType(self.llvm_type(cx),
-                                  cx.data_layout().instruction_address_space as c_uint)
-        }
-    }
-
     fn llvm_cconv(&self) -> llvm::CallConv {
         match self.conv {
             Conv::C => llvm::CCallConv,

src/librustc_codegen_llvm/builder.rs

@@ -1,6 +1,6 @@
 use crate::llvm::{AtomicRmwBinOp, AtomicOrdering, SynchronizationScope};
 use crate::llvm::{self, False, BasicBlock};
-use crate::common::Funclet;
+use crate::common::{Funclet, val_addr_space, val_addr_space_opt};
 use crate::context::CodegenCx;
 use crate::type_::Type;
 use crate::type_of::LayoutLlvmExt;
@@ -18,7 +18,7 @@ use rustc_codegen_ssa::traits::*;
 use rustc_codegen_ssa::base::to_immediate;
 use rustc_codegen_ssa::mir::operand::{OperandValue, OperandRef};
 use rustc_codegen_ssa::mir::place::PlaceRef;
-use rustc_target::spec::{HasTargetSpec, Target};
+use rustc_target::spec::{HasTargetSpec, Target, AddrSpaceIdx};
 use std::borrow::Cow;
 use std::ffi::CStr;
 use std::ops::{Deref, Range};
@@ -546,8 +546,10 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
     ) -> Self {
         let zero = self.const_usize(0);
         let count = self.const_usize(count);
-        let start = dest.project_index(&mut self, zero).llval;
-        let end = dest.project_index(&mut self, count).llval;
+        let before_start = dest.project_index(&mut self, zero).llval;
+        let before_end = dest.project_index(&mut self, count).llval;
+        let start = self.flat_addr_cast(before_start);
+        let end = self.flat_addr_cast(before_end);
 
         let mut header_bx = self.build_sibling_block("repeat_loop_header");
         let mut body_bx = self.build_sibling_block("repeat_loop_body");
@@ -724,23 +726,56 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
     }
 
     fn ptrtoint(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
+        let val_casted = self.flat_addr_cast(val);
         unsafe {
-            llvm::LLVMBuildPtrToInt(self.llbuilder, val, dest_ty, UNNAMED)
+            llvm::LLVMBuildPtrToInt(self.llbuilder, val_casted, dest_ty, UNNAMED)
         }
     }
 
     fn inttoptr(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
+        let dest_ty_copied = dest_ty.copy_addr_space(self.cx().flat_addr_space());
         unsafe {
-            llvm::LLVMBuildIntToPtr(self.llbuilder, val, dest_ty, UNNAMED)
+            llvm::LLVMBuildIntToPtr(self.llbuilder, val, dest_ty_copied, UNNAMED)
         }
     }
 
     fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
         unsafe {
-            llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, UNNAMED)
+            llvm::LLVMBuildBitCast(self.llbuilder, val,
+                                   dest_ty.copy_addr_space(val_addr_space(val)), UNNAMED)
         }
     }
 
+    fn as_ptr_cast(
+        &mut self, val: &'ll Value,
+        addr_space: AddrSpaceIdx, dest_ty: &'ll Type
+    ) -> &'ll Value {
+        let val_addr = self.addrspace_cast(val, addr_space);
+        self.pointercast(val_addr, dest_ty.copy_addr_space(addr_space))
+    }
+
+    fn flat_as_ptr_cast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
+        self.as_ptr_cast(val, self.cx().flat_addr_space(), dest_ty)
+    }
+
+    fn addrspace_cast(&mut self, val: &'ll Value, dest: AddrSpaceIdx) -> &'ll Value {
+        let src_ty = self.cx().val_ty(val);
+
+        if src_ty.is_ptr() && src_ty.address_space() != dest {
+            let dest_ty = src_ty.copy_addr_space(dest);
+            self.cx().check_addr_space_cast(val, dest_ty);
+            unsafe {
+                llvm::LLVMBuildAddrSpaceCast(self.llbuilder, val,
+                                             dest_ty, UNNAMED)
+            }
+        } else {
+            val
+        }
+    }
+
+    fn flat_addr_cast(&mut self, val: &'ll Value) -> &'ll Value {
+        self.addrspace_cast(val, self.cx().flat_addr_space())
+    }
 
     fn intcast(&mut self, val: &'ll Value, dest_ty: &'ll Type, is_signed: bool) -> &'ll Value {
         unsafe {
@@ -750,15 +785,26 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
 
     fn pointercast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value {
         unsafe {
-            llvm::LLVMBuildPointerCast(self.llbuilder, val, dest_ty, UNNAMED)
+            llvm::LLVMBuildPointerCast(self.llbuilder, val,
+                                       dest_ty.copy_addr_space(val_addr_space(val)), UNNAMED)
         }
     }
 
     /* Comparisons */
     fn icmp(&mut self, op: IntPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value {
-        let op = llvm::IntPredicate::from_generic(op);
+        match (val_addr_space_opt(lhs), val_addr_space_opt(rhs)) {
+            (Some(l), Some(r)) if l == r => {},
+            (Some(l), Some(r)) if l != r => {
+                bug!("Tried to compare pointers of different address spaces: lhs {:?} rhs {:?}",
+                     lhs, rhs);
+            },
+            _ => {},
+        }
+
         unsafe {
-            llvm::LLVMBuildICmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED)
+            llvm::LLVMBuildICmp(self.llbuilder,
+                                llvm::IntPredicate::from_generic(op) as c_uint,
+                                lhs, rhs, UNNAMED)
         }
     }
 
@@ -818,7 +864,8 @@ impl BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
         flags: MemFlags,
     ) {
         let ptr_width = &self.sess().target.target.target_pointer_width;
-        let intrinsic_key = format!("llvm.memset.p0i8.i{}", ptr_width);
+        let intrinsic_key = format!("llvm.memset.p{}i8.i{}",
+                                    self.val_ty(ptr).address_space(), ptr_width);
         let llintrinsicfn = self.get_intrinsic(&intrinsic_key);
         let ptr = self.pointercast(ptr, self.type_i8p());
         let align = self.const_u32(align.bytes() as u32);
@@ -1231,7 +1278,8 @@ impl Builder<'a, 'll, 'tcx> {
     fn check_store(&mut self, val: &'ll Value, ptr: &'ll Value) -> &'ll Value {
         let dest_ptr_ty = self.cx.val_ty(ptr);
         let stored_ty = self.cx.val_ty(val);
-        let stored_ptr_ty = self.cx.type_ptr_to(stored_ty);
+        let stored_ptr_ty = self.cx.type_as_ptr_to(stored_ty,
+                                                   dest_ptr_ty.address_space());
 
         assert_eq!(self.cx.type_kind(dest_ptr_ty), TypeKind::Pointer);
 
@@ -1277,7 +1325,18 @@ impl Builder<'a, 'll, 'tcx> {
                     debug!("type mismatch in function call of {:?}. \
                             Expected {:?} for param {}, got {:?}; injecting bitcast",
                            llfn, expected_ty, i, actual_ty);
-                    self.bitcast(actual_val, expected_ty)
+                    if expected_ty.is_ptr() && actual_ty.is_ptr() {
+                        let actual_val_addr = self.addrspace_cast(actual_val,
+                                                                  expected_ty.address_space());
+                        self.pointercast(actual_val_addr, expected_ty)
+                    } else {
+                        let actual_val_addr = if actual_ty.is_ptr() {
+                            self.flat_addr_cast(actual_val)
+                        } else {
+                            actual_val
+                        };
+                        self.bitcast(actual_val_addr, expected_ty)
+                    }
                 } else {
                     actual_val
                 }
@@ -1303,7 +1362,15 @@ impl Builder<'a, 'll, 'tcx> {
             return;
         }
 
-        let lifetime_intrinsic = self.cx.get_intrinsic(intrinsic);
+        let addr_space = self.cx.val_ty(ptr).address_space();
+        // Old LLVMs don't have the address space specific intrinsics.
+        // So as a semi-crude workaround, don't specialize if in the
+        // default address space.
+        let lifetime_intrinsic = if let AddrSpaceIdx(0) = addr_space {
+            self.cx.get_intrinsic(intrinsic)
+        } else {
+            self.cx.get_intrinsic(&format!("{}.p{}i8", intrinsic, addr_space))
+        };
 
         let ptr = self.pointercast(ptr, self.cx.type_i8p());
         self.call(lifetime_intrinsic, &[self.cx.const_u64(size), ptr], None);

src/librustc_codegen_llvm/common.rs

@@ -13,7 +13,9 @@ use rustc_codegen_ssa::traits::*;
 use crate::consts::const_alloc_to_llvm;
 use rustc::ty::layout::{HasDataLayout, LayoutOf, self, TyLayout, Size, Align};
 use rustc::mir::interpret::{Scalar, GlobalAlloc, Allocation};
+use rustc_codegen_ssa::common::TypeKind;
 use rustc_codegen_ssa::mir::place::PlaceRef;
+use rustc_target::spec::AddrSpaceIdx;
 
 use libc::{c_uint, c_char};
 
@@ -135,9 +137,8 @@ impl CodegenCx<'ll, 'tcx> {
                                                     s.len() as c_uint,
                                                     !null_terminated as Bool);
             let sym = self.generate_local_symbol_name("str");
-            let g = self.define_global(&sym[..], self.val_ty(sc)).unwrap_or_else(||{
-                bug!("symbol `{}` is already defined", sym);
-            });
+            let g = self.define_global(&sym[..], self.val_ty(sc), self.const_addr_space())
+                .unwrap_or_else(||bug!("symbol `{}` is already defined", sym));
             llvm::LLVMSetInitializer(g, sc);
             llvm::LLVMSetGlobalConstant(g, True);
             llvm::LLVMRustSetLinkage(g, llvm::Linkage::InternalLinkage);
@@ -271,6 +272,10 @@ impl ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         }
     }
 
+    fn const_as_cast(&self, val: &'ll Value, addr_space: AddrSpaceIdx) -> &'ll Value {
+        self.const_addrcast(val, addr_space)
+    }
+
     fn scalar_to_backend(
         &self,
         cv: Scalar,
@@ -285,11 +290,17 @@ impl ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
             },
             Scalar::Raw { data, size } => {
                 assert_eq!(size as u64, layout.value.size(self).bytes());
-                let llval = self.const_uint_big(self.type_ix(bitsize), data);
-                if layout.value == layout::Pointer {
-                    unsafe { llvm::LLVMConstIntToPtr(llval, llty) }
+                let llconstval = self.const_uint_big(self.type_ix(bitsize), data);
+                let flat_llty = llty.copy_addr_space(self.flat_addr_space());
+                let llval = if layout.value == layout::Pointer {
+                    unsafe { llvm::LLVMConstIntToPtr(llconstval, flat_llty) }
                 } else {
-                    self.const_bitcast(llval, llty)
+                    self.const_bitcast(llconstval, flat_llty)
+                };
+                if llty.is_ptr() {
+                    self.const_as_cast(llval, llty.address_space())
+                } else {
+                    llval
                 }
             },
             Scalar::Ptr(ptr) => {
@@ -298,7 +309,8 @@ impl ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
                     Some(GlobalAlloc::Memory(alloc)) => {
                         let init = const_alloc_to_llvm(self, alloc);
                         if alloc.mutability == Mutability::Mutable {
-                            self.static_addr_of_mut(init, alloc.align, None)
+                            self.static_addr_of_mut(init, alloc.align, None,
+                                                    self.mutable_addr_space())
                         } else {
                             self.static_addr_of(init, alloc.align, None)
                         }
@@ -312,11 +324,12 @@ impl ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> {
                     }
                     None => bug!("missing allocation {:?}", ptr.alloc_id),
                 };
-                let llval = unsafe { llvm::LLVMConstInBoundsGEP(
+                let llval_ref = unsafe { llvm::LLVMConstInBoundsGEP(
                     self.const_bitcast(base_addr, self.type_i8p()),
                     &self.const_usize(ptr.offset.bytes()),
                     1,
                 ) };
+                let llval = self.const_flat_as_cast(llval_ref);
                 if layout.value != layout::Pointer {
                     unsafe { llvm::LLVMConstPtrToInt(llval, llty) }
                 } else {
@@ -356,6 +369,19 @@ pub fn val_ty(v: &'ll Value) -> &'ll Type {
     }
 }
 
+pub fn val_addr_space_opt(v: &'ll Value) -> Option<AddrSpaceIdx> {
+    let ty = val_ty(v);
+    if ty.kind() == TypeKind::Pointer {
+        Some(ty.address_space())
+    } else {
+        None
+    }
+}
+
+pub fn val_addr_space(v: &'ll Value) -> AddrSpaceIdx {
+    val_addr_space_opt(v).unwrap_or_default()
+}
+
 pub fn bytes_in_context(llcx: &'ll llvm::Context, bytes: &[u8]) -> &'ll Value {
     unsafe {
         let ptr = bytes.as_ptr() as *const c_char;