diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs
index ef72e6efb946b..0d2642299d825 100644
--- a/compiler/rustc_codegen_cranelift/src/constant.rs
+++ b/compiler/rustc_codegen_cranelift/src/constant.rs
@@ -251,6 +251,11 @@ pub(crate) fn codegen_const_value<'tcx>(
.iconst(fx.pointer_type, i64::try_from(end.checked_sub(start).unwrap()).unwrap());
CValue::by_val_pair(ptr, len, layout)
}
+ ConstValue::CustomSlice { data, length } => {
+ let ptr = pointer_for_allocation(fx, data).get_addr(fx);
+ let len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(length).unwrap());
+ CValue::by_val_pair(ptr, len, layout)
+ }
}
}
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
index 6937e658ed5ee..bad2bca244c54 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
@@ -720,7 +720,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
dest.write_cvalue(fx, val);
};
- pref_align_of | needs_drop | type_id | type_name | variant_count, () {
+ pref_align_of | needs_drop | type_id | type_name | variant_count | validity_invariants_of, () {
let const_val =
fx.tcx.const_eval_instance(ParamEnv::reveal_all(), instance, None).unwrap();
let val = crate::constant::codegen_const_value(
diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
index 0ed4c3f1d9430..fa567d139497b 100644
--- a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
@@ -106,7 +106,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
| sym::needs_drop
| sym::type_id
| sym::type_name
- | sym::variant_count => {
+ | sym::variant_count
+ | sym::validity_invariants_of => {
let value = bx
.tcx()
.const_eval_instance(ty::ParamEnv::reveal_all(), instance, None)
diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs
index 08be4c0a7b622..ddb88d97b7f9b 100644
--- a/compiler/rustc_codegen_ssa/src/mir/operand.rs
+++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs
@@ -100,6 +100,22 @@ impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
let b_llval = bx.const_usize((end - start) as u64);
OperandValue::Pair(a_llval, b_llval)
}
+ ConstValue::CustomSlice { data, length } => {
+ let Abi::ScalarPair(a_scalar, _) = layout.abi else {
+ bug!("from_const: invalid ScalarPair layout: {:#?}", layout);
+ };
+ let a = Scalar::from_pointer(
+ Pointer::new(bx.tcx().create_memory_alloc(data), Size::ZERO),
+ &bx.tcx(),
+ );
+ let a_llval = bx.scalar_to_backend(
+ a,
+ a_scalar,
+ bx.scalar_pair_element_backend_type(layout, 0, true),
+ );
+ let b_llval = bx.const_usize(length as u64);
+ OperandValue::Pair(a_llval, b_llval)
+ }
ConstValue::ByRef { alloc, offset } => {
return bx.load_operand(bx.from_const_alloc(layout, alloc, offset));
}
diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs
index e51c51cf45e5e..95d3951160e44 100644
--- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs
+++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs
@@ -5,6 +5,7 @@
use std::convert::TryFrom;
use rustc_hir::def_id::DefId;
+use rustc_hir::lang_items::LangItem;
use rustc_middle::mir::{
self,
interpret::{ConstValue, GlobalId, InterpResult, Scalar},
@@ -16,6 +17,7 @@ use rustc_middle::ty::subst::SubstsRef;
use rustc_middle::ty::{Ty, TyCtxt};
use rustc_span::symbol::{sym, Symbol};
use rustc_target::abi::{Abi, Align, InitKind, Primitive, Size};
+use rustc_target::spec::SanitizerSet;
use super::{
util::ensure_monomorphic_enough, CheckInAllocMsg, ImmTy, InterpCx, Machine, OpTy, PlaceTy,
@@ -24,6 +26,7 @@ use super::{
mod caller_location;
mod type_name;
+mod validity_invariants_of;
fn numeric_intrinsic<Tag>(name: Symbol, bits: u128, kind: Primitive) -> Scalar<Tag> {
let size = match kind {
@@ -103,6 +106,23 @@ pub(crate) fn eval_nullary_intrinsic<'tcx>(
| ty::Tuple(_)
| ty::Error(_) => ConstValue::from_machine_usize(0u64, &tcx),
},
+ sym::validity_invariants_of => {
+ let msan = tcx.sess.opts.debugging_opts.sanitizer.contains(SanitizerSet::MEMORY);
+ let disable = tcx.sess.opts.debugging_opts.no_validity_invariant_checks;
+
+ let strictness = if disable {
+ validity_invariants_of::InvariantStrictness::Disable
+ } else if msan {
+ validity_invariants_of::InvariantStrictness::All
+ } else {
+ validity_invariants_of::InvariantStrictness::Normal
+ };
+
+ ensure_monomorphic_enough(tcx, tp_ty)?;
+ let (data, length) =
+ validity_invariants_of::alloc_validity_invariants_of(tcx, tp_ty, strictness);
+ ConstValue::CustomSlice { data, length }
+ }
other => bug!("`{}` is not a zero arg intrinsic", other),
})
}
@@ -162,6 +182,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
| sym::needs_drop
| sym::type_id
| sym::type_name
+ | sym::validity_invariants_of
| sym::variant_count => {
let gid = GlobalId { instance, promoted: None };
let ty = match intrinsic_name {
@@ -169,6 +190,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
sym::needs_drop => self.tcx.types.bool,
sym::type_id => self.tcx.types.u64,
sym::type_name => self.tcx.mk_static_str(),
+ sym::validity_invariants_of => {
+ let item = self.tcx.require_lang_item(LangItem::ValidityInvariant, None);
+ let ty = self.tcx.type_of(item);
+ self.tcx.mk_imm_ref(self.tcx.lifetimes.re_static, self.tcx.mk_slice(ty))
+ }
_ => bug!(),
};
let val =
diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics/validity_invariants_of.rs b/compiler/rustc_const_eval/src/interpret/intrinsics/validity_invariants_of.rs
new file mode 100644
index 0000000000000..4ff1629798fc1
--- /dev/null
+++ b/compiler/rustc_const_eval/src/interpret/intrinsics/validity_invariants_of.rs
@@ -0,0 +1,171 @@
+use rustc_data_structures::fx::FxHashMap;
+use rustc_hir::lang_items::LangItem;
+use rustc_middle::mir::interpret::{AllocRange, Allocation, ConstAllocation, Scalar as MirScalar};
+use rustc_middle::mir::Mutability;
+use rustc_middle::ty::layout::LayoutCx;
+use rustc_middle::ty::{ParamEnv, ParamEnvAnd};
+use rustc_middle::ty::{Ty, TyCtxt};
+use rustc_target::abi::{
+ Abi, FieldsShape, HasDataLayout, Integer, Primitive, Scalar, Size, TyAndLayout, Variants,
+ WrappingRange,
+};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+enum InvariantSize {
+ U8,
+ U16,
+ U32,
+ U64,
+ U128,
+ Pointer,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+struct InvariantKey {
+ offset: Size,
+ size: InvariantSize,
+}
+
+// FIXME: Don't add duplicate invariants (maybe use a HashMap?)
+fn add_invariants<'tcx>(
+ tcx: TyCtxt<'tcx>,
+ ty: Ty<'tcx>,
+ invs: &mut FxHashMap<InvariantKey, WrappingRange>,
+ offset: Size,
+ strictness: InvariantStrictness,
+) {
+ if strictness == InvariantStrictness::Disable {
+ return;
+ }
+
+ let x = tcx.layout_of(ParamEnvAnd { param_env: ParamEnv::reveal_all(), value: ty });
+
+ if let Ok(layout) = x {
+ if let Abi::Scalar(Scalar::Initialized { value, valid_range }) = layout.layout.abi() {
+ let size = match value {
+ Primitive::Int(Integer::I8, _) => InvariantSize::U8,
+ Primitive::Int(Integer::I16, _) => InvariantSize::U16,
+ Primitive::Int(Integer::I32, _) => InvariantSize::U32,
+ Primitive::Int(Integer::I64, _) => InvariantSize::U64,
+ Primitive::Int(Integer::I128, _) => InvariantSize::U128,
+ Primitive::F32 => InvariantSize::U32,
+ Primitive::F64 => InvariantSize::U64,
+ Primitive::Pointer => InvariantSize::Pointer,
+ };
+
+ if !valid_range.is_full_for(value.size(&tcx)) || strictness == InvariantStrictness::All
+ {
+ // Pick the first scalar we see, this means NonZeroU8(u8) ends up with only one
+ // invariant, the stricter one.
+ let _: Result<_, _> = invs.try_insert(InvariantKey { offset, size }, valid_range);
+ }
+ }
+
+ //dbg!(&ty, &layout);
+ if !matches!(layout.layout.variants(), Variants::Single { .. }) {
+ // We *don't* want to look for fields inside enums.
+ return;
+ }
+
+ let param_env = ParamEnv::reveal_all();
+ let layout_cx = LayoutCx { tcx, param_env };
+
+ match layout.layout.fields() {
+ FieldsShape::Primitive => {}
+ FieldsShape::Union(_) => {}
+ FieldsShape::Array { stride, count } => {
+ // We may wish to bail out if we're generating too many invariants.
+ // That would lead to false negatives, though.
+ for idx in 0..*count {
+ let off = offset + *stride * idx;
+ let f = layout.field(&layout_cx, idx as usize);
+ add_invariants(tcx, f.ty, invs, off, strictness);
+ }
+ }
+ FieldsShape::Arbitrary { offsets, .. } => {
+ for (idx, &field_offset) in offsets.iter().enumerate() {
+ let f = layout.field(&layout_cx, idx);
+ if f.ty == ty {
+ // Some types contain themselves as fields, such as
+ // &mut [T]
+ // Easy solution is to just not recurse then.
+ } else {
+ add_invariants(tcx, f.ty, invs, offset + field_offset, strictness);
+ }
+ }
+ }
+ }
+ }
+}
+
+fn get_layout_of_invariant<'tcx>(tcx: TyCtxt<'tcx>) -> TyAndLayout<'tcx, Ty<'tcx>> {
+ let item = tcx.require_lang_item(LangItem::ValidityInvariant, None);
+ let ty = tcx.type_of(item);
+ let layout = tcx
+ .layout_of(ParamEnv::reveal_all().and(ty))
+ .expect("invalid layout for ValidityInvariant lang item");
+ layout
+}
+
+#[derive(PartialEq, Clone, Copy, Eq)]
+pub(crate) enum InvariantStrictness {
+ Disable,
+ Normal,
+ All,
+}
+
+/// Directly returns a `ConstAllocation` containing a list of validity invariants of the given type.
+pub(crate) fn alloc_validity_invariants_of<'tcx>(
+ tcx: TyCtxt<'tcx>,
+ ty: Ty<'tcx>,
+ strictness: InvariantStrictness,
+) -> (ConstAllocation<'tcx>, usize) {
+ let mut invs = FxHashMap::default();
+
+ let layout = tcx.data_layout();
+ let validity_invariant = get_layout_of_invariant(tcx);
+
+ add_invariants(tcx, ty, &mut invs, Size::ZERO, strictness);
+
+ let allocation_size = validity_invariant.layout.size() * invs.len() as u64;
+ let mut alloc =
+ Allocation::uninit(allocation_size, validity_invariant.layout.align().abi, true).unwrap();
+
+ let offset_off = validity_invariant.layout.fields().offset(0);
+ let size_off = validity_invariant.layout.fields().offset(1);
+ let start_off = validity_invariant.layout.fields().offset(2);
+ let end_off = validity_invariant.layout.fields().offset(3);
+
+ for (idx, invariant) in invs.iter().enumerate() {
+ let offset = idx as u64 * validity_invariant.layout.size();
+
+ let offset_range = AllocRange { start: offset + offset_off, size: layout.pointer_size };
+ alloc
+ .write_scalar(
+ &tcx,
+ offset_range,
+ MirScalar::from_machine_usize(invariant.0.offset.bytes(), &tcx).into(),
+ )
+ .unwrap();
+
+ let size_range = AllocRange { start: offset + size_off, size: Size::from_bytes(1) };
+ alloc
+ .write_scalar(&tcx, size_range, MirScalar::from_u8(invariant.0.size as u8).into())
+ .unwrap();
+
+ let offset_range = AllocRange { start: offset + start_off, size: Size::from_bytes(16) };
+ alloc
+ .write_scalar(&tcx, offset_range, MirScalar::from_u128(invariant.1.start).into())
+ .unwrap();
+
+ let offset_range = AllocRange { start: offset + end_off, size: Size::from_bytes(16) };
+ alloc
+ .write_scalar(&tcx, offset_range, MirScalar::from_u128(invariant.1.end).into())
+ .unwrap();
+ }
+
+ // The allocation is not mutable, we just needed write_scalar.
+ alloc.mutability = Mutability::Not;
+
+ (tcx.intern_const_alloc(alloc), invs.len())
+}
diff --git a/compiler/rustc_const_eval/src/interpret/operand.rs b/compiler/rustc_const_eval/src/interpret/operand.rs
index 6b05a49575fd9..ebbff75f0a102 100644
--- a/compiler/rustc_const_eval/src/interpret/operand.rs
+++ b/compiler/rustc_const_eval/src/interpret/operand.rs
@@ -692,6 +692,19 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
self,
))
}
+ ConstValue::CustomSlice { data, length } => {
+ // We rely on mutability being set correctly in `data` to prevent writes
+ // where none should happen.
+ let ptr = Pointer::new(
+ self.tcx.create_memory_alloc(data),
+ Size::ZERO, // offset: 0
+ );
+ Operand::Immediate(Immediate::new_slice(
+ Scalar::from_pointer(self.global_base_pointer(ptr)?, &*self.tcx),
+ u64::try_from(length).unwrap(),
+ self,
+ ))
+ }
};
Ok(OpTy { op, layout })
}
diff --git a/compiler/rustc_hir/src/lang_items.rs b/compiler/rustc_hir/src/lang_items.rs
index 9e5d781892487..1f38b8219fe60 100644
--- a/compiler/rustc_hir/src/lang_items.rs
+++ b/compiler/rustc_hir/src/lang_items.rs
@@ -327,6 +327,7 @@ language_item_table! {
Range, sym::Range, range_struct, Target::Struct, GenericRequirement::None;
RangeToInclusive, sym::RangeToInclusive, range_to_inclusive_struct, Target::Struct, GenericRequirement::None;
RangeTo, sym::RangeTo, range_to_struct, Target::Struct, GenericRequirement::None;
+ ValidityInvariant, sym::ValidityInvariant, validity_invariant_struct, Target::Struct, GenericRequirement::None;
}
pub enum GenericRequirement {
diff --git a/compiler/rustc_middle/src/mir/interpret/value.rs b/compiler/rustc_middle/src/mir/interpret/value.rs
index e80918d5e5d03..610abe4b187a1 100644
--- a/compiler/rustc_middle/src/mir/interpret/value.rs
+++ b/compiler/rustc_middle/src/mir/interpret/value.rs
@@ -37,6 +37,9 @@ pub enum ConstValue<'tcx> {
/// Used only for `&[u8]` and `&str`
Slice { data: ConstAllocation<'tcx>, start: usize, end: usize },
+ /// Like `Slice`, but for types that aren't 1 byte long.
+ CustomSlice { data: ConstAllocation<'tcx>, length: usize },
+
/// A value not represented/representable by `Scalar` or `Slice`
ByRef {
/// The backing memory of the value, may contain more memory than needed for just the value
@@ -61,6 +64,9 @@ impl<'a, 'tcx> Lift<'tcx> for ConstValue<'a> {
ConstValue::ByRef { alloc, offset } => {
ConstValue::ByRef { alloc: tcx.lift(alloc)?, offset }
}
+ ConstValue::CustomSlice { data, length } => {
+ ConstValue::CustomSlice { data: tcx.lift(data)?, length }
+ }
})
}
}
@@ -69,7 +75,9 @@ impl<'tcx> ConstValue<'tcx> {
#[inline]
pub fn try_to_scalar(&self) -> Option<Scalar<AllocId>> {
match *self {
- ConstValue::ByRef { .. } | ConstValue::Slice { .. } => None,
+ ConstValue::ByRef { .. }
+ | ConstValue::Slice { .. }
+ | ConstValue::CustomSlice { .. } => None,
ConstValue::Scalar(val) => Some(val),
}
}
@@ -258,6 +266,11 @@ impl<Tag> Scalar<Tag> {
Scalar::Int(i.into())
}
+ #[inline]
+ pub fn from_u128(i: u128) -> Self {
+ Scalar::Int(i.into())
+ }
+
#[inline]
pub fn from_machine_usize(i: u64, cx: &impl HasDataLayout) -> Self {
Self::from_uint(i, cx.data_layout().pointer_size)
diff --git a/compiler/rustc_middle/src/mir/pretty.rs b/compiler/rustc_middle/src/mir/pretty.rs
index 462c0ada3cf87..d44d2567cc388 100644
--- a/compiler/rustc_middle/src/mir/pretty.rs
+++ b/compiler/rustc_middle/src/mir/pretty.rs
@@ -451,6 +451,7 @@ impl<'tcx> Visitor<'tcx> for ExtraComments<'tcx> {
let fmt_val = |val: &ConstValue<'tcx>| match val {
ConstValue::Scalar(s) => format!("Scalar({:?})", s),
ConstValue::Slice { .. } => format!("Slice(..)"),
+ ConstValue::CustomSlice { .. } => format!("CustomSlice(..)"),
ConstValue::ByRef { .. } => format!("ByRef(..)"),
};
@@ -679,7 +680,9 @@ pub fn write_allocations<'tcx>(
ConstValue::Scalar(interpret::Scalar::Int { .. }) => {
Either::Left(Either::Right(std::iter::empty()))
}
- ConstValue::ByRef { alloc, .. } | ConstValue::Slice { data: alloc, .. } => {
+ ConstValue::ByRef { alloc, .. }
+ | ConstValue::Slice { data: alloc, .. }
+ | ConstValue::CustomSlice { data: alloc, .. } => {
Either::Right(alloc_ids_from_alloc(alloc))
}
}
diff --git a/compiler/rustc_session/src/options.rs b/compiler/rustc_session/src/options.rs
index 441e1f9f6a2b8..ccb591a552256 100644
--- a/compiler/rustc_session/src/options.rs
+++ b/compiler/rustc_session/src/options.rs
@@ -1380,6 +1380,8 @@ options! {
"run LLVM in non-parallel mode (while keeping codegen-units and ThinLTO)"),
no_unique_section_names: bool = (false, parse_bool, [TRACKED],
"do not use unique names for text and data sections when -Z function-sections is used"),
+ no_validity_invariant_checks: bool = (false, parse_bool, [TRACKED],
+ "do not generate any validity invariants in the validity_invariants_of intrinsic"),
no_profiler_runtime: bool = (false, parse_no_flag, [TRACKED],
"prevent automatic injection of the profiler_builtins crate"),
normalize_docs: bool = (false, parse_bool, [TRACKED],
diff --git a/compiler/rustc_span/src/symbol.rs b/compiler/rustc_span/src/symbol.rs
index 6daf811e26f14..e0afbccff53e0 100644
--- a/compiler/rustc_span/src/symbol.rs
+++ b/compiler/rustc_span/src/symbol.rs
@@ -277,6 +277,7 @@ symbols! {
TyKind,
Unknown,
UnsafeArg,
+ ValidityInvariant,
Vec,
VecDeque,
Wrapper,
@@ -1524,6 +1525,7 @@ symbols! {
va_list,
va_start,
val,
+ validity_invariants_of,
values,
var,
variant_count,
diff --git a/compiler/rustc_typeck/src/check/intrinsic.rs b/compiler/rustc_typeck/src/check/intrinsic.rs
index 7fe710cf8f4f2..3e73864d7a777 100644
--- a/compiler/rustc_typeck/src/check/intrinsic.rs
+++ b/compiler/rustc_typeck/src/check/intrinsic.rs
@@ -9,6 +9,7 @@ use crate::require_same_types;
use rustc_errors::struct_span_err;
use rustc_hir as hir;
+use rustc_hir::lang_items::LangItem;
use rustc_middle::traits::{ObligationCause, ObligationCauseCode};
use rustc_middle::ty::subst::Subst;
use rustc_middle::ty::{self, TyCtxt};
@@ -102,7 +103,8 @@ pub fn intrinsic_operation_unsafety(intrinsic: Symbol) -> hir::Unsafety {
| sym::type_name
| sym::forget
| sym::black_box
- | sym::variant_count => hir::Unsafety::Normal,
+ | sym::variant_count
+ | sym::validity_invariants_of => hir::Unsafety::Normal,
_ => hir::Unsafety::Unsafe,
}
}
@@ -191,6 +193,12 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
sym::needs_drop => (1, Vec::new(), tcx.types.bool),
sym::type_name => (1, Vec::new(), tcx.mk_static_str()),
+ sym::validity_invariants_of => {
+ let item = tcx.require_lang_item(LangItem::ValidityInvariant, None);
+ let ty = tcx.type_of(item);
+ let slice = tcx.mk_imm_ref(tcx.lifetimes.re_static, tcx.mk_slice(ty));
+ (1, Vec::new(), slice)
+ }
sym::type_id => (1, Vec::new(), tcx.types.u64),
sym::offset | sym::arith_offset => (
1,
diff --git a/library/core/src/intrinsics.rs b/library/core/src/intrinsics.rs
index ba837ea9a7898..01f3d419bfa7e 100644
--- a/library/core/src/intrinsics.rs
+++ b/library/core/src/intrinsics.rs
@@ -1976,6 +1976,13 @@ extern "rust-intrinsic" {
/// [`std::hint::black_box`]: crate::hint::black_box
#[rustc_const_unstable(feature = "const_black_box", issue = "none")]
pub fn black_box<T>(dummy: T) -> T;
+
+ /// Returns a list of invaraints that must be valid in order for T to be valid.
+ ///
+ /// This is used internally to allow for runtime assertions inside `MaybeUninit::assume_init`
+ #[cfg(not(bootstrap))]
+ #[rustc_const_unstable(feature = "const_intrinsic_validity_invariants_of", issue = "none")]
+ pub fn validity_invariants_of<T>() -> &'static [Invariant];
}
// Some functions are defined here because they accidentally got made
@@ -2138,6 +2145,40 @@ pub const unsafe fn copy_nonoverlapping<T>(src: *const T, dst: *mut T, count: us
}
}
+#[derive(Debug, Clone, Copy, PartialEq)]
+#[repr(u8)]
+#[allow(dead_code)] // https://github.com/rust-lang/rust/issues/85677
+pub enum InvariantSize {
+ U8 = 0,
+ U16 = 1,
+ U32 = 2,
+ U64 = 3,
+ U128 = 4,
+ Pointer = 5,
+}
+
+/// An Invariant is a field and its valid range. The valid range may be full, in which case it
+/// should still be branched on, to allow tools like memory sanitizer to verify the memory is not
+/// uninit.
+// Be sure to keep the fields in order (offset, size, start, end), validity_invariants_of.rs needs
+// that.
+#[cfg(not(bootstrap))]
+#[derive(Debug, Clone, Copy, PartialEq)]
+#[lang = "ValidityInvariant"]
+pub struct Invariant {
+ /// The offset in bytes from the start of the struct
+ pub offset: usize,
+ /// The size/type of the invariant.
+ pub size: InvariantSize,
+ /// The start point of the valid range of this field. Is allowed to be > valid_range_end, see
+ /// <https://doc.rust-lang.org/nightly/nightly-rustc/rustc_target/abi/struct.WrappingRange.html>
+ /// which this follows the semantics of.
+ pub valid_range_start: u128,
+
+ /// The end point of the range.
+ pub valid_range_end: u128,
+}
+
/// Copies `count * size_of::<T>()` bytes from `src` to `dst`. The source
/// and destination may overlap.
///
@@ -2394,3 +2435,71 @@ where
{
called_in_const.call_once(arg)
}
+
+#[cfg(bootstrap)]
+pub const unsafe fn assert_validity_of<T>(_: *const T) -> bool {
+ true
+}
+
+#[cfg(not(bootstrap))]
+/// Asserts that the value at `value` is a valid T.
+///
+/// Best effort, can miss some UB, and is UB if the value is invalid.
+pub unsafe fn assert_validity_of<T>(value: *const T) -> bool {
+ // We have to do this, since we call assert_validity_of inside MaybeUninit::assume_init
+ // and if we had used ptr::read_unaligned, that would be a recursive call.
+ #[repr(packed)]
+ struct Unaligned<T>(T);
+
+ // SAFETY: The pointer dereferences here are valid if `value` is valid.
+ unsafe {
+ let invariants = validity_invariants_of::<T>();
+ for invariant in invariants {
+ let off = invariant.offset as usize;
+ let start = invariant.valid_range_start;
+ let end = invariant.valid_range_end;
+
+ let (value, max): (u128, u128) = match invariant.size {
+ InvariantSize::U8 => ((*(value.cast::<u8>().add(off))).into(), u8::MAX.into()),
+ InvariantSize::U16 => (
+ (*value.cast::<u8>().add(off).cast::<Unaligned<u16>>()).0.into(),
+ u16::MAX.into(),
+ ),
+ InvariantSize::U32 => (
+ (*value.cast::<u8>().add(off).cast::<Unaligned<u32>>()).0.into(),
+ u32::MAX.into(),
+ ),
+ InvariantSize::U64 => (
+ (*value.cast::<u8>().add(off).cast::<Unaligned<u64>>()).0.into(),
+ u64::MAX.into(),
+ ),
+ InvariantSize::U128 => (
+ (*value.cast::<u8>().add(off).cast::<Unaligned<u128>>()).0.into(),
+ u128::MAX.into(),
+ ),
+ InvariantSize::Pointer => (
+ (*value.cast::<u8>().add(off).cast::<Unaligned<*const ()>>()).0.addr() as u128,
+ usize::MAX as u128,
+ ),
+ };
+
+ if start > end {
+ if !((start..=max).contains(&value) || (0..=end).contains(&value)) {
+ panic!(
+ "read value {value} which was not in range 0..={end} or {start}..={max} at offset {off} in type {}",
+ core::any::type_name::<T>()
+ );
+ }
+ } else {
+ if !(start..=end).contains(&value) {
+ panic!(
+ "read value {value} which was not in range {start}..={end} at offset {off} in type {}",
+ core::any::type_name::<T>()
+ );
+ }
+ }
+ }
+
+ true
+ }
+}
diff --git a/library/core/src/lib.rs b/library/core/src/lib.rs
index 093c7d298734a..6e1d304e50cd2 100644
--- a/library/core/src/lib.rs
+++ b/library/core/src/lib.rs
@@ -156,7 +156,6 @@
#![feature(const_slice_from_ref)]
#![feature(const_slice_index)]
#![feature(const_is_char_boundary)]
-//
// Language features:
#![feature(abi_unadjusted)]
#![feature(allow_internal_unsafe)]
diff --git a/library/core/src/mem/maybe_uninit.rs b/library/core/src/mem/maybe_uninit.rs
index b4ea536083392..0ceb0b6f41c83 100644
--- a/library/core/src/mem/maybe_uninit.rs
+++ b/library/core/src/mem/maybe_uninit.rs
@@ -626,11 +626,17 @@ impl<T> MaybeUninit<T> {
#[inline(always)]
#[rustc_diagnostic_item = "assume_init"]
#[track_caller]
+ #[rustc_allow_const_fn_unstable(const_refs_to_cell)]
pub const unsafe fn assume_init(self) -> T {
// SAFETY: the caller must guarantee that `self` is initialized.
// This also means that `self` must be a `value` variant.
unsafe {
intrinsics::assert_inhabited::<T>();
+
+ intrinsics::assert_unsafe_precondition!(intrinsics::assert_validity_of::<T>(
+ &self as *const MaybeUninit<T> as *const T
+ ));
+
ManuallyDrop::into_inner(self.value)
}
}
@@ -795,6 +801,11 @@ impl<T> MaybeUninit<T> {
// This also means that `self` must be a `value` variant.
unsafe {
intrinsics::assert_inhabited::<T>();
+
+ intrinsics::assert_unsafe_precondition!(intrinsics::assert_validity_of::<T>(
+ self as *const MaybeUninit<T> as *const T
+ ));
+
&*self.as_ptr()
}
}
@@ -912,6 +923,11 @@ impl<T> MaybeUninit<T> {
// This also means that `self` must be a `value` variant.
unsafe {
intrinsics::assert_inhabited::<T>();
+
+ intrinsics::assert_unsafe_precondition!(intrinsics::assert_validity_of::<T>(
+ self as *const MaybeUninit<T> as *const T
+ ));
+
&mut *self.as_mut_ptr()
}
}
diff --git a/src/test/ui/intrinsics/validity_invariants_of.rs b/src/test/ui/intrinsics/validity_invariants_of.rs
new file mode 100644
index 0000000000000..d5156053316ab
--- /dev/null
+++ b/src/test/ui/intrinsics/validity_invariants_of.rs
@@ -0,0 +1,97 @@
+// run-pass
+// revisions: disabled normal sanitized
+// [disabled]compile-flags: -Zno-validity-invariant-checks
+// [sanitized]compile-flags: -Z sanitizer=memory
+#![feature(core_intrinsics, const_intrinsic_validity_invariants_of)]
+
+use std::mem::MaybeUninit;
+use std::intrinsics::{Invariant, InvariantSize, validity_invariants_of, assert_validity_of};
+use std::num::NonZeroU16;
+
+#[repr(C)]
+struct MyStruct {
+ a: NonZeroU16,
+ b: u8,
+ c: bool,
+}
+
+const MY_STRUCT_INVARIANTS: &'static [Invariant] = validity_invariants_of::<MyStruct>();
+const OPTION_INVARIANTS: &'static [Invariant] = validity_invariants_of::<Option<fn()>>();
+
+fn main() {
+ let mut invs: Vec<Invariant> = MY_STRUCT_INVARIANTS.to_vec();
+
+ invs.sort_by_key(|x| x.offset);
+
+ if cfg!(disabled) {
+ assert_eq!(&invs, &[]);
+ } else if cfg!(normal) {
+ assert_eq!(&invs, &[
+ Invariant {
+ offset: 0,
+ size: InvariantSize::U16,
+ valid_range_start: 1,
+ valid_range_end: 65535
+ },
+ Invariant {
+ offset: 3,
+ size: InvariantSize::U8,
+ valid_range_start: 0,
+ valid_range_end: 1
+ },
+ ]);
+ } else {
+ assert!(cfg!(sanitized));
+
+ assert_eq!(&invs, &[
+ Invariant {
+ offset: 0,
+ size: InvariantSize::U16,
+ valid_range_start: 1,
+ valid_range_end: 65535
+ },
+ Invariant {
+ offset: 2,
+ size: InvariantSize::U8,
+ valid_range_start: 0,
+ valid_range_end: 255
+ },
+ Invariant {
+ offset: 3,
+ size: InvariantSize::U8,
+ valid_range_start: 0,
+ valid_range_end: 1
+ },
+ ]);
+ }
+
+
+ unsafe {
+ let v = MyStruct { a: NonZeroU16::new(1).unwrap(), b: 2, c: true };
+ assert!(assert_validity_of(&v as *const _));
+ }
+
+ if cfg!(sanitized) {
+ assert_eq!(OPTION_INVARIANTS, &[
+ Invariant {
+ offset: 0,
+ size: InvariantSize::Pointer,
+ valid_range_start: 1,
+ valid_range_end: 0,
+ },
+ ]);
+ } else {
+ assert_eq!(OPTION_INVARIANTS, &[]);
+ }
+
+
+ unsafe {
+ let p = MaybeUninit::<Option<fn()>>::zeroed().as_ptr();
+ assert!(assert_validity_of(p));
+ }
+
+ // There's two code paths for generating the data, be sure to test that compile time and
+ // runtime matches.
+ assert_eq!(validity_invariants_of::<Option<fn()>>(), OPTION_INVARIANTS);
+ assert_eq!(validity_invariants_of::<MyStruct>(), MY_STRUCT_INVARIANTS);
+}
diff --git a/src/test/ui/sanitize/memory.rs b/src/test/ui/sanitize/memory.rs
index b53f19a5b01aa..2da871a62fcb9 100644
--- a/src/test/ui/sanitize/memory.rs
+++ b/src/test/ui/sanitize/memory.rs
@@ -1,7 +1,7 @@
// needs-sanitizer-support
// needs-sanitizer-memory
//
-// compile-flags: -Z sanitizer=memory -Zsanitizer-memory-track-origins -O
+// compile-flags: -Z sanitizer=memory -Zno-validity-invariant-checks -Zsanitizer-memory-track-origins -O
//
// run-fail
// error-pattern: MemorySanitizer: use-of-uninitialized-value
@@ -10,6 +10,8 @@
//
// This test case intentionally limits the usage of the std,
// since it will be linked with an uninstrumented version of it.
+//
+// -Zno-validity-invariant-checks is needed in order to not fail inside the assume_init
#![feature(core_intrinsics)]
#![feature(start)]