4
4
// collections, resulting in having to optimize down excess IR multiple times.
5
5
// Your performance intuition is useless. Run perf.
6
6
7
+ use super :: size_in_bytes:: SizeInBytes ;
7
8
use crate :: error:: Error ;
9
+ use crate :: intrinsics:: unchecked_sub;
10
+ use crate :: mem:: SizedTypeProperties ;
8
11
use crate :: ptr:: { Alignment , NonNull } ;
9
- use crate :: { assert_unsafe_precondition, cmp , fmt, mem} ;
12
+ use crate :: { assert_unsafe_precondition, fmt, mem} ;
10
13
11
14
// While this function is used in one place and its implementation
12
15
// could be inlined, the previous attempts to do so made rustc
@@ -37,7 +40,7 @@ const fn size_align<T>() -> (usize, usize) {
37
40
#[ lang = "alloc_layout" ]
38
41
pub struct Layout {
39
42
// size of the requested block of memory, measured in bytes.
40
- size : usize ,
43
+ size : SizeInBytes ,
41
44
42
45
// alignment of the requested block of memory, measured in bytes.
43
46
// we ensure that this is always a power-of-two, because API's
@@ -68,22 +71,22 @@ impl Layout {
68
71
pub const fn from_size_align ( size : usize , align : usize ) -> Result < Self , LayoutError > {
69
72
if Layout :: is_size_align_valid ( size, align) {
70
73
// SAFETY: Layout::is_size_align_valid checks the preconditions for this call.
71
- unsafe { Ok ( Layout { size, align : mem:: transmute ( align) } ) }
74
+ unsafe { Ok ( Layout { size : mem :: transmute ( size ) , align : mem:: transmute ( align) } ) }
72
75
} else {
73
76
Err ( LayoutError )
74
77
}
75
78
}
76
79
77
80
const fn is_size_align_valid ( size : usize , align : usize ) -> bool {
78
81
let Some ( align) = Alignment :: new ( align) else { return false } ;
79
- if size > Self :: max_size_for_align ( align) {
82
+ if size > Self :: max_size_for_align ( align) . as_usize ( ) {
80
83
return false ;
81
84
}
82
85
true
83
86
}
84
87
85
88
#[ inline( always) ]
86
- const fn max_size_for_align ( align : Alignment ) -> usize {
89
+ const fn max_size_for_align ( align : Alignment ) -> SizeInBytes {
87
90
// (power-of-two implies align != 0.)
88
91
89
92
// Rounded up size is:
@@ -98,18 +101,28 @@ impl Layout {
98
101
//
99
102
// Above implies that checking for summation overflow is both
100
103
// necessary and sufficient.
101
- isize:: MAX as usize - ( align. as_usize ( ) - 1 )
104
+
105
+ // SAFETY: the maximum possible alignment is `isize::MAX + 1`,
106
+ // so the first subtraction cannot overflow. The minimum possible
107
+ // alignment is `1`, so the subtraction returns as most `isize::MAX`,
108
+ // and thus the calculated `max_size` is guaranteed in-range.
109
+ unsafe {
110
+ let max_size = unchecked_sub ( isize:: MAX as usize + 1 , align. as_usize ( ) ) ;
111
+ SizeInBytes :: new_unchecked ( max_size)
112
+ }
102
113
}
103
114
104
- /// Internal helper constructor to skip revalidating alignment validity.
115
+ /// Internal helper constructor to check only the inter-field invariant,
116
+ /// trusting the types to enforce the per-field invariants.
105
117
#[ inline]
106
- const fn from_size_alignment ( size : usize , align : Alignment ) -> Result < Self , LayoutError > {
107
- if size > Self :: max_size_for_align ( align) {
108
- return Err ( LayoutError ) ;
118
+ const fn from_size_alignment ( size : SizeInBytes , align : Alignment ) -> Result < Self , LayoutError > {
119
+ // FIXME: remove the `as_usize`s once we can use `const PartialOrd`
120
+ if size. as_usize ( ) <= Self :: max_size_for_align ( align) . as_usize ( ) {
121
+ // SAFETY: Layout::size invariants checked above.
122
+ Ok ( Layout { size, align } )
123
+ } else {
124
+ Err ( LayoutError )
109
125
}
110
-
111
- // SAFETY: Layout::size invariants checked above.
112
- Ok ( Layout { size, align } )
113
126
}
114
127
115
128
/// Creates a layout, bypassing all checks.
@@ -134,7 +147,7 @@ impl Layout {
134
147
) => Layout :: is_size_align_valid( size, align)
135
148
) ;
136
149
// SAFETY: the caller is required to uphold the preconditions.
137
- unsafe { Layout { size, align : mem:: transmute ( align) } }
150
+ unsafe { Layout { size : mem :: transmute ( size ) , align : mem:: transmute ( align) } }
138
151
}
139
152
140
153
/// The minimum size in bytes for a memory block of this layout.
@@ -143,7 +156,7 @@ impl Layout {
143
156
#[ must_use]
144
157
#[ inline]
145
158
pub const fn size ( & self ) -> usize {
146
- self . size
159
+ self . size . as_usize ( )
147
160
}
148
161
149
162
/// The minimum byte alignment for a memory block of this layout.
@@ -252,9 +265,14 @@ impl Layout {
252
265
/// Returns an error if the combination of `self.size()` and the given
253
266
/// `align` violates the conditions listed in [`Layout::from_size_align`].
254
267
#[ stable( feature = "alloc_layout_manipulation" , since = "1.44.0" ) ]
268
+ #[ rustc_const_unstable( feature = "const_alloc_layout" , issue = "67521" ) ]
255
269
#[ inline]
256
- pub fn align_to ( & self , align : usize ) -> Result < Self , LayoutError > {
257
- Layout :: from_size_align ( self . size ( ) , cmp:: max ( self . align ( ) , align) )
270
+ pub const fn align_to ( & self , align : usize ) -> Result < Self , LayoutError > {
271
+ if let Some ( align) = Alignment :: new ( align) {
272
+ Layout :: from_size_alignment ( self . size , Alignment :: max ( self . align , align) )
273
+ } else {
274
+ Err ( LayoutError )
275
+ }
258
276
}
259
277
260
278
/// Returns the amount of padding we must insert after `self`
@@ -279,29 +297,43 @@ impl Layout {
279
297
without modifying the `Layout`"]
280
298
#[ inline]
281
299
pub const fn padding_needed_for ( & self , align : usize ) -> usize {
282
- let len = self . size ( ) ;
300
+ // FIXME: Can we just change the type on this to `Alignment`?
301
+ let Some ( align) = Alignment :: new ( align) else { return usize:: MAX } ;
302
+ self . padding_bytes_needed_for ( align) . as_usize ( )
303
+ }
304
+
305
+ #[ inline]
306
+ const fn padding_bytes_needed_for ( & self , align : Alignment ) -> SizeInBytes {
307
+ let len = self . size ;
308
+ let align_m1 = SizeInBytes :: alignment_minus_one ( align) ;
309
+ let len_rounded_up = len. add_wide ( align_m1) & !align_m1. as_usize ( ) ;
283
310
311
+ // SAFETY:
284
312
// Rounded up value is:
285
313
// len_rounded_up = (len + align - 1) & !(align - 1);
286
314
// and then we return the padding difference: `len_rounded_up - len`.
287
315
//
288
- // We use modular arithmetic throughout :
316
+ // The arithmetic we do here can never overflow :
289
317
//
290
318
// 1. align is guaranteed to be > 0, so align - 1 is always
291
319
// valid.
292
320
//
293
- // 2. `len + align - 1` can overflow by at most `align - 1`,
294
- // so the &-mask with `!(align - 1)` will ensure that in the
295
- // case of overflow, `len_rounded_up` will itself be 0.
296
- // Thus the returned padding, when added to `len`, yields 0,
297
- // which trivially satisfies the alignment `align`.
321
+ // 2. len is at most `isize::MAX`, so adding `align - 1` can never
322
+ // overflow a `usize`.
298
323
//
299
- // (Of course, attempts to allocate blocks of memory whose
300
- // size and padding overflow in the above manner should cause
301
- // the allocator to yield an error anyway.)
302
-
303
- let len_rounded_up = len. wrapping_add ( align) . wrapping_sub ( 1 ) & !align. wrapping_sub ( 1 ) ;
304
- len_rounded_up. wrapping_sub ( len)
324
+ // 3. masking by the alignment can remove at most `align - 1`,
325
+ // which is what we just added, so the subtraction cannot overflow.
326
+ //
327
+ // 4. the resulting padding is thus at most `align - 1`, but the largest
328
+ // possible alignment is `isize::MAX + 1`, and thus the padding
329
+ // will always fit in `SizeInBytes`.
330
+ //
331
+ // (Size 0 Align MAX is already aligned, so doesn't need any padding,
332
+ // but Size 1 Align MAX has the largest padding requirement: `isize::MAX`.)
333
+ unsafe {
334
+ let padding = unchecked_sub ( len_rounded_up, len. as_usize ( ) ) ;
335
+ SizeInBytes :: new_unchecked ( padding)
336
+ }
305
337
}
306
338
307
339
/// Creates a layout by rounding the size of this layout up to a multiple
@@ -315,12 +347,12 @@ impl Layout {
315
347
without modifying the original"]
316
348
#[ inline]
317
349
pub const fn pad_to_align ( & self ) -> Layout {
318
- let pad = self . padding_needed_for ( self . align ( ) ) ;
350
+ let pad = self . padding_bytes_needed_for ( self . align ) ;
319
351
// This cannot overflow. Quoting from the invariant of Layout:
320
352
// > `size`, when rounded up to the nearest multiple of `align`,
321
353
// > must not overflow isize (i.e., the rounded value must be
322
354
// > less than or equal to `isize::MAX`)
323
- let new_size = self . size ( ) + pad;
355
+ let new_size = self . size . add_wide ( pad) ;
324
356
325
357
// SAFETY: padded size is guaranteed to not exceed `isize::MAX`.
326
358
unsafe { Layout :: from_size_align_unchecked ( new_size, self . align ( ) ) }
@@ -333,20 +365,36 @@ impl Layout {
333
365
/// layout of the array and `offs` is the distance between the start
334
366
/// of each element in the array.
335
367
///
368
+ /// (That distance between elements is sometimes known as "stride".)
369
+ ///
336
370
/// On arithmetic overflow, returns `LayoutError`.
371
+ ///
372
+ /// # Examples
373
+ ///
374
+ /// ```
375
+ /// #![feature(alloc_layout_extra)]
376
+ /// use std::alloc::Layout;
377
+ ///
378
+ /// // All rust types have a size that's a multiple of their alignment.
379
+ /// let normal = Layout::from_size_align(12, 4).unwrap();
380
+ /// let repeated = normal.repeat(3).unwrap();
381
+ /// assert_eq!(repeated, (Layout::from_size_align(36, 4).unwrap(), 12));
382
+ ///
383
+ /// // But you can manually make layouts which don't meet that rule.
384
+ /// let padding_needed = Layout::from_size_align(6, 4).unwrap();
385
+ /// let repeated = padding_needed.repeat(3).unwrap();
386
+ /// assert_eq!(repeated, (Layout::from_size_align(24, 4).unwrap(), 8));
387
+ /// ```
337
388
#[ unstable( feature = "alloc_layout_extra" , issue = "55724" ) ]
389
+ #[ rustc_const_unstable( feature = "const_alloc_layout" , issue = "67521" ) ]
338
390
#[ inline]
339
- pub fn repeat ( & self , n : usize ) -> Result < ( Self , usize ) , LayoutError > {
340
- // This cannot overflow. Quoting from the invariant of Layout:
341
- // > `size`, when rounded up to the nearest multiple of `align`,
342
- // > must not overflow isize (i.e., the rounded value must be
343
- // > less than or equal to `isize::MAX`)
344
- let padded_size = self . size ( ) + self . padding_needed_for ( self . align ( ) ) ;
345
- let alloc_size = padded_size. checked_mul ( n) . ok_or ( LayoutError ) ?;
346
-
347
- // The safe constructor is called here to enforce the isize size limit.
348
- let layout = Layout :: from_size_alignment ( alloc_size, self . align ) ?;
349
- Ok ( ( layout, padded_size) )
391
+ pub const fn repeat ( & self , n : usize ) -> Result < ( Self , usize ) , LayoutError > {
392
+ let padded = self . pad_to_align ( ) ;
393
+ if let Ok ( repeated) = padded. repeat_packed ( n) {
394
+ Ok ( ( repeated, padded. size ( ) ) )
395
+ } else {
396
+ Err ( LayoutError )
397
+ }
350
398
}
351
399
352
400
/// Creates a layout describing the record for `self` followed by
@@ -395,17 +443,20 @@ impl Layout {
395
443
/// # assert_eq!(repr_c(&[u64, u32, u16, u32]), Ok((s, vec![0, 8, 12, 16])));
396
444
/// ```
397
445
#[ stable( feature = "alloc_layout_manipulation" , since = "1.44.0" ) ]
446
+ #[ rustc_const_unstable( feature = "const_alloc_layout" , issue = "67521" ) ]
398
447
#[ inline]
399
- pub fn extend ( & self , next : Self ) -> Result < ( Self , usize ) , LayoutError > {
400
- let new_align = cmp:: max ( self . align , next. align ) ;
401
- let pad = self . padding_needed_for ( next. align ( ) ) ;
402
-
403
- let offset = self . size ( ) . checked_add ( pad) . ok_or ( LayoutError ) ?;
404
- let new_size = offset. checked_add ( next. size ( ) ) . ok_or ( LayoutError ) ?;
405
-
406
- // The safe constructor is called here to enforce the isize size limit.
407
- let layout = Layout :: from_size_alignment ( new_size, new_align) ?;
408
- Ok ( ( layout, offset) )
448
+ pub const fn extend ( & self , next : Self ) -> Result < ( Self , usize ) , LayoutError > {
449
+ let new_align = Alignment :: max ( self . align , next. align ) ;
450
+ let pad = self . padding_bytes_needed_for ( next. align ) ;
451
+
452
+ if let Some ( offset) = self . size . checked_add ( pad)
453
+ && let Some ( new_size) = offset. checked_add ( next. size )
454
+ && let Ok ( layout) = Layout :: from_size_alignment ( new_size, new_align)
455
+ {
456
+ Ok ( ( layout, offset. as_usize ( ) ) )
457
+ } else {
458
+ Err ( LayoutError )
459
+ }
409
460
}
410
461
411
462
/// Creates a layout describing the record for `n` instances of
@@ -421,11 +472,15 @@ impl Layout {
421
472
///
422
473
/// On arithmetic overflow, returns `LayoutError`.
423
474
#[ unstable( feature = "alloc_layout_extra" , issue = "55724" ) ]
475
+ #[ rustc_const_unstable( feature = "const_alloc_layout" , issue = "67521" ) ]
424
476
#[ inline]
425
- pub fn repeat_packed ( & self , n : usize ) -> Result < Self , LayoutError > {
426
- let size = self . size ( ) . checked_mul ( n) . ok_or ( LayoutError ) ?;
427
- // The safe constructor is called here to enforce the isize size limit.
428
- Layout :: from_size_alignment ( size, self . align )
477
+ pub const fn repeat_packed ( & self , n : usize ) -> Result < Self , LayoutError > {
478
+ if let Some ( size) = self . size . checked_mul ( n) {
479
+ // The safe constructor is called here to enforce the isize size limit.
480
+ Layout :: from_size_alignment ( size, self . align )
481
+ } else {
482
+ Err ( LayoutError )
483
+ }
429
484
}
430
485
431
486
/// Creates a layout describing the record for `self` followed by
@@ -435,11 +490,15 @@ impl Layout {
435
490
///
436
491
/// On arithmetic overflow, returns `LayoutError`.
437
492
#[ unstable( feature = "alloc_layout_extra" , issue = "55724" ) ]
493
+ #[ rustc_const_unstable( feature = "const_alloc_layout" , issue = "67521" ) ]
438
494
#[ inline]
439
- pub fn extend_packed ( & self , next : Self ) -> Result < Self , LayoutError > {
440
- let new_size = self . size ( ) . checked_add ( next. size ( ) ) . ok_or ( LayoutError ) ?;
441
- // The safe constructor is called here to enforce the isize size limit.
442
- Layout :: from_size_alignment ( new_size, self . align )
495
+ pub const fn extend_packed ( & self , next : Self ) -> Result < Self , LayoutError > {
496
+ if let Some ( new_size) = self . size . checked_add ( next. size ) {
497
+ // The safe constructor is called here to enforce the isize size limit.
498
+ Layout :: from_size_alignment ( new_size, self . align )
499
+ } else {
500
+ Err ( LayoutError )
501
+ }
443
502
}
444
503
445
504
/// Creates a layout describing the record for a `[T; n]`.
@@ -451,21 +510,21 @@ impl Layout {
451
510
#[ inline]
452
511
pub const fn array < T > ( n : usize ) -> Result < Self , LayoutError > {
453
512
// Reduce the amount of code we need to monomorphize per `T`.
454
- return inner ( mem :: size_of :: < T > ( ) , Alignment :: of :: < T > ( ) , n) ;
513
+ return inner ( T :: LAYOUT , n) ;
455
514
456
515
#[ inline]
457
- const fn inner (
458
- element_size : usize ,
459
- align : Alignment ,
460
- n : usize ,
461
- ) -> Result < Layout , LayoutError > {
516
+ const fn inner ( element_layout : Layout , n : usize ) -> Result < Layout , LayoutError > {
517
+ let Layout { size, align } = element_layout;
518
+ let element_size = size. as_usize ( ) ;
519
+
462
520
// We need to check two things about the size:
463
521
// - That the total size won't overflow a `usize`, and
464
522
// - That the total size still fits in an `isize`.
465
523
// By using division we can check them both with a single threshold.
466
524
// That'd usually be a bad idea, but thankfully here the element size
467
525
// and alignment are constants, so the compiler will fold all of it.
468
- if element_size != 0 && n > Layout :: max_size_for_align ( align) / element_size {
526
+ if element_size != 0 && n > Layout :: max_size_for_align ( align) . as_usize ( ) / element_size
527
+ {
469
528
return Err ( LayoutError ) ;
470
529
}
471
530
0 commit comments