Auto merge of #126299 - scottmcm:tune-sliceindex-ubchecks, r=saethlin · model-checking/verify-rust-std@ae7f43e (original) (raw)
`@@ -2,7 +2,6 @@
`
2
2
``
3
3
`use crate::intrinsics::const_eval_select;
`
4
4
`use crate::ops;
`
5
``
`-
use crate::ptr;
`
6
5
`use crate::ub_checks::assert_unsafe_precondition;
`
7
6
``
8
7
`#[stable(feature = "rust1", since = "1.0.0")]
`
`@@ -106,6 +105,47 @@ const fn slice_end_index_overflow_fail() -> ! {
`
106
105
`panic!("attempted to index slice up to maximum usize");
`
107
106
`}
`
108
107
``
``
108
`+
// The UbChecks are great for catching bugs in the unsafe methods, but including
`
``
109
`+
// them in safe indexing is unnecessary and hurts inlining and debug runtime perf.
`
``
110
`+
// Both the safe and unsafe public methods share these helpers,
`
``
111
`+
// which use intrinsics directly to get no extra checks.
`
``
112
+
``
113
`+
#[inline(always)]
`
``
114
`+
const unsafe fn get_noubcheck(ptr: *const [T], index: usize) -> *const T {
`
``
115
`+
let ptr = ptr as *const T;
`
``
116
`+
// SAFETY: The caller already checked these preconditions
`
``
117
`+
unsafe { crate::intrinsics::offset(ptr, index) }
`
``
118
`+
}
`
``
119
+
``
120
`+
#[inline(always)]
`
``
121
`+
const unsafe fn get_mut_noubcheck(ptr: *mut [T], index: usize) -> *mut T {
`
``
122
`+
let ptr = ptr as *mut T;
`
``
123
`+
// SAFETY: The caller already checked these preconditions
`
``
124
`+
unsafe { crate::intrinsics::offset(ptr, index) }
`
``
125
`+
}
`
``
126
+
``
127
`+
#[inline(always)]
`
``
128
`+
const unsafe fn get_offset_len_noubcheck(
`
``
129
`+
ptr: *const [T],
`
``
130
`+
offset: usize,
`
``
131
`+
len: usize,
`
``
132
`+
) -> *const [T] {
`
``
133
`+
// SAFETY: The caller already checked these preconditions
`
``
134
`+
let ptr = unsafe { get_noubcheck(ptr, offset) };
`
``
135
`+
crate::intrinsics::aggregate_raw_ptr(ptr, len)
`
``
136
`+
}
`
``
137
+
``
138
`+
#[inline(always)]
`
``
139
`+
const unsafe fn get_offset_len_mut_noubcheck(
`
``
140
`+
ptr: *mut [T],
`
``
141
`+
offset: usize,
`
``
142
`+
len: usize,
`
``
143
`+
) -> *mut [T] {
`
``
144
`+
// SAFETY: The caller already checked these preconditions
`
``
145
`+
let ptr = unsafe { get_mut_noubcheck(ptr, offset) };
`
``
146
`+
crate::intrinsics::aggregate_raw_ptr(ptr, len)
`
``
147
`+
}
`
``
148
+
109
149
`mod private_slice_index {
`
110
150
`use super::ops;
`
111
151
`#[stable(feature = "slice_get_slice", since = "1.28.0")]
`
`@@ -203,13 +243,17 @@ unsafe impl SliceIndex<[T]> for usize {
`
203
243
`#[inline]
`
204
244
`fn get(self, slice: &[T]) -> Option<&T> {
`
205
245
`` // SAFETY: self is checked to be in bounds.
``
206
``
`-
if self < slice.len() { unsafe { Some(&*self.get_unchecked(slice)) } } else { None }
`
``
246
`+
if self < slice.len() { unsafe { Some(&*get_noubcheck(slice, self)) } } else { None }
`
207
247
`}
`
208
248
``
209
249
`#[inline]
`
210
250
`fn get_mut(self, slice: &mut [T]) -> Option<&mut T> {
`
211
``
`` -
// SAFETY: self is checked to be in bounds.
``
212
``
`-
if self < slice.len() { unsafe { Some(&mut *self.get_unchecked_mut(slice)) } } else { None }
`
``
251
`+
if self < slice.len() {
`
``
252
`` +
// SAFETY: self is checked to be in bounds.
``
``
253
`+
unsafe { Some(&mut *get_mut_noubcheck(slice, self)) }
`
``
254
`+
} else {
`
``
255
`+
None
`
``
256
`+
}
`
213
257
`}
`
214
258
``
215
259
`#[inline]
`
`@@ -227,7 +271,7 @@ unsafe impl SliceIndex<[T]> for usize {
`
227
271
`// Use intrinsics::assume instead of hint::assert_unchecked so that we don't check the
`
228
272
`// precondition of this function twice.
`
229
273
`crate::intrinsics::assume(self < slice.len());
`
230
``
`-
slice.as_ptr().add(self)
`
``
274
`+
get_noubcheck(slice, self)
`
231
275
`}
`
232
276
`}
`
233
277
``
`@@ -239,7 +283,7 @@ unsafe impl SliceIndex<[T]> for usize {
`
239
283
`(this: usize = self, len: usize = slice.len()) => this < len
`
240
284
`);
`
241
285
`` // SAFETY: see comments for get_unchecked above.
``
242
``
`-
unsafe { slice.as_mut_ptr().add(self) }
`
``
286
`+
unsafe { get_mut_noubcheck(slice, self) }
`
243
287
`}
`
244
288
``
245
289
`#[inline]
`
`@@ -265,7 +309,7 @@ unsafe impl SliceIndex<[T]> for ops::IndexRange {
`
265
309
`fn get(self, slice: &[T]) -> Option<&[T]> {
`
266
310
`if self.end() <= slice.len() {
`
267
311
`` // SAFETY: self is checked to be valid and in bounds above.
``
268
``
`-
unsafe { Some(&*self.get_unchecked(slice)) }
`
``
312
`+
unsafe { Some(&*get_offset_len_noubcheck(slice, self.start(), self.len())) }
`
269
313
`} else {
`
270
314
`None
`
271
315
`}
`
`@@ -275,7 +319,7 @@ unsafe impl SliceIndex<[T]> for ops::IndexRange {
`
275
319
`fn get_mut(self, slice: &mut [T]) -> Option<&mut [T]> {
`
276
320
`if self.end() <= slice.len() {
`
277
321
`` // SAFETY: self is checked to be valid and in bounds above.
``
278
``
`-
unsafe { Some(&mut *self.get_unchecked_mut(slice)) }
`
``
322
`+
unsafe { Some(&mut *get_offset_len_mut_noubcheck(slice, self.start(), self.len())) }
`
279
323
`} else {
`
280
324
`None
`
281
325
`}
`
`@@ -292,7 +336,7 @@ unsafe impl SliceIndex<[T]> for ops::IndexRange {
`
292
336
`` // cannot be longer than isize::MAX. They also guarantee that
``
293
337
`` // self is in bounds of slice so self cannot overflow an isize,
``
294
338
`` // so the call to add is safe.
``
295
``
`-
unsafe { ptr::slice_from_raw_parts(slice.as_ptr().add(self.start()), self.len()) }
`
``
339
`+
unsafe { get_offset_len_noubcheck(slice, self.start(), self.len()) }
`
296
340
`}
`
297
341
``
298
342
`#[inline]
`
`@@ -304,14 +348,14 @@ unsafe impl SliceIndex<[T]> for ops::IndexRange {
`
304
348
`);
`
305
349
``
306
350
`` // SAFETY: see comments for get_unchecked above.
``
307
``
`-
unsafe { ptr::slice_from_raw_parts_mut(slice.as_mut_ptr().add(self.start()), self.len()) }
`
``
351
`+
unsafe { get_offset_len_mut_noubcheck(slice, self.start(), self.len()) }
`
308
352
`}
`
309
353
``
310
354
`#[inline]
`
311
355
`fn index(self, slice: &[T]) -> &[T] {
`
312
356
`if self.end() <= slice.len() {
`
313
357
`` // SAFETY: self is checked to be valid and in bounds above.
``
314
``
`-
unsafe { &*self.get_unchecked(slice) }
`
``
358
`+
unsafe { &*get_offset_len_noubcheck(slice, self.start(), self.len()) }
`
315
359
`} else {
`
316
360
`slice_end_index_len_fail(self.end(), slice.len())
`
317
361
`}
`
`@@ -321,7 +365,7 @@ unsafe impl SliceIndex<[T]> for ops::IndexRange {
`
321
365
`fn index_mut(self, slice: &mut [T]) -> &mut [T] {
`
322
366
`if self.end() <= slice.len() {
`
323
367
`` // SAFETY: self is checked to be valid and in bounds above.
``
324
``
`-
unsafe { &mut *self.get_unchecked_mut(slice) }
`
``
368
`+
unsafe { &mut *get_offset_len_mut_noubcheck(slice, self.start(), self.len()) }
`
325
369
`} else {
`
326
370
`slice_end_index_len_fail(self.end(), slice.len())
`
327
371
`}
`
`@@ -338,21 +382,26 @@ unsafe impl SliceIndex<[T]> for ops::Range {
`
338
382
``
339
383
`#[inline]
`
340
384
`fn get(self, slice: &[T]) -> Option<&[T]> {
`
341
``
`-
if self.start > self.end || self.end > slice.len() {
`
342
``
`-
None
`
343
``
`-
} else {
`
``
385
`` +
// Using checked_sub is a safe way to get SubUnchecked in MIR
``
``
386
`+
if let Some(new_len) = usize::checked_sub(self.end, self.start)
`
``
387
`+
&& self.end <= slice.len()
`
``
388
`+
{
`
344
389
`` // SAFETY: self is checked to be valid and in bounds above.
``
345
``
`-
unsafe { Some(&*self.get_unchecked(slice)) }
`
``
390
`+
unsafe { Some(&*get_offset_len_noubcheck(slice, self.start, new_len)) }
`
``
391
`+
} else {
`
``
392
`+
None
`
346
393
`}
`
347
394
`}
`
348
395
``
349
396
`#[inline]
`
350
397
`fn get_mut(self, slice: &mut [T]) -> Option<&mut [T]> {
`
351
``
`-
if self.start > self.end || self.end > slice.len() {
`
352
``
`-
None
`
353
``
`-
} else {
`
``
398
`+
if let Some(new_len) = usize::checked_sub(self.end, self.start)
`
``
399
`+
&& self.end <= slice.len()
`
``
400
`+
{
`
354
401
`` // SAFETY: self is checked to be valid and in bounds above.
``
355
``
`-
unsafe { Some(&mut *self.get_unchecked_mut(slice)) }
`
``
402
`+
unsafe { Some(&mut *get_offset_len_mut_noubcheck(slice, self.start, new_len)) }
`
``
403
`+
} else {
`
``
404
`+
None
`
356
405
`}
`
357
406
`}
`
358
407
``
`@@ -373,8 +422,10 @@ unsafe impl SliceIndex<[T]> for ops::Range {
`
373
422
`` // self is in bounds of slice so self cannot overflow an isize,
``
374
423
`` // so the call to add is safe and the length calculation cannot overflow.
``
375
424
`unsafe {
`
376
``
`-
let new_len = self.end.unchecked_sub(self.start);
`
377
``
`-
ptr::slice_from_raw_parts(slice.as_ptr().add(self.start), new_len)
`
``
425
`+
// Using the intrinsic avoids a superfluous UB check,
`
``
426
`` +
// since the one on this method already checked end >= start.
``
``
427
`+
let new_len = crate::intrinsics::unchecked_sub(self.end, self.start);
`
``
428
`+
get_offset_len_noubcheck(slice, self.start, new_len)
`
378
429
`}
`
379
430
`}
`
380
431
``
`@@ -391,31 +442,34 @@ unsafe impl SliceIndex<[T]> for ops::Range {
`
391
442
`);
`
392
443
`` // SAFETY: see comments for get_unchecked above.
``
393
444
`unsafe {
`
394
``
`-
let new_len = self.end.unchecked_sub(self.start);
`
395
``
`-
ptr::slice_from_raw_parts_mut(slice.as_mut_ptr().add(self.start), new_len)
`
``
445
`+
let new_len = crate::intrinsics::unchecked_sub(self.end, self.start);
`
``
446
`+
get_offset_len_mut_noubcheck(slice, self.start, new_len)
`
396
447
`}
`
397
448
`}
`
398
449
``
399
450
`#[inline(always)]
`
400
451
`fn index(self, slice: &[T]) -> &[T] {
`
401
``
`-
if self.start > self.end {
`
402
``
`-
slice_index_order_fail(self.start, self.end);
`
403
``
`-
} else if self.end > slice.len() {
`
``
452
`` +
// Using checked_sub is a safe way to get SubUnchecked in MIR
``
``
453
`+
let Some(new_len) = usize::checked_sub(self.end, self.start) else {
`
``
454
`+
slice_index_order_fail(self.start, self.end)
`
``
455
`+
};
`
``
456
`+
if self.end > slice.len() {
`
404
457
`slice_end_index_len_fail(self.end, slice.len());
`
405
458
`}
`
406
459
`` // SAFETY: self is checked to be valid and in bounds above.
``
407
``
`-
unsafe { &*self.get_unchecked(slice) }
`
``
460
`+
unsafe { &*get_offset_len_noubcheck(slice, self.start, new_len) }
`
408
461
`}
`
409
462
``
410
463
`#[inline]
`
411
464
`fn index_mut(self, slice: &mut [T]) -> &mut [T] {
`
412
``
`-
if self.start > self.end {
`
413
``
`-
slice_index_order_fail(self.start, self.end);
`
414
``
`-
} else if self.end > slice.len() {
`
``
465
`+
let Some(new_len) = usize::checked_sub(self.end, self.start) else {
`
``
466
`+
slice_index_order_fail(self.start, self.end)
`
``
467
`+
};
`
``
468
`+
if self.end > slice.len() {
`
415
469
`slice_end_index_len_fail(self.end, slice.len());
`
416
470
`}
`
417
471
`` // SAFETY: self is checked to be valid and in bounds above.
``
418
``
`-
unsafe { &mut *self.get_unchecked_mut(slice) }
`
``
472
`+
unsafe { &mut *get_offset_len_mut_noubcheck(slice, self.start, new_len) }
`
419
473
`}
`
420
474
`}
`
421
475
``