Less syscalls for the copy_file_range probe · model-checking/verify-rust-std@fe05246 (original) (raw)
`@@ -560,6 +560,12 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) ->
`
560
560
`// We store the availability in a global to avoid unnecessary syscalls
`
561
561
`static HAS_COPY_FILE_RANGE: AtomicU8 = AtomicU8::new(NOT_PROBED);
`
562
562
``
``
563
`+
let mut have_probed = match HAS_COPY_FILE_RANGE.load(Ordering::Relaxed) {
`
``
564
`+
NOT_PROBED => false,
`
``
565
`+
UNAVAILABLE => return CopyResult::Fallback(0),
`
``
566
`+
_ => true,
`
``
567
`+
};
`
``
568
+
563
569
`syscall! {
`
564
570
`fn copy_file_range(
`
565
571
` fd_in: libc::c_int,
`
`@@ -571,26 +577,6 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) ->
`
571
577
`) -> libc::ssize_t
`
572
578
`}
`
573
579
``
574
``
`-
match HAS_COPY_FILE_RANGE.load(Ordering::Relaxed) {
`
575
``
`-
NOT_PROBED => {
`
576
``
`-
// EPERM can indicate seccomp filters or an immutable file.
`
577
``
`-
// To distinguish these cases we probe with invalid file descriptors which should result in EBADF if the syscall is supported
`
578
``
`-
// and some other error (ENOSYS or EPERM) if it's not available
`
579
``
`-
let result = unsafe {
`
580
``
`-
cvt(copy_file_range(INVALID_FD, ptr::null_mut(), INVALID_FD, ptr::null_mut(), 1, 0))
`
581
``
`-
};
`
582
``
-
583
``
`-
if matches!(result.map_err(|e| e.raw_os_error()), Err(Some(EBADF))) {
`
584
``
`-
HAS_COPY_FILE_RANGE.store(AVAILABLE, Ordering::Relaxed);
`
585
``
`-
} else {
`
586
``
`-
HAS_COPY_FILE_RANGE.store(UNAVAILABLE, Ordering::Relaxed);
`
587
``
`-
return CopyResult::Fallback(0);
`
588
``
`-
}
`
589
``
`-
}
`
590
``
`-
UNAVAILABLE => return CopyResult::Fallback(0),
`
591
``
`-
_ => {}
`
592
``
`-
};
`
593
``
-
594
580
`let mut written = 0u64;
`
595
581
`while written < max_len {
`
596
582
`let bytes_to_copy = cmp::min(max_len - written, usize::MAX as u64);
`
`@@ -604,6 +590,11 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) ->
`
604
590
`cvt(copy_file_range(reader, ptr::null_mut(), writer, ptr::null_mut(), bytes_to_copy, 0))
`
605
591
`};
`
606
592
``
``
593
`+
if !have_probed && copy_result.is_ok() {
`
``
594
`+
have_probed = true;
`
``
595
`+
HAS_COPY_FILE_RANGE.store(AVAILABLE, Ordering::Relaxed);
`
``
596
`+
}
`
``
597
+
607
598
`match copy_result {
`
608
599
`Ok(0) if written == 0 => {
`
609
600
`// fallback to work around several kernel bugs where copy_file_range will fail to
`
`@@ -616,10 +607,44 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) ->
`
616
607
`Ok(0) => return CopyResult::Ended(written), // reached EOF
`
617
608
`Ok(ret) => written += ret as u64,
`
618
609
`Err(err) => {
`
619
``
`-
return match err.raw_os_error() {
`
``
610
`+
let raw_os_error = match err.raw_os_error() {
`
``
611
`+
Some(raw) => raw,
`
``
612
`+
_ => return CopyResult::Error(err, written),
`
``
613
`+
};
`
``
614
`+
return match raw_os_error {
`
620
615
`// when file offset + max_length > u64::MAX
`
621
``
`-
Some(EOVERFLOW) => CopyResult::Fallback(written),
`
622
``
`-
Some(ENOSYS | EXDEV | EINVAL | EPERM | EOPNOTSUPP | EBADF) if written == 0 => {
`
``
616
`+
EOVERFLOW => CopyResult::Fallback(written),
`
``
617
`+
ENOSYS | EXDEV | EINVAL | EPERM | EOPNOTSUPP | EBADF if written == 0 => {
`
``
618
`+
if !have_probed {
`
``
619
`+
if raw_os_error == ENOSYS {
`
``
620
`+
HAS_COPY_FILE_RANGE.store(UNAVAILABLE, Ordering::Relaxed);
`
``
621
`+
} else {
`
``
622
`+
// EPERM can indicate seccomp filters or an
`
``
623
`+
// immutable file. To distinguish these cases
`
``
624
`+
// we probe with invalid file descriptors which
`
``
625
`+
// should result in EBADF if the syscall is
`
``
626
`+
// supported and some other error (ENOSYS or
`
``
627
`+
// EPERM) if it's not available.
`
``
628
`+
let result = unsafe {
`
``
629
`+
cvt(copy_file_range(
`
``
630
`+
INVALID_FD,
`
``
631
`+
ptr::null_mut(),
`
``
632
`+
INVALID_FD,
`
``
633
`+
ptr::null_mut(),
`
``
634
`+
1,
`
``
635
`+
0,
`
``
636
`+
))
`
``
637
`+
};
`
``
638
+
``
639
`+
if matches!(result.map_err(|e| e.raw_os_error()), Err(Some(EBADF)))
`
``
640
`+
{
`
``
641
`+
HAS_COPY_FILE_RANGE.store(AVAILABLE, Ordering::Relaxed);
`
``
642
`+
} else {
`
``
643
`+
HAS_COPY_FILE_RANGE.store(UNAVAILABLE, Ordering::Relaxed);
`
``
644
`+
}
`
``
645
`+
}
`
``
646
`+
}
`
``
647
+
623
648
`// Try fallback io::copy if either:
`
624
649
`// - Kernel version is < 4.5 (ENOSYS¹)
`
625
650
`// - Files are mounted on different fs (EXDEV)
`