Less syscalls for the copy_file_range probe · model-checking/verify-rust-std@fe05246 (original) (raw)

`@@ -560,6 +560,12 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) ->

`

560

560

`// We store the availability in a global to avoid unnecessary syscalls

`

561

561

`static HAS_COPY_FILE_RANGE: AtomicU8 = AtomicU8::new(NOT_PROBED);

`

562

562

``

``

563

`+

let mut have_probed = match HAS_COPY_FILE_RANGE.load(Ordering::Relaxed) {

`

``

564

`+

NOT_PROBED => false,

`

``

565

`+

UNAVAILABLE => return CopyResult::Fallback(0),

`

``

566

`+

_ => true,

`

``

567

`+

};

`

``

568

+

563

569

`syscall! {

`

564

570

`fn copy_file_range(

`

565

571

` fd_in: libc::c_int,

`

`@@ -571,26 +577,6 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) ->

`

571

577

`) -> libc::ssize_t

`

572

578

`}

`

573

579

``

574

``

`-

match HAS_COPY_FILE_RANGE.load(Ordering::Relaxed) {

`

575

``

`-

NOT_PROBED => {

`

576

``

`-

// EPERM can indicate seccomp filters or an immutable file.

`

577

``

`-

// To distinguish these cases we probe with invalid file descriptors which should result in EBADF if the syscall is supported

`

578

``

`-

// and some other error (ENOSYS or EPERM) if it's not available

`

579

``

`-

let result = unsafe {

`

580

``

`-

cvt(copy_file_range(INVALID_FD, ptr::null_mut(), INVALID_FD, ptr::null_mut(), 1, 0))

`

581

``

`-

};

`

582

``

-

583

``

`-

if matches!(result.map_err(|e| e.raw_os_error()), Err(Some(EBADF))) {

`

584

``

`-

HAS_COPY_FILE_RANGE.store(AVAILABLE, Ordering::Relaxed);

`

585

``

`-

} else {

`

586

``

`-

HAS_COPY_FILE_RANGE.store(UNAVAILABLE, Ordering::Relaxed);

`

587

``

`-

return CopyResult::Fallback(0);

`

588

``

`-

}

`

589

``

`-

}

`

590

``

`-

UNAVAILABLE => return CopyResult::Fallback(0),

`

591

``

`-

_ => {}

`

592

``

`-

};

`

593

``

-

594

580

`let mut written = 0u64;

`

595

581

`while written < max_len {

`

596

582

`let bytes_to_copy = cmp::min(max_len - written, usize::MAX as u64);

`

`@@ -604,6 +590,11 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) ->

`

604

590

`cvt(copy_file_range(reader, ptr::null_mut(), writer, ptr::null_mut(), bytes_to_copy, 0))

`

605

591

`};

`

606

592

``

``

593

`+

if !have_probed && copy_result.is_ok() {

`

``

594

`+

have_probed = true;

`

``

595

`+

HAS_COPY_FILE_RANGE.store(AVAILABLE, Ordering::Relaxed);

`

``

596

`+

}

`

``

597

+

607

598

`match copy_result {

`

608

599

`Ok(0) if written == 0 => {

`

609

600

`// fallback to work around several kernel bugs where copy_file_range will fail to

`

`@@ -616,10 +607,44 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) ->

`

616

607

`Ok(0) => return CopyResult::Ended(written), // reached EOF

`

617

608

`Ok(ret) => written += ret as u64,

`

618

609

`Err(err) => {

`

619

``

`-

return match err.raw_os_error() {

`

``

610

`+

let raw_os_error = match err.raw_os_error() {

`

``

611

`+

Some(raw) => raw,

`

``

612

`+

_ => return CopyResult::Error(err, written),

`

``

613

`+

};

`

``

614

`+

return match raw_os_error {

`

620

615

`// when file offset + max_length > u64::MAX

`

621

``

`-

Some(EOVERFLOW) => CopyResult::Fallback(written),

`

622

``

`-

Some(ENOSYS | EXDEV | EINVAL | EPERM | EOPNOTSUPP | EBADF) if written == 0 => {

`

``

616

`+

EOVERFLOW => CopyResult::Fallback(written),

`

``

617

`+

ENOSYS | EXDEV | EINVAL | EPERM | EOPNOTSUPP | EBADF if written == 0 => {

`

``

618

`+

if !have_probed {

`

``

619

`+

if raw_os_error == ENOSYS {

`

``

620

`+

HAS_COPY_FILE_RANGE.store(UNAVAILABLE, Ordering::Relaxed);

`

``

621

`+

} else {

`

``

622

`+

// EPERM can indicate seccomp filters or an

`

``

623

`+

// immutable file. To distinguish these cases

`

``

624

`+

// we probe with invalid file descriptors which

`

``

625

`+

// should result in EBADF if the syscall is

`

``

626

`+

// supported and some other error (ENOSYS or

`

``

627

`+

// EPERM) if it's not available.

`

``

628

`+

let result = unsafe {

`

``

629

`+

cvt(copy_file_range(

`

``

630

`+

INVALID_FD,

`

``

631

`+

ptr::null_mut(),

`

``

632

`+

INVALID_FD,

`

``

633

`+

ptr::null_mut(),

`

``

634

`+

1,

`

``

635

`+

0,

`

``

636

`+

))

`

``

637

`+

};

`

``

638

+

``

639

`+

if matches!(result.map_err(|e| e.raw_os_error()), Err(Some(EBADF)))

`

``

640

`+

{

`

``

641

`+

HAS_COPY_FILE_RANGE.store(AVAILABLE, Ordering::Relaxed);

`

``

642

`+

} else {

`

``

643

`+

HAS_COPY_FILE_RANGE.store(UNAVAILABLE, Ordering::Relaxed);

`

``

644

`+

}

`

``

645

`+

}

`

``

646

`+

}

`

``

647

+

623

648

`// Try fallback io::copy if either:

`

624

649

`// - Kernel version is < 4.5 (ENOSYS¹)

`

625

650

`// - Files are mounted on different fs (EXDEV)

`