vendor/rustix-0.37.6/src/backend/linux_raw/io/syscalls.rs

   1 //! linux_raw syscalls supporting `rustix::io`.
   2 //!
   3 //! # Safety
   4 //!
   5 //! See the `rustix::backend` module documentation for details.
   6 #![allow(unsafe_code)]
   7 #![allow(clippy::undocumented_unsafe_blocks)]
   8
   9 use super::super::c;
  10 #[cfg(target_pointer_width = "64")]
  11 use super::super::conv::loff_t_from_u64;
  12 use super::super::conv::{
  13     by_ref, c_int, c_uint, opt_mut, pass_usize, raw_fd, ret, ret_c_uint, ret_discarded_fd,
  14     ret_owned_fd, ret_usize, slice, slice_mut, zero,
  15 };
  16 #[cfg(target_pointer_width = "32")]
  17 use super::super::conv::{hi, lo};
  18 use crate::fd::{AsFd, BorrowedFd, OwnedFd, RawFd};
  19 #[cfg(any(target_os = "android", target_os = "linux"))]
  20 use crate::io::SpliceFlags;
  21 use crate::io::{
  22     self, epoll, DupFlags, EventfdFlags, FdFlags, IoSlice, IoSliceMut, IoSliceRaw, PipeFlags,
  23     PollFd, ReadWriteFlags,
  24 };
  25 #[cfg(all(feature = "fs", feature = "net"))]
  26 use crate::net::{RecvFlags, SendFlags};
  27 use core::cmp;
  28 use core::mem::MaybeUninit;
  29 #[cfg(target_os = "espidf")]
  30 use linux_raw_sys::general::F_DUPFD;
  31 use linux_raw_sys::general::{
  32     epoll_event, EPOLL_CTL_ADD, EPOLL_CTL_DEL, EPOLL_CTL_MOD, F_DUPFD_CLOEXEC, F_GETFD, F_SETFD,
  33     UIO_MAXIOV,
  34 };
  35 use linux_raw_sys::ioctl::{BLKPBSZGET, BLKSSZGET, FICLONE, FIONBIO, FIONREAD, TIOCEXCL, TIOCNXCL};
  36 #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))]
  37 use {
  38     super::super::conv::{opt_ref, size_of},
  39     linux_raw_sys::general::{__kernel_timespec, sigset_t},
  40 };
  41
  42 #[inline]
  43 pub(crate) fn read(fd: BorrowedFd<'_>, buf: &mut [u8]) -> io::Result<usize> {
  44     let (buf_addr_mut, buf_len) = slice_mut(buf);
  45
  46     unsafe { ret_usize(syscall!(__NR_read, fd, buf_addr_mut, buf_len)) }
  47 }
  48
  49 #[inline]
  50 pub(crate) fn pread(fd: BorrowedFd<'_>, buf: &mut [u8], pos: u64) -> io::Result<usize> {
  51     let (buf_addr_mut, buf_len) = slice_mut(buf);
  52
  53     // <https://github.com/torvalds/linux/blob/fcadab740480e0e0e9fa9bd272acd409884d431a/arch/arm64/kernel/sys32.c#L75>
  54     #[cfg(all(
  55         target_pointer_width = "32",
  56         any(target_arch = "arm", target_arch = "mips", target_arch = "power"),
  57     ))]
  58     unsafe {
  59         ret_usize(syscall!(
  60             __NR_pread64,
  61             fd,
  62             buf_addr_mut,
  63             buf_len,
  64             zero(),
  65             hi(pos),
  66             lo(pos)
  67         ))
  68     }
  69     #[cfg(all(
  70         target_pointer_width = "32",
  71         not(any(target_arch = "arm", target_arch = "mips", target_arch = "power")),
  72     ))]
  73     unsafe {
  74         ret_usize(syscall!(
  75             __NR_pread64,
  76             fd,
  77             buf_addr_mut,
  78             buf_len,
  79             hi(pos),
  80             lo(pos)
  81         ))
  82     }
  83     #[cfg(target_pointer_width = "64")]
  84     unsafe {
  85         ret_usize(syscall!(
  86             __NR_pread64,
  87             fd,
  88             buf_addr_mut,
  89             buf_len,
  90             loff_t_from_u64(pos)
  91         ))
  92     }
  93 }
  94
  95 #[inline]
  96 pub(crate) fn readv(fd: BorrowedFd<'_>, bufs: &mut [IoSliceMut<'_>]) -> io::Result<usize> {
  97     let (bufs_addr, bufs_len) = slice(&bufs[..cmp::min(bufs.len(), max_iov())]);
  98
  99     unsafe { ret_usize(syscall!(__NR_readv, fd, bufs_addr, bufs_len)) }
 100 }
 101
 102 #[inline]
 103 pub(crate) fn preadv(
 104     fd: BorrowedFd<'_>,
 105     bufs: &mut [IoSliceMut<'_>],
 106     pos: u64,
 107 ) -> io::Result<usize> {
 108     let (bufs_addr, bufs_len) = slice(&bufs[..cmp::min(bufs.len(), max_iov())]);
 109
 110     #[cfg(target_pointer_width = "32")]
 111     unsafe {
 112         ret_usize(syscall!(
 113             __NR_preadv,
 114             fd,
 115             bufs_addr,
 116             bufs_len,
 117             hi(pos),
 118             lo(pos)
 119         ))
 120     }
 121     #[cfg(target_pointer_width = "64")]
 122     unsafe {
 123         ret_usize(syscall!(
 124             __NR_preadv,
 125             fd,
 126             bufs_addr,
 127             bufs_len,
 128             loff_t_from_u64(pos)
 129         ))
 130     }
 131 }
 132
 133 #[inline]
 134 pub(crate) fn preadv2(
 135     fd: BorrowedFd<'_>,
 136     bufs: &mut [IoSliceMut<'_>],
 137     pos: u64,
 138     flags: ReadWriteFlags,
 139 ) -> io::Result<usize> {
 140     let (bufs_addr, bufs_len) = slice(&bufs[..cmp::min(bufs.len(), max_iov())]);
 141
 142     #[cfg(target_pointer_width = "32")]
 143     unsafe {
 144         ret_usize(syscall!(
 145             __NR_preadv2,
 146             fd,
 147             bufs_addr,
 148             bufs_len,
 149             hi(pos),
 150             lo(pos),
 151             flags
 152         ))
 153     }
 154     #[cfg(target_pointer_width = "64")]
 155     unsafe {
 156         ret_usize(syscall!(
 157             __NR_preadv2,
 158             fd,
 159             bufs_addr,
 160             bufs_len,
 161             loff_t_from_u64(pos),
 162             flags
 163         ))
 164     }
 165 }
 166
 167 #[inline]
 168 pub(crate) fn write(fd: BorrowedFd<'_>, buf: &[u8]) -> io::Result<usize> {
 169     let (buf_addr, buf_len) = slice(buf);
 170
 171     unsafe { ret_usize(syscall_readonly!(__NR_write, fd, buf_addr, buf_len)) }
 172 }
 173
 174 #[inline]
 175 pub(crate) fn pwrite(fd: BorrowedFd<'_>, buf: &[u8], pos: u64) -> io::Result<usize> {
 176     let (buf_addr, buf_len) = slice(buf);
 177
 178     // <https://github.com/torvalds/linux/blob/fcadab740480e0e0e9fa9bd272acd409884d431a/arch/arm64/kernel/sys32.c#L81-L83>
 179     #[cfg(all(
 180         target_pointer_width = "32",
 181         any(target_arch = "arm", target_arch = "mips", target_arch = "power"),
 182     ))]
 183     unsafe {
 184         ret_usize(syscall_readonly!(
 185             __NR_pwrite64,
 186             fd,
 187             buf_addr,
 188             buf_len,
 189             zero(),
 190             hi(pos),
 191             lo(pos)
 192         ))
 193     }
 194     #[cfg(all(
 195         target_pointer_width = "32",
 196         not(any(target_arch = "arm", target_arch = "mips", target_arch = "power")),
 197     ))]
 198     unsafe {
 199         ret_usize(syscall_readonly!(
 200             __NR_pwrite64,
 201             fd,
 202             buf_addr,
 203             buf_len,
 204             hi(pos),
 205             lo(pos)
 206         ))
 207     }
 208     #[cfg(target_pointer_width = "64")]
 209     unsafe {
 210         ret_usize(syscall_readonly!(
 211             __NR_pwrite64,
 212             fd,
 213             buf_addr,
 214             buf_len,
 215             loff_t_from_u64(pos)
 216         ))
 217     }
 218 }
 219
 220 #[inline]
 221 pub(crate) fn writev(fd: BorrowedFd<'_>, bufs: &[IoSlice<'_>]) -> io::Result<usize> {
 222     let (bufs_addr, bufs_len) = slice(&bufs[..cmp::min(bufs.len(), max_iov())]);
 223
 224     unsafe { ret_usize(syscall_readonly!(__NR_writev, fd, bufs_addr, bufs_len)) }
 225 }
 226
 227 #[inline]
 228 pub(crate) fn pwritev(fd: BorrowedFd<'_>, bufs: &[IoSlice<'_>], pos: u64) -> io::Result<usize> {
 229     let (bufs_addr, bufs_len) = slice(&bufs[..cmp::min(bufs.len(), max_iov())]);
 230
 231     #[cfg(target_pointer_width = "32")]
 232     unsafe {
 233         ret_usize(syscall_readonly!(
 234             __NR_pwritev,
 235             fd,
 236             bufs_addr,
 237             bufs_len,
 238             hi(pos),
 239             lo(pos)
 240         ))
 241     }
 242     #[cfg(target_pointer_width = "64")]
 243     unsafe {
 244         ret_usize(syscall_readonly!(
 245             __NR_pwritev,
 246             fd,
 247             bufs_addr,
 248             bufs_len,
 249             loff_t_from_u64(pos)
 250         ))
 251     }
 252 }
 253
 254 #[inline]
 255 pub(crate) fn pwritev2(
 256     fd: BorrowedFd<'_>,
 257     bufs: &[IoSlice<'_>],
 258     pos: u64,
 259     flags: ReadWriteFlags,
 260 ) -> io::Result<usize> {
 261     let (bufs_addr, bufs_len) = slice(&bufs[..cmp::min(bufs.len(), max_iov())]);
 262
 263     #[cfg(target_pointer_width = "32")]
 264     unsafe {
 265         ret_usize(syscall_readonly!(
 266             __NR_pwritev2,
 267             fd,
 268             bufs_addr,
 269             bufs_len,
 270             hi(pos),
 271             lo(pos),
 272             flags
 273         ))
 274     }
 275     #[cfg(target_pointer_width = "64")]
 276     unsafe {
 277         ret_usize(syscall_readonly!(
 278             __NR_pwritev2,
 279             fd,
 280             bufs_addr,
 281             bufs_len,
 282             loff_t_from_u64(pos),
 283             flags
 284         ))
 285     }
 286 }
 287
 288 /// The maximum number of buffers that can be passed into a vectored I/O system
 289 /// call on the current platform.
 290 const fn max_iov() -> usize {
 291     UIO_MAXIOV as usize
 292 }
 293
 294 #[inline]
 295 pub(crate) unsafe fn close(fd: RawFd) {
 296     // See the documentation for [`io::close`] for why errors are ignored.
 297     syscall_readonly!(__NR_close, raw_fd(fd)).decode_void();
 298 }
 299
 300 #[inline]
 301 pub(crate) fn eventfd(initval: u32, flags: EventfdFlags) -> io::Result<OwnedFd> {
 302     unsafe { ret_owned_fd(syscall_readonly!(__NR_eventfd2, c_uint(initval), flags)) }
 303 }
 304
 305 #[inline]
 306 pub(crate) fn ioctl_fionread(fd: BorrowedFd<'_>) -> io::Result<u64> {
 307     unsafe {
 308         let mut result = MaybeUninit::<c::c_int>::uninit();
 309         ret(syscall!(__NR_ioctl, fd, c_uint(FIONREAD), &mut result))?;
 310         Ok(result.assume_init() as u64)
 311     }
 312 }
 313
 314 #[inline]
 315 pub(crate) fn ioctl_fionbio(fd: BorrowedFd<'_>, value: bool) -> io::Result<()> {
 316     unsafe {
 317         let data = c::c_int::from(value);
 318         ret(syscall_readonly!(
 319             __NR_ioctl,
 320             fd,
 321             c_uint(FIONBIO),
 322             by_ref(&data)
 323         ))
 324     }
 325 }
 326
 327 #[inline]
 328 pub(crate) fn ioctl_tiocexcl(fd: BorrowedFd<'_>) -> io::Result<()> {
 329     unsafe { ret(syscall_readonly!(__NR_ioctl, fd, c_uint(TIOCEXCL))) }
 330 }
 331
 332 #[inline]
 333 pub(crate) fn ioctl_tiocnxcl(fd: BorrowedFd<'_>) -> io::Result<()> {
 334     unsafe { ret(syscall_readonly!(__NR_ioctl, fd, c_uint(TIOCNXCL))) }
 335 }
 336
 337 #[inline]
 338 pub(crate) fn ioctl_blksszget(fd: BorrowedFd) -> io::Result<u32> {
 339     let mut result = MaybeUninit::<c::c_uint>::uninit();
 340     unsafe {
 341         ret(syscall!(__NR_ioctl, fd, c_uint(BLKSSZGET), &mut result))?;
 342         Ok(result.assume_init() as u32)
 343     }
 344 }
 345
 346 #[inline]
 347 pub(crate) fn ioctl_blkpbszget(fd: BorrowedFd) -> io::Result<u32> {
 348     let mut result = MaybeUninit::<c::c_uint>::uninit();
 349     unsafe {
 350         ret(syscall!(__NR_ioctl, fd, c_uint(BLKPBSZGET), &mut result))?;
 351         Ok(result.assume_init() as u32)
 352     }
 353 }
 354
 355 #[inline]
 356 pub(crate) fn ioctl_ficlone(fd: BorrowedFd<'_>, src_fd: BorrowedFd<'_>) -> io::Result<()> {
 357     unsafe { ret(syscall_readonly!(__NR_ioctl, fd, c_uint(FICLONE), src_fd)) }
 358 }
 359
 360 #[cfg(all(feature = "fs", feature = "net"))]
 361 pub(crate) fn is_read_write(fd: BorrowedFd<'_>) -> io::Result<(bool, bool)> {
 362     let (mut read, mut write) = crate::fs::fd::_is_file_read_write(fd)?;
 363     let mut not_socket = false;
 364     if read {
 365         // Do a `recv` with `PEEK` and `DONTWAIT` for 1 byte. A 0 indicates
 366         // the read side is shut down; an `EWOULDBLOCK` indicates the read
 367         // side is still open.
 368         //
 369         // TODO: This code would benefit from having a better way to read into
 370         // uninitialized memory.
 371         let mut buf = [0];
 372         match super::super::net::syscalls::recv(fd, &mut buf, RecvFlags::PEEK | RecvFlags::DONTWAIT)
 373         {
 374             Ok(0) => read = false,
 375             Err(err) => {
 376                 #[allow(unreachable_patterns)] // `EAGAIN` may equal `EWOULDBLOCK`
 377                 match err {
 378                     io::Errno::AGAIN | io::Errno::WOULDBLOCK => (),
 379                     io::Errno::NOTSOCK => not_socket = true,
 380                     _ => return Err(err),
 381                 }
 382             }
 383             Ok(_) => (),
 384         }
 385     }
 386     if write && !not_socket {
 387         // Do a `send` with `DONTWAIT` for 0 bytes. An `EPIPE` indicates
 388         // the write side is shut down.
 389         #[allow(unreachable_patterns)] // `EAGAIN` equals `EWOULDBLOCK`
 390         match super::super::net::syscalls::send(fd, &[], SendFlags::DONTWAIT) {
 391             // TODO or-patterns when we don't need 1.51
 392             Err(io::Errno::AGAIN) => (),
 393             Err(io::Errno::WOULDBLOCK) => (),
 394             Err(io::Errno::NOTSOCK) => (),
 395             Err(io::Errno::PIPE) => write = false,
 396             Err(err) => return Err(err),
 397             Ok(_) => (),
 398         }
 399     }
 400     Ok((read, write))
 401 }
 402
 403 #[inline]
 404 pub(crate) fn dup(fd: BorrowedFd<'_>) -> io::Result<OwnedFd> {
 405     unsafe { ret_owned_fd(syscall_readonly!(__NR_dup, fd)) }
 406 }
 407
 408 #[inline]
 409 pub(crate) fn dup2(fd: BorrowedFd<'_>, new: &mut OwnedFd) -> io::Result<()> {
 410     #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))]
 411     {
 412         // We don't need to worry about the difference between `dup2` and
 413         // `dup3` when the file descriptors are equal because we have an
 414         // `&mut OwnedFd` which means `fd` doesn't alias it.
 415         dup3(fd, new, DupFlags::empty())
 416     }
 417
 418     #[cfg(not(any(target_arch = "aarch64", target_arch = "riscv64")))]
 419     unsafe {
 420         ret_discarded_fd(syscall_readonly!(__NR_dup2, fd, new.as_fd()))
 421     }
 422 }
 423
 424 #[inline]
 425 pub(crate) fn dup3(fd: BorrowedFd<'_>, new: &mut OwnedFd, flags: DupFlags) -> io::Result<()> {
 426     unsafe { ret_discarded_fd(syscall_readonly!(__NR_dup3, fd, new.as_fd(), flags)) }
 427 }
 428
 429 #[inline]
 430 pub(crate) fn fcntl_getfd(fd: BorrowedFd<'_>) -> io::Result<FdFlags> {
 431     #[cfg(target_pointer_width = "32")]
 432     unsafe {
 433         ret_c_uint(syscall_readonly!(__NR_fcntl64, fd, c_uint(F_GETFD)))
 434             .map(FdFlags::from_bits_truncate)
 435     }
 436     #[cfg(target_pointer_width = "64")]
 437     unsafe {
 438         ret_c_uint(syscall_readonly!(__NR_fcntl, fd, c_uint(F_GETFD)))
 439             .map(FdFlags::from_bits_truncate)
 440     }
 441 }
 442
 443 #[inline]
 444 pub(crate) fn fcntl_setfd(fd: BorrowedFd<'_>, flags: FdFlags) -> io::Result<()> {
 445     #[cfg(target_pointer_width = "32")]
 446     unsafe {
 447         ret(syscall_readonly!(__NR_fcntl64, fd, c_uint(F_SETFD), flags))
 448     }
 449     #[cfg(target_pointer_width = "64")]
 450     unsafe {
 451         ret(syscall_readonly!(__NR_fcntl, fd, c_uint(F_SETFD), flags))
 452     }
 453 }
 454
 455 #[cfg(target_os = "espidf")]
 456 #[inline]
 457 pub(crate) fn fcntl_dupfd(fd: BorrowedFd<'_>, min: RawFd) -> io::Result<OwnedFd> {
 458     #[cfg(target_pointer_width = "32")]
 459     unsafe {
 460         ret_owned_fd(syscall_readonly!(
 461             __NR_fcntl64,
 462             fd,
 463             c_uint(F_DUPFD),
 464             raw_fd(min)
 465         ))
 466     }
 467     #[cfg(target_pointer_width = "64")]
 468     unsafe {
 469         ret_owned_fd(syscall_readonly!(
 470             __NR_fcntl,
 471             fd,
 472             c_uint(F_DUPFD),
 473             raw_fd(min)
 474         ))
 475     }
 476 }
 477
 478 #[inline]
 479 pub(crate) fn fcntl_dupfd_cloexec(fd: BorrowedFd<'_>, min: RawFd) -> io::Result<OwnedFd> {
 480     #[cfg(target_pointer_width = "32")]
 481     unsafe {
 482         ret_owned_fd(syscall_readonly!(
 483             __NR_fcntl64,
 484             fd,
 485             c_uint(F_DUPFD_CLOEXEC),
 486             raw_fd(min)
 487         ))
 488     }
 489     #[cfg(target_pointer_width = "64")]
 490     unsafe {
 491         ret_owned_fd(syscall_readonly!(
 492             __NR_fcntl,
 493             fd,
 494             c_uint(F_DUPFD_CLOEXEC),
 495             raw_fd(min)
 496         ))
 497     }
 498 }
 499
 500 #[inline]
 501 pub(crate) fn pipe_with(flags: PipeFlags) -> io::Result<(OwnedFd, OwnedFd)> {
 502     unsafe {
 503         let mut result = MaybeUninit::<[OwnedFd; 2]>::uninit();
 504         ret(syscall!(__NR_pipe2, &mut result, flags))?;
 505         let [p0, p1] = result.assume_init();
 506         Ok((p0, p1))
 507     }
 508 }
 509
 510 #[inline]
 511 pub(crate) fn pipe() -> io::Result<(OwnedFd, OwnedFd)> {
 512     // aarch64 and risc64 omit `__NR_pipe`. On mips, `__NR_pipe` uses a special
 513     // calling convention, but using it is not worth complicating our syscall
 514     // wrapping infrastructure at this time.
 515     #[cfg(any(
 516         target_arch = "aarch64",
 517         target_arch = "mips",
 518         target_arch = "mips64",
 519         target_arch = "riscv64",
 520     ))]
 521     {
 522         pipe_with(PipeFlags::empty())
 523     }
 524     #[cfg(not(any(
 525         target_arch = "aarch64",
 526         target_arch = "mips",
 527         target_arch = "mips64",
 528         target_arch = "riscv64",
 529     )))]
 530     unsafe {
 531         let mut result = MaybeUninit::<[OwnedFd; 2]>::uninit();
 532         ret(syscall!(__NR_pipe, &mut result))?;
 533         let [p0, p1] = result.assume_init();
 534         Ok((p0, p1))
 535     }
 536 }
 537
 538 #[inline]
 539 pub(crate) fn poll(fds: &mut [PollFd<'_>], timeout: c::c_int) -> io::Result<usize> {
 540     let (fds_addr_mut, fds_len) = slice_mut(fds);
 541
 542     #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))]
 543     unsafe {
 544         let timeout = if timeout >= 0 {
 545             Some(__kernel_timespec {
 546                 tv_sec: (timeout as i64) / 1000,
 547                 tv_nsec: (timeout as i64) % 1000 * 1_000_000,
 548             })
 549         } else {
 550             None
 551         };
 552         ret_usize(syscall!(
 553             __NR_ppoll,
 554             fds_addr_mut,
 555             fds_len,
 556             opt_ref(timeout.as_ref()),
 557             zero(),
 558             size_of::<sigset_t, _>()
 559         ))
 560     }
 561     #[cfg(not(any(target_arch = "aarch64", target_arch = "riscv64")))]
 562     unsafe {
 563         ret_usize(syscall!(__NR_poll, fds_addr_mut, fds_len, c_int(timeout)))
 564     }
 565 }
 566
 567 #[inline]
 568 pub(crate) fn epoll_create(flags: epoll::CreateFlags) -> io::Result<OwnedFd> {
 569     unsafe { ret_owned_fd(syscall_readonly!(__NR_epoll_create1, flags)) }
 570 }
 571
 572 #[inline]
 573 pub(crate) unsafe fn epoll_add(
 574     epfd: BorrowedFd<'_>,
 575     fd: c::c_int,
 576     event: &epoll_event,
 577 ) -> io::Result<()> {
 578     ret(syscall_readonly!(
 579         __NR_epoll_ctl,
 580         epfd,
 581         c_uint(EPOLL_CTL_ADD),
 582         raw_fd(fd),
 583         by_ref(event)
 584     ))
 585 }
 586
 587 #[inline]
 588 pub(crate) unsafe fn epoll_mod(
 589     epfd: BorrowedFd<'_>,
 590     fd: c::c_int,
 591     event: &epoll_event,
 592 ) -> io::Result<()> {
 593     ret(syscall_readonly!(
 594         __NR_epoll_ctl,
 595         epfd,
 596         c_uint(EPOLL_CTL_MOD),
 597         raw_fd(fd),
 598         by_ref(event)
 599     ))
 600 }
 601
 602 #[inline]
 603 pub(crate) unsafe fn epoll_del(epfd: BorrowedFd<'_>, fd: c::c_int) -> io::Result<()> {
 604     ret(syscall_readonly!(
 605         __NR_epoll_ctl,
 606         epfd,
 607         c_uint(EPOLL_CTL_DEL),
 608         raw_fd(fd),
 609         zero()
 610     ))
 611 }
 612
 613 #[inline]
 614 pub(crate) fn epoll_wait(
 615     epfd: BorrowedFd<'_>,
 616     events: *mut epoll_event,
 617     num_events: usize,
 618     timeout: c::c_int,
 619 ) -> io::Result<usize> {
 620     #[cfg(not(any(target_arch = "aarch64", target_arch = "riscv64")))]
 621     unsafe {
 622         ret_usize(syscall!(
 623             __NR_epoll_wait,
 624             epfd,
 625             events,
 626             pass_usize(num_events),
 627             c_int(timeout)
 628         ))
 629     }
 630     #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))]
 631     unsafe {
 632         ret_usize(syscall!(
 633             __NR_epoll_pwait,
 634             epfd,
 635             events,
 636             pass_usize(num_events),
 637             c_int(timeout),
 638             zero()
 639         ))
 640     }
 641 }
 642
 643 #[cfg(any(target_os = "android", target_os = "linux"))]
 644 #[inline]
 645 pub fn splice(
 646     fd_in: BorrowedFd,
 647     off_in: Option<&mut u64>,
 648     fd_out: BorrowedFd,
 649     off_out: Option<&mut u64>,
 650     len: usize,
 651     flags: SpliceFlags,
 652 ) -> io::Result<usize> {
 653     unsafe {
 654         ret_usize(syscall!(
 655             __NR_splice,
 656             fd_in,
 657             opt_mut(off_in),
 658             fd_out,
 659             opt_mut(off_out),
 660             pass_usize(len),
 661             c_uint(flags.bits())
 662         ))
 663     }
 664 }
 665
 666 #[cfg(any(target_os = "android", target_os = "linux"))]
 667 #[inline]
 668 pub unsafe fn vmsplice(
 669     fd: BorrowedFd,
 670     bufs: &[IoSliceRaw],
 671     flags: SpliceFlags,
 672 ) -> io::Result<usize> {
 673     let (bufs_addr, bufs_len) = slice(&bufs[..cmp::min(bufs.len(), max_iov())]);
 674     ret_usize(syscall!(
 675         __NR_vmsplice,
 676         fd,
 677         bufs_addr,
 678         bufs_len,
 679         c_uint(flags.bits())
 680     ))
 681 }