diff --git a/Cargo.lock b/Cargo.lock index 34b2dac52..cf710ea00 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3275,6 +3275,7 @@ dependencies = [ "getopts", "lazy_static", "libc", + "nix 0.20.0", "once_cell", "termion", "thiserror", diff --git a/src/uu/cat/Cargo.toml b/src/uu/cat/Cargo.toml index d80514385..d4f137d7e 100644 --- a/src/uu/cat/Cargo.toml +++ b/src/uu/cat/Cargo.toml @@ -18,7 +18,7 @@ path = "src/cat.rs" clap = { version = "2.33", features = ["wrap_help"] } thiserror = "1.0" atty = "0.2" -uucore = { version=">=0.0.9", package="uucore", path="../../uucore", features=["fs"] } +uucore = { version=">=0.0.9", package="uucore", path="../../uucore", features=["fs", "pipes"] } uucore_procs = { version=">=0.0.6", package="uucore_procs", path="../../uucore_procs" } [target.'cfg(unix)'.dependencies] diff --git a/src/uu/cat/src/cat.rs b/src/uu/cat/src/cat.rs index 9459ee86a..baf8af6d5 100644 --- a/src/uu/cat/src/cat.rs +++ b/src/uu/cat/src/cat.rs @@ -29,8 +29,6 @@ use std::os::unix::io::AsRawFd; /// Linux splice support #[cfg(any(target_os = "linux", target_os = "android"))] mod splice; -#[cfg(any(target_os = "linux", target_os = "android"))] -use std::os::unix::io::RawFd; /// Unix domain socket support #[cfg(unix)] @@ -137,10 +135,18 @@ struct OutputState { one_blank_kept: bool, } +#[cfg(unix)] +trait FdReadable: Read + AsRawFd {} +#[cfg(not(unix))] +trait FdReadable: Read {} + +#[cfg(unix)] +impl FdReadable for T where T: Read + AsRawFd {} +#[cfg(not(unix))] +impl FdReadable for T where T: Read {} + /// Represents an open file handle, stream, or other device -struct InputHandle { - #[cfg(any(target_os = "linux", target_os = "android"))] - file_descriptor: RawFd, +struct InputHandle { reader: R, is_interactive: bool, } @@ -297,7 +303,7 @@ pub fn uu_app() -> App<'static, 'static> { ) } -fn cat_handle( +fn cat_handle( handle: &mut InputHandle, options: &OutputOptions, state: &mut OutputState, @@ -319,8 +325,6 @@ fn cat_path( if path == "-" { let stdin = io::stdin(); let mut handle = InputHandle { - #[cfg(any(target_os = "linux", target_os = "android"))] - file_descriptor: stdin.as_raw_fd(), reader: stdin, is_interactive: atty::is(atty::Stream::Stdin), }; @@ -333,8 +337,6 @@ fn cat_path( let socket = UnixStream::connect(path)?; socket.shutdown(Shutdown::Write)?; let mut handle = InputHandle { - #[cfg(any(target_os = "linux", target_os = "android"))] - file_descriptor: socket.as_raw_fd(), reader: socket, is_interactive: false, }; @@ -347,8 +349,6 @@ fn cat_path( return Err(CatError::OutputIsInput); } let mut handle = InputHandle { - #[cfg(any(target_os = "linux", target_os = "android"))] - file_descriptor: file.as_raw_fd(), reader: file, is_interactive: false, }; @@ -437,14 +437,14 @@ fn get_input_type(path: &str) -> CatResult { /// Writes handle to stdout with no configuration. This allows a /// simple memory copy. -fn write_fast(handle: &mut InputHandle) -> CatResult<()> { +fn write_fast(handle: &mut InputHandle) -> CatResult<()> { let stdout = io::stdout(); let mut stdout_lock = stdout.lock(); #[cfg(any(target_os = "linux", target_os = "android"))] { // If we're on Linux or Android, try to use the splice() system call // for faster writing. If it works, we're done. - if !splice::write_fast_using_splice(handle, stdout_lock.as_raw_fd())? { + if !splice::write_fast_using_splice(handle, &stdout_lock)? { return Ok(()); } } @@ -462,7 +462,7 @@ fn write_fast(handle: &mut InputHandle) -> CatResult<()> { /// Outputs file contents to stdout in a line-by-line fashion, /// propagating any errors that might occur. -fn write_lines( +fn write_lines( handle: &mut InputHandle, options: &OutputOptions, state: &mut OutputState, diff --git a/src/uu/cat/src/splice.rs b/src/uu/cat/src/splice.rs index 108997c4a..26802c7e6 100644 --- a/src/uu/cat/src/splice.rs +++ b/src/uu/cat/src/splice.rs @@ -1,11 +1,11 @@ -use super::{CatResult, InputHandle}; +use super::{CatResult, FdReadable, InputHandle}; -use nix::fcntl::{splice, SpliceFFlags}; -use nix::unistd::{self, pipe}; -use std::fs::File; -use std::io::Read; -use std::os::unix::io::{FromRawFd, RawFd}; +use nix::unistd; +use std::os::unix::io::{AsRawFd, RawFd}; +use uucore::pipes::{pipe, splice, splice_exact}; + +const SPLICE_SIZE: usize = 1024 * 128; const BUF_SIZE: usize = 1024 * 16; /// This function is called from `write_fast()` on Linux and Android. The @@ -16,36 +16,25 @@ const BUF_SIZE: usize = 1024 * 16; /// The `bool` in the result value indicates if we need to fall back to normal /// copying or not. False means we don't have to. #[inline] -pub(super) fn write_fast_using_splice( +pub(super) fn write_fast_using_splice( handle: &mut InputHandle, - write_fd: RawFd, + write_fd: &impl AsRawFd, ) -> CatResult { let (pipe_rd, pipe_wr) = pipe()?; - // Ensure the pipe is closed when the function returns. - // SAFETY: The file descriptors do not have other owners. - let _handles = unsafe { (File::from_raw_fd(pipe_rd), File::from_raw_fd(pipe_wr)) }; - loop { - match splice( - handle.file_descriptor, - None, - pipe_wr, - None, - BUF_SIZE, - SpliceFFlags::empty(), - ) { + match splice(&handle.reader, &pipe_wr, SPLICE_SIZE) { Ok(n) => { if n == 0 { return Ok(false); } - if splice_exact(pipe_rd, write_fd, n).is_err() { + if splice_exact(&pipe_rd, write_fd, n).is_err() { // If the first splice manages to copy to the intermediate // pipe, but the second splice to stdout fails for some reason // we can recover by copying the data that we have from the // intermediate pipe to stdout using normal read/write. Then // we tell the caller to fall back. - copy_exact(pipe_rd, write_fd, n)?; + copy_exact(pipe_rd.as_raw_fd(), write_fd.as_raw_fd(), n)?; return Ok(true); } } @@ -56,35 +45,23 @@ pub(super) fn write_fast_using_splice( } } -/// Splice wrapper which handles short writes. -#[inline] -fn splice_exact(read_fd: RawFd, write_fd: RawFd, num_bytes: usize) -> nix::Result<()> { - let mut left = num_bytes; - loop { - let written = splice(read_fd, None, write_fd, None, left, SpliceFFlags::empty())?; - left -= written; - if left == 0 { - break; - } - } - Ok(()) -} - -/// Caller must ensure that `num_bytes <= BUF_SIZE`, otherwise this function -/// will panic. The way we use this function in `write_fast_using_splice` -/// above is safe because `splice` is set to write at most `BUF_SIZE` to the -/// pipe. -#[inline] +/// Move exactly `num_bytes` bytes from `read_fd` to `write_fd`. +/// +/// Panics if not enough bytes can be read. fn copy_exact(read_fd: RawFd, write_fd: RawFd, num_bytes: usize) -> nix::Result<()> { let mut left = num_bytes; let mut buf = [0; BUF_SIZE]; - loop { - let read = unistd::read(read_fd, &mut buf[..left])?; - let written = unistd::write(write_fd, &buf[..read])?; - left -= written; - if left == 0 { - break; + while left > 0 { + let read = unistd::read(read_fd, &mut buf)?; + assert_ne!(read, 0, "unexpected end of pipe"); + let mut written = 0; + while written < read { + match unistd::write(write_fd, &buf[written..read])? { + 0 => panic!(), + n => written += n, + } } + left -= read; } Ok(()) } diff --git a/src/uu/wc/Cargo.toml b/src/uu/wc/Cargo.toml index 80d6014da..5884f3746 100644 --- a/src/uu/wc/Cargo.toml +++ b/src/uu/wc/Cargo.toml @@ -16,7 +16,7 @@ path = "src/wc.rs" [dependencies] clap = { version = "2.33", features = ["wrap_help"] } -uucore = { version=">=0.0.9", package="uucore", path="../../uucore" } +uucore = { version=">=0.0.9", package="uucore", path="../../uucore", features=["pipes"] } uucore_procs = { version=">=0.0.6", package="uucore_procs", path="../../uucore_procs" } bytecount = "0.6.2" utf-8 = "0.7.6" diff --git a/src/uu/wc/src/count_fast.rs b/src/uu/wc/src/count_fast.rs index b4078d6be..9351b6871 100644 --- a/src/uu/wc/src/count_fast.rs +++ b/src/uu/wc/src/count_fast.rs @@ -3,7 +3,7 @@ use crate::word_count::WordCount; use super::WordCountable; #[cfg(any(target_os = "linux", target_os = "android"))] -use std::fs::{File, OpenOptions}; +use std::fs::OpenOptions; use std::io::{self, ErrorKind, Read}; #[cfg(unix)] @@ -11,34 +11,17 @@ use libc::S_IFREG; #[cfg(unix)] use nix::sys::stat; #[cfg(any(target_os = "linux", target_os = "android"))] -use std::os::unix::io::{AsRawFd, FromRawFd, RawFd}; +use std::os::unix::io::AsRawFd; #[cfg(any(target_os = "linux", target_os = "android"))] use libc::S_IFIFO; #[cfg(any(target_os = "linux", target_os = "android"))] -use nix::fcntl::{splice, SpliceFFlags}; -#[cfg(any(target_os = "linux", target_os = "android"))] -use nix::unistd::pipe; +use uucore::pipes::{pipe, splice, splice_exact}; const BUF_SIZE: usize = 16 * 1024; #[cfg(any(target_os = "linux", target_os = "android"))] const SPLICE_SIZE: usize = 128 * 1024; -/// Splice wrapper which handles short writes -#[cfg(any(target_os = "linux", target_os = "android"))] -#[inline] -fn splice_exact(read_fd: RawFd, write_fd: RawFd, num_bytes: usize) -> nix::Result<()> { - let mut left = num_bytes; - loop { - let written = splice(read_fd, None, write_fd, None, left, SpliceFFlags::empty())?; - left -= written; - if left == 0 { - break; - } - } - Ok(()) -} - /// This is a Linux-specific function to count the number of bytes using the /// `splice` system call, which is faster than using `read`. /// @@ -46,13 +29,14 @@ fn splice_exact(read_fd: RawFd, write_fd: RawFd, num_bytes: usize) -> nix::Resul /// caller will fall back to a simpler method. #[inline] #[cfg(any(target_os = "linux", target_os = "android"))] -fn count_bytes_using_splice(fd: RawFd) -> Result { +fn count_bytes_using_splice(fd: &impl AsRawFd) -> Result { let null_file = OpenOptions::new() .write(true) .open("/dev/null") .map_err(|_| 0_usize)?; - let null = null_file.as_raw_fd(); - let null_rdev = stat::fstat(null).map_err(|_| 0_usize)?.st_rdev; + let null_rdev = stat::fstat(null_file.as_raw_fd()) + .map_err(|_| 0_usize)? + .st_rdev; if (stat::major(null_rdev), stat::minor(null_rdev)) != (1, 3) { // This is not a proper /dev/null, writing to it is probably bad // Bit of an edge case, but it has been known to happen @@ -60,17 +44,13 @@ fn count_bytes_using_splice(fd: RawFd) -> Result { } let (pipe_rd, pipe_wr) = pipe().map_err(|_| 0_usize)?; - // Ensure the pipe is closed when the function returns. - // SAFETY: The file descriptors do not have other owners. - let _handles = unsafe { (File::from_raw_fd(pipe_rd), File::from_raw_fd(pipe_wr)) }; - let mut byte_count = 0; loop { - match splice(fd, None, pipe_wr, None, SPLICE_SIZE, SpliceFFlags::empty()) { + match splice(fd, &pipe_wr, SPLICE_SIZE) { Ok(0) => break, Ok(res) => { byte_count += res; - if splice_exact(pipe_rd, null, res).is_err() { + if splice_exact(&pipe_rd, &null_file, res).is_err() { return Err(byte_count); } } @@ -106,7 +86,7 @@ pub(crate) fn count_bytes_fast(handle: &mut T) -> (usize, Opti // Else, if we're on Linux and our file is a FIFO pipe // (or stdin), we use splice to count the number of bytes. if (stat.st_mode & S_IFIFO) != 0 { - match count_bytes_using_splice(fd) { + match count_bytes_using_splice(handle) { Ok(n) => return (n, None), Err(n) => byte_count = n, } diff --git a/src/uu/yes/Cargo.toml b/src/uu/yes/Cargo.toml index b963d4974..ea2aae3b1 100644 --- a/src/uu/yes/Cargo.toml +++ b/src/uu/yes/Cargo.toml @@ -16,7 +16,7 @@ path = "src/yes.rs" [dependencies] clap = { version = "2.33", features = ["wrap_help"] } -uucore = { version=">=0.0.9", package="uucore", path="../../uucore" } +uucore = { version=">=0.0.9", package="uucore", path="../../uucore", features=["pipes"] } uucore_procs = { version=">=0.0.6", package="uucore_procs", path="../../uucore_procs" } [target.'cfg(any(target_os = "linux", target_os = "android"))'.dependencies] diff --git a/src/uu/yes/src/splice.rs b/src/uu/yes/src/splice.rs index b0573bc9e..6f025d6a9 100644 --- a/src/uu/yes/src/splice.rs +++ b/src/uu/yes/src/splice.rs @@ -16,18 +16,11 @@ //! make any effort to rescue data from the pipe if splice() fails, we can //! just fall back and start over from the beginning. -use std::{ - fs::File, - io, - os::unix::io::{AsRawFd, FromRawFd}, -}; +use std::{io, os::unix::io::AsRawFd}; -use nix::{ - errno::Errno, - fcntl::SpliceFFlags, - libc::S_IFIFO, - sys::{stat::fstat, uio::IoVec}, -}; +use nix::{errno::Errno, libc::S_IFIFO, sys::stat::fstat}; + +use uucore::pipes::{pipe, splice_exact, vmsplice}; pub(crate) fn splice_data(bytes: &[u8], out: &impl AsRawFd) -> Result<()> { let is_pipe = fstat(out.as_raw_fd())?.st_mode & S_IFIFO != 0; @@ -36,7 +29,7 @@ pub(crate) fn splice_data(bytes: &[u8], out: &impl AsRawFd) -> Result<()> { loop { let mut bytes = bytes; while !bytes.is_empty() { - let len = vmsplice(out, bytes)?; + let len = vmsplice(out, bytes).map_err(maybe_unsupported)?; bytes = &bytes[len..]; } } @@ -45,14 +38,8 @@ pub(crate) fn splice_data(bytes: &[u8], out: &impl AsRawFd) -> Result<()> { loop { let mut bytes = bytes; while !bytes.is_empty() { - let len = vmsplice(&write, bytes)?; - let mut remaining = len; - while remaining > 0 { - match splice(&read, out, remaining)? { - 0 => panic!("Unexpected end of pipe"), - n => remaining -= n, - }; - } + let len = vmsplice(&write, bytes).map_err(maybe_unsupported)?; + splice_exact(&read, out, len).map_err(maybe_unsupported)?; bytes = &bytes[len..]; } } @@ -81,30 +68,3 @@ fn maybe_unsupported(error: nix::Error) -> Error { _ => error.into(), } } - -fn splice(source: &impl AsRawFd, target: &impl AsRawFd, len: usize) -> Result { - nix::fcntl::splice( - source.as_raw_fd(), - None, - target.as_raw_fd(), - None, - len, - SpliceFFlags::empty(), - ) - .map_err(maybe_unsupported) -} - -fn vmsplice(target: &impl AsRawFd, bytes: &[u8]) -> Result { - nix::fcntl::vmsplice( - target.as_raw_fd(), - &[IoVec::from_slice(bytes)], - SpliceFFlags::empty(), - ) - .map_err(maybe_unsupported) -} - -fn pipe() -> nix::Result<(File, File)> { - let (read, write) = nix::unistd::pipe()?; - // SAFETY: The file descriptors do not have other owners. - unsafe { Ok((File::from_raw_fd(read), File::from_raw_fd(write))) } -} diff --git a/src/uucore/Cargo.toml b/src/uucore/Cargo.toml index a04f565aa..80a3a115b 100644 --- a/src/uucore/Cargo.toml +++ b/src/uucore/Cargo.toml @@ -31,6 +31,7 @@ data-encoding-macro = { version="0.1.12", optional=true } z85 = { version="3.0.3", optional=true } libc = { version="0.2.15", optional=true } once_cell = "1.8.0" +nix = { version="0.20", optional=true } [dev-dependencies] clap = "2.33.3" @@ -57,3 +58,4 @@ signals = [] utf8 = [] utmpx = ["time", "libc", "dns-lookup"] wide = [] +pipes = ["nix"] diff --git a/src/uucore/src/lib/features.rs b/src/uucore/src/lib/features.rs index 60be88664..42d8ffbe7 100644 --- a/src/uucore/src/lib/features.rs +++ b/src/uucore/src/lib/features.rs @@ -19,6 +19,8 @@ pub mod mode; pub mod entries; #[cfg(all(unix, feature = "perms"))] pub mod perms; +#[cfg(all(unix, feature = "pipes"))] +pub mod pipes; #[cfg(all(unix, feature = "process"))] pub mod process; diff --git a/src/uucore/src/lib/features/pipes.rs b/src/uucore/src/lib/features/pipes.rs new file mode 100644 index 000000000..b375982dd --- /dev/null +++ b/src/uucore/src/lib/features/pipes.rs @@ -0,0 +1,69 @@ +/// Thin pipe-related wrappers around functions from the `nix` crate. +use std::fs::File; +#[cfg(any(target_os = "linux", target_os = "android"))] +use std::os::unix::io::AsRawFd; +use std::os::unix::io::FromRawFd; + +#[cfg(any(target_os = "linux", target_os = "android"))] +use nix::{fcntl::SpliceFFlags, sys::uio::IoVec}; + +pub use nix::{Error, Result}; + +/// A wrapper around [`nix::unistd::Pipe`] that ensures the pipe is cleaned up. +/// +/// Returns two `File` objects: everything written to the second can be read +/// from the first. +pub fn pipe() -> Result<(File, File)> { + let (read, write) = nix::unistd::pipe()?; + // SAFETY: The file descriptors do not have other owners. + unsafe { Ok((File::from_raw_fd(read), File::from_raw_fd(write))) } +} + +/// Less noisy wrapper around [`nix::fcntl::splice`]. +/// +/// Up to `len` bytes are moved from `source` to `target`. Returns the number +/// of successfully moved bytes. +/// +/// At least one of `source` and `target` must be some sort of pipe. +/// To get around this requirement, consider splicing from your source into +/// a [`pipe`] and then from the pipe into your target (with `splice_exact`): +/// this is still very efficient. +#[cfg(any(target_os = "linux", target_os = "android"))] +pub fn splice(source: &impl AsRawFd, target: &impl AsRawFd, len: usize) -> Result { + nix::fcntl::splice( + source.as_raw_fd(), + None, + target.as_raw_fd(), + None, + len, + SpliceFFlags::empty(), + ) +} + +/// Splice wrapper which fully finishes the write. +/// +/// Exactly `len` bytes are moved from `source` into `target`. +/// +/// Panics if `source` runs out of data before `len` bytes have been moved. +#[cfg(any(target_os = "linux", target_os = "android"))] +pub fn splice_exact(source: &impl AsRawFd, target: &impl AsRawFd, len: usize) -> Result<()> { + let mut left = len; + while left != 0 { + let written = splice(source, target, left)?; + assert_ne!(written, 0, "unexpected end of data"); + left -= written; + } + Ok(()) +} + +/// Copy data from `bytes` into `target`, which must be a pipe. +/// +/// Returns the number of successfully copied bytes. +#[cfg(any(target_os = "linux", target_os = "android"))] +pub fn vmsplice(target: &impl AsRawFd, bytes: &[u8]) -> Result { + nix::fcntl::vmsplice( + target.as_raw_fd(), + &[IoVec::from_slice(bytes)], + SpliceFFlags::empty(), + ) +} diff --git a/src/uucore/src/lib/lib.rs b/src/uucore/src/lib/lib.rs index 925246c24..79cc2afc3 100644 --- a/src/uucore/src/lib/lib.rs +++ b/src/uucore/src/lib/lib.rs @@ -49,6 +49,8 @@ pub use crate::features::mode; pub use crate::features::entries; #[cfg(all(unix, feature = "perms"))] pub use crate::features::perms; +#[cfg(all(unix, feature = "pipes"))] +pub use crate::features::pipes; #[cfg(all(unix, feature = "process"))] pub use crate::features::process; #[cfg(all(unix, not(target_os = "fuchsia"), feature = "signals"))]