From 2b5b0c82c162a19da7b0ba587d2cde8d87dd4823 Mon Sep 17 00:00:00 2001 From: Joining7943 <111500881+Joining7943@users.noreply.github.com> Date: Mon, 30 Jan 2023 08:53:28 +0100 Subject: [PATCH 01/94] tests/util: Bump rlimit version to 0.9.1 to be able to use `prlimit` on android --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- tests/common/util.rs | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6d210d303..05b2a1e1e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1833,9 +1833,9 @@ dependencies = [ [[package]] name = "rlimit" -version = "0.8.3" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7278a1ec8bfd4a4e07515c589f5ff7b309a373f987393aef44813d9dcf87aa3" +checksum = "f8a29d87a652dc4d43c586328706bb5cdff211f3f39a530f240b53f7221dab8e" dependencies = [ "libc", ] diff --git a/Cargo.toml b/Cargo.toml index ebd8685ca..9e5e0570a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -482,7 +482,7 @@ rstest = "0.16.0" [target.'cfg(any(target_os = "linux", target_os = "android"))'.dev-dependencies] procfs = { version = "0.14", default-features = false } -rlimit = "0.8.3" +rlimit = "0.9.1" [target.'cfg(unix)'.dev-dependencies] nix = { workspace=true, features=["process", "signal", "user"] } diff --git a/tests/common/util.rs b/tests/common/util.rs index ed4ecf8e9..eee47eca9 100644 --- a/tests/common/util.rs +++ b/tests/common/util.rs @@ -8,7 +8,7 @@ #![allow(dead_code)] use pretty_assertions::assert_eq; -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "android"))] use rlimit::prlimit; use rstest::rstest; #[cfg(unix)] @@ -1406,7 +1406,7 @@ impl UCommand { let child = command.spawn().unwrap(); - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", target_os = "android"))] for &(resource, soft_limit, hard_limit) in &self.limits { prlimit( child.id() as i32, From fdf0f96a0195126ea1593414c3918d3ab3db1e95 Mon Sep 17 00:00:00 2001 From: Joining7943 <111500881+Joining7943@users.noreply.github.com> Date: Sun, 22 Jan 2023 17:46:25 +0100 Subject: [PATCH 02/94] tests/util: Restructure UCommand to run in shell per default. Introduce constant TESTS_BINARY. Summary of changes in tests/util: * Introduce global constant TESTS_BINARY holding the path to `coreutils` * Implement running an arbitrary command in a sh or cmd shell per default. * Implement std::fmt::Display for UCommand. * Change usages of UCommand::util_name from &Option to Option * `UCommand::new_from_tmp`: Use OsStr directly instead of TempDir -> String -> OsStr * Collect arguments in `UCommand::args` field * Build environment variables in `UCommand` itself instead of `std::process::Command` * Move building of std::process:Command from fields in UCommand to own method `build` * Remove assertions of UCommand::has_run in arg, args and env. Summary of changes in tests/by-util: * Remove usages of UCommand::raw. Fix tests to use UCommand::to_string. * test_test: Adjust test_invalid_utf8_integer_compare to use `UCommand::args` * test_chmod: run_single_test * test_pwd: symlinked_env * test_cp: Fix the usage of &Option in `test_src_base_dot` Refactor test_src_base_dot to not use UCommand::new but ts.ucmd() instead --- tests/by-util/test_chmod.rs | 7 +- tests/by-util/test_cp.rs | 6 +- tests/by-util/test_pwd.rs | 2 +- tests/by-util/test_test.rs | 16 +-- tests/common/util.rs | 258 +++++++++++++++++++++++------------- 5 files changed, 178 insertions(+), 111 deletions(-) diff --git a/tests/by-util/test_chmod.rs b/tests/by-util/test_chmod.rs index feb756632..072756604 100644 --- a/tests/by-util/test_chmod.rs +++ b/tests/by-util/test_chmod.rs @@ -48,15 +48,12 @@ fn run_single_test(test: &TestCase, at: &AtPath, mut ucmd: UCommand) { let r = ucmd.run(); if !r.succeeded() { println!("{}", r.stderr_str()); - panic!("{:?}: failed", ucmd.raw); + panic!("{}: failed", ucmd); } let perms = at.metadata(TEST_FILE).permissions().mode(); if perms != test.after { - panic!( - "{:?}: expected: {:o} got: {:o}", - ucmd.raw, test.after, perms - ); + panic!("{}: expected: {:o} got: {:o}", ucmd, test.after, perms); } } diff --git a/tests/by-util/test_cp.rs b/tests/by-util/test_cp.rs index e6e8a6ba2..790383ded 100644 --- a/tests/by-util/test_cp.rs +++ b/tests/by-util/test_cp.rs @@ -2524,9 +2524,9 @@ fn test_src_base_dot() { let at = ts.fixtures.clone(); at.mkdir("x"); at.mkdir("y"); - let mut ucmd = UCommand::new(ts.bin_path, &Some(ts.util_name), at.plus("y"), true); - - ucmd.args(&["--verbose", "-r", "../x/.", "."]) + ts.ucmd() + .current_dir(at.plus("y")) + .args(&["--verbose", "-r", "../x/.", "."]) .succeeds() .no_stderr() .no_stdout(); diff --git a/tests/by-util/test_pwd.rs b/tests/by-util/test_pwd.rs index 0ae0cc909..461c597bd 100644 --- a/tests/by-util/test_pwd.rs +++ b/tests/by-util/test_pwd.rs @@ -60,7 +60,7 @@ fn symlinked_env() -> Env { // Note: on Windows this requires admin permissions at.symlink_dir("subdir", "symdir"); let root = PathBuf::from(at.root_dir_resolved()); - ucmd.raw.current_dir(root.join("symdir")); + ucmd.current_dir(root.join("symdir")); #[cfg(not(windows))] ucmd.env("PWD", root.join("symdir")); Env { diff --git a/tests/by-util/test_test.rs b/tests/by-util/test_test.rs index 146809f48..b4fb1b0ce 100644 --- a/tests/by-util/test_test.rs +++ b/tests/by-util/test_test.rs @@ -300,19 +300,15 @@ fn test_invalid_utf8_integer_compare() { let source = [0x66, 0x6f, 0x80, 0x6f]; let arg = OsStr::from_bytes(&source[..]); - let mut cmd = new_ucmd!(); - cmd.arg("123").arg("-ne"); - cmd.raw.arg(arg); - - cmd.run() + new_ucmd!() + .args(&[OsStr::new("123"), OsStr::new("-ne"), arg]) + .run() .code_is(2) .stderr_is("test: invalid integer $'fo\\x80o'\n"); - let mut cmd = new_ucmd!(); - cmd.raw.arg(arg); - cmd.arg("-eq").arg("456"); - - cmd.run() + new_ucmd!() + .args(&[arg, OsStr::new("-eq"), OsStr::new("456")]) + .run() .code_is(2) .stderr_is("test: invalid integer $'fo\\x80o'\n"); } diff --git a/tests/common/util.rs b/tests/common/util.rs index eee47eca9..0f3f0d7d8 100644 --- a/tests/common/util.rs +++ b/tests/common/util.rs @@ -3,7 +3,7 @@ // * For the full copyright and license information, please view the LICENSE // * file that was distributed with this source code. -//spell-checker: ignore (linux) rlimit prlimit coreutil ggroups uchild uncaptured scmd +//spell-checker: ignore (linux) rlimit prlimit coreutil ggroups uchild uncaptured scmd SHLVL #![allow(dead_code)] @@ -15,7 +15,7 @@ use rstest::rstest; use std::borrow::Cow; #[cfg(not(windows))] use std::ffi::CString; -use std::ffi::OsStr; +use std::ffi::{OsStr, OsString}; use std::fs::{self, hard_link, remove_file, File, OpenOptions}; use std::io::{self, BufWriter, Read, Result, Write}; #[cfg(unix)] @@ -34,7 +34,6 @@ use std::thread::{sleep, JoinHandle}; use std::time::{Duration, Instant}; use std::{env, hint, thread}; use tempfile::{Builder, TempDir}; -use uucore::Args; static TESTS_DIR: &str = "tests"; static FIXTURES_DIR: &str = "fixtures"; @@ -46,6 +45,8 @@ static MULTIPLE_STDIN_MEANINGLESS: &str = "Ucommand is designed around a typical static NO_STDIN_MEANINGLESS: &str = "Setting this flag has no effect if there is no stdin"; +pub const TESTS_BINARY: &str = env!("CARGO_BIN_EXE_coreutils"); + /// Test if the program is running under CI pub fn is_ci() -> bool { std::env::var("CI") @@ -1096,7 +1097,7 @@ impl TestScenario { pub fn new(util_name: &str) -> Self { let tmpd = Rc::new(TempDir::new().unwrap()); let ts = Self { - bin_path: PathBuf::from(env!("CARGO_BIN_EXE_coreutils")), + bin_path: PathBuf::from(TESTS_BINARY), util_name: String::from(util_name), fixtures: AtPath::new(tmpd.as_ref().path()), tmpd, @@ -1127,13 +1128,13 @@ impl TestScenario { util_name: T, env_clear: bool, ) -> UCommand { - UCommand::new_from_tmp(bin, &Some(util_name), self.tmpd.clone(), env_clear) + UCommand::new_from_tmp(bin, Some(util_name), self.tmpd.clone(), env_clear) } /// Returns builder for invoking any system command. Paths given are treated /// relative to the environment's unique temporary test directory. pub fn cmd>(&self, bin: S) -> UCommand { - UCommand::new_from_tmp::(bin, &None, self.tmpd.clone(), true) + UCommand::new_from_tmp::(bin, None, self.tmpd.clone(), true) } /// Returns builder for invoking any uutils command. Paths given are treated @@ -1153,7 +1154,7 @@ impl TestScenario { /// Differs from the builder returned by `cmd` in that `cmd_keepenv` does not call /// `Command::env_clear` (Clears the entire environment map for the child process.) pub fn cmd_keepenv>(&self, bin: S) -> UCommand { - UCommand::new_from_tmp::(bin, &None, self.tmpd.clone(), false) + UCommand::new_from_tmp::(bin, None, self.tmpd.clone(), false) } } @@ -1165,16 +1166,19 @@ impl TestScenario { /// 3. it provides convenience construction arguments to set the Command working directory and/or clear its environment. #[derive(Debug)] pub struct UCommand { - pub raw: Command, - comm_string: String, - bin_path: String, - util_name: Option, + args: Vec, + env_vars: Vec<(OsString, OsString)>, + current_dir: Option, + env_clear: bool, + bin_path: Option, + util_name: Option, has_run: bool, ignore_stdin_write_error: bool, stdin: Option, stdout: Option, stderr: Option, bytes_into_stdin: Option>, + // TODO: Why android? #[cfg(any(target_os = "linux", target_os = "android"))] limits: Vec<(rlimit::Resource, u64, u64)>, stderr_to_stdout: bool, @@ -1183,74 +1187,51 @@ pub struct UCommand { } impl UCommand { - pub fn new, S: AsRef, U: AsRef>( - bin_path: T, - util_name: &Option, - curdir: U, - env_clear: bool, - ) -> Self { - let bin_path = bin_path.as_ref(); - let util_name = util_name.as_ref().map(std::convert::AsRef::as_ref); - - let mut ucmd = Self { + pub fn new() -> Self { + Self { tmpd: None, has_run: false, - raw: { - let mut cmd = Command::new(bin_path); - cmd.current_dir(curdir.as_ref()); - if env_clear { - cmd.env_clear(); - if cfg!(windows) { - // spell-checker:ignore (dll) rsaenh - // %SYSTEMROOT% is required on Windows to initialize crypto provider - // ... and crypto provider is required for std::rand - // From `procmon`: RegQueryValue HKLM\SOFTWARE\Microsoft\Cryptography\Defaults\Provider\Microsoft Strong Cryptographic Provider\Image Path - // SUCCESS Type: REG_SZ, Length: 66, Data: %SystemRoot%\system32\rsaenh.dll" - if let Some(systemroot) = env::var_os("SYSTEMROOT") { - cmd.env("SYSTEMROOT", systemroot); - } - } else { - // if someone is setting LD_PRELOAD, there's probably a good reason for it - if let Some(ld_preload) = env::var_os("LD_PRELOAD") { - cmd.env("LD_PRELOAD", ld_preload); - } - } - } - cmd - }, - comm_string: String::from(bin_path.to_str().unwrap()), - bin_path: bin_path.to_str().unwrap().to_string(), - util_name: util_name.map(|un| un.to_str().unwrap().to_string()), + bin_path: None, + current_dir: None, + args: vec![], + env_clear: true, + env_vars: vec![], + util_name: None, ignore_stdin_write_error: false, bytes_into_stdin: None, stdin: None, stdout: None, stderr: None, + // TODO: Why android? #[cfg(any(target_os = "linux", target_os = "android"))] limits: vec![], stderr_to_stdout: false, timeout: Some(Duration::from_secs(30)), - }; - - if let Some(un) = util_name { - ucmd.arg(un); } - - ucmd } pub fn new_from_tmp, S: AsRef>( bin_path: T, - util_name: &Option, + util_name: Option, tmpd: Rc, env_clear: bool, ) -> Self { - let tmpd_path_buf = String::from(tmpd.as_ref().path().to_str().unwrap()); - let mut ucmd: Self = Self::new(bin_path, util_name, tmpd_path_buf, env_clear); + let mut ucmd: Self = Self::new(); + ucmd.bin_path = Some(PathBuf::from(bin_path.as_ref())); + ucmd.util_name = util_name.map(|s| s.as_ref().to_os_string()); ucmd.tmpd = Some(tmpd); + ucmd.env_clear = env_clear; ucmd } + pub fn current_dir(&mut self, current_dir: T) -> &mut Self + where + T: AsRef, + { + self.current_dir = Some(current_dir.as_ref().into()); + self + } + pub fn set_stdin>(&mut self, stdin: T) -> &mut Self { self.stdin = Some(stdin.into()); self @@ -1274,29 +1255,14 @@ impl UCommand { /// Add a parameter to the invocation. Path arguments are treated relative /// to the test environment directory. pub fn arg>(&mut self, arg: S) -> &mut Self { - assert!(!self.has_run, "{}", ALREADY_RUN); - self.comm_string.push(' '); - self.comm_string - .push_str(arg.as_ref().to_str().unwrap_or_default()); - self.raw.arg(arg.as_ref()); + self.args.push(arg.as_ref().into()); self } /// Add multiple parameters to the invocation. Path arguments are treated relative /// to the test environment directory. pub fn args>(&mut self, args: &[S]) -> &mut Self { - assert!(!self.has_run, "{}", MULTIPLE_STDIN_MEANINGLESS); - let strings = args - .iter() - .map(|s| s.as_ref().to_os_string()) - .collect_ignore(); - - for s in strings { - self.comm_string.push(' '); - self.comm_string.push_str(&s); - } - - self.raw.args(args.as_ref()); + self.args.extend(args.iter().map(|s| s.as_ref().into())); self } @@ -1331,11 +1297,12 @@ impl UCommand { K: AsRef, V: AsRef, { - assert!(!self.has_run, "{}", ALREADY_RUN); - self.raw.env(key, val); + self.env_vars + .push((key.as_ref().into(), val.as_ref().into())); self } + // TODO: Why android? #[cfg(any(target_os = "linux", target_os = "android"))] pub fn with_limit( &mut self, @@ -1359,26 +1326,85 @@ impl UCommand { self } - /// Spawns the command, feeds the stdin if any, and returns the - /// child process immediately. - pub fn run_no_wait(&mut self) -> UChild { - assert!(!self.has_run, "{}", ALREADY_RUN); - self.has_run = true; - log_info("run", &self.comm_string); + // TODO: make public? + fn build(&mut self) -> (Command, Option, Option) { + if self.bin_path.is_some() { + if let Some(util_name) = &self.util_name { + self.args.insert(0, OsString::from(util_name)); + } + } else if let Some(util_name) = &self.util_name { + self.bin_path = Some(PathBuf::from(TESTS_BINARY)); + self.args.insert(0, OsString::from(util_name)); + } else if cfg!(unix) { + let bin_path = if cfg!(target_os = "android") { + PathBuf::from("/system/bin/sh") + } else { + PathBuf::from("/bin/sh") + }; + self.bin_path = Some(bin_path); + let c_arg = OsString::from("-c"); + if !self.args.contains(&c_arg) { + self.args.insert(0, c_arg); + } + } else { + self.bin_path = Some(PathBuf::from("cmd")); + let c_arg = OsString::from("/C"); + let k_arg = OsString::from("/K"); + if !self + .args + .iter() + .any(|s| s.eq_ignore_ascii_case(&c_arg) || s.eq_ignore_ascii_case(&k_arg)) + { + self.args.insert(0, c_arg); + } + }; + + let mut command = Command::new(self.bin_path.as_ref().unwrap()); + command.args(&self.args); + + if self.tmpd.is_none() { + self.tmpd = Some(Rc::new(tempfile::tempdir().unwrap())); + } + + if let Some(current_dir) = &self.current_dir { + command.current_dir(current_dir); + } else { + command.current_dir(self.tmpd.as_ref().unwrap().path()); + } + + if self.env_clear { + command.env_clear(); + if cfg!(windows) { + // spell-checker:ignore (dll) rsaenh + // %SYSTEMROOT% is required on Windows to initialize crypto provider + // ... and crypto provider is required for std::rand + // From `procmon`: RegQueryValue HKLM\SOFTWARE\Microsoft\Cryptography\Defaults\Provider\Microsoft Strong Cryptographic Provider\Image Path + // SUCCESS Type: REG_SZ, Length: 66, Data: %SystemRoot%\system32\rsaenh.dll" + if let Some(systemroot) = env::var_os("SYSTEMROOT") { + command.env("SYSTEMROOT", systemroot); + } + } else { + // if someone is setting LD_PRELOAD, there's probably a good reason for it + if let Some(ld_preload) = env::var_os("LD_PRELOAD") { + command.env("LD_PRELOAD", ld_preload); + } + } + } + + for (key, value) in &self.env_vars { + command.env(key, value); + } let mut captured_stdout = None; let mut captured_stderr = None; - let command = if self.stderr_to_stdout { + if self.stderr_to_stdout { let mut output = CapturedOutput::default(); - let command = self - .raw + command .stdin(self.stdin.take().unwrap_or_else(Stdio::null)) .stdout(Stdio::from(output.try_clone().unwrap())) .stderr(Stdio::from(output.try_clone().unwrap())); captured_stdout = Some(output); - - command } else { let stdout = if self.stdout.is_some() { self.stdout.take().unwrap() @@ -1398,12 +1424,25 @@ impl UCommand { stdio }; - self.raw + command .stdin(self.stdin.take().unwrap_or_else(Stdio::null)) .stdout(stdout) - .stderr(stderr) + .stderr(stderr); }; + (command, captured_stdout, captured_stderr) + } + + /// Spawns the command, feeds the stdin if any, and returns the + /// child process immediately. + pub fn run_no_wait(&mut self) -> UChild { + // TODO: remove? + assert!(!self.has_run, "{}", ALREADY_RUN); + self.has_run = true; + + let (mut command, captured_stdout, captured_stderr) = self.build(); + log_info("run", self.to_string()); + let child = command.spawn().unwrap(); #[cfg(any(target_os = "linux", target_os = "android"))] @@ -1465,6 +1504,17 @@ impl UCommand { } } +impl std::fmt::Display for UCommand { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut comm_string: Vec = vec![self + .bin_path + .as_ref() + .map_or("".to_string(), |p| p.display().to_string())]; + comm_string.extend(self.args.iter().map(|s| s.to_string_lossy().to_string())); + f.write_str(&comm_string.join(" ")) + } +} + /// Stored the captured output in a temporary file. The file is deleted as soon as /// [`CapturedOutput`] is dropped. #[derive(Debug)] @@ -1704,8 +1754,11 @@ impl UChild { ) -> Self { Self { raw: child, - bin_path: ucommand.bin_path.clone(), - util_name: ucommand.util_name.clone(), + bin_path: ucommand.bin_path.as_ref().unwrap().display().to_string(), + util_name: ucommand + .util_name + .clone() + .map(|s| s.to_string_lossy().to_string()), captured_stdout, captured_stderr, ignore_stdin_write_error: ucommand.ignore_stdin_write_error, @@ -2443,7 +2496,7 @@ mod tests { pub fn run_cmd>(cmd: T) -> CmdResult { let mut ucmd = UCommand::new_from_tmp::<&str, String>( "sh", - &None, + None, Rc::new(tempfile::tempdir().unwrap()), true, ); @@ -2456,7 +2509,7 @@ mod tests { pub fn run_cmd>(cmd: T) -> CmdResult { let mut ucmd = UCommand::new_from_tmp::<&str, String>( "cmd", - &None, + None, Rc::new(tempfile::tempdir().unwrap()), true, ); @@ -3200,4 +3253,25 @@ mod tests { let ts = TestScenario::new("sleep"); ts.ucmd().timeout(Duration::from_secs(60)).arg("1.0").run(); } + + #[cfg(feature = "echo")] + #[test] + fn test_ucommand_when_default() { + let shell_cmd = format!("{} echo -n hello", TESTS_BINARY); + + let mut command = UCommand::new(); + command.arg(&shell_cmd).succeeds().stdout_is("hello"); + + #[cfg(target_os = "android")] + let (expected_bin, expected_arg) = (PathBuf::from("/system/bin/sh"), OsString::from("-c")); + #[cfg(all(unix, not(target_os = "android")))] + let (expected_bin, expected_arg) = (PathBuf::from("/bin/sh"), OsString::from("-c")); + #[cfg(windows)] + let (expected_bin, expected_arg) = (PathBuf::from("cmd"), OsString::from("/C")); + + std::assert_eq!(&expected_bin, command.bin_path.as_ref().unwrap()); + assert!(command.util_name.is_none()); + std::assert_eq!(command.args, &[expected_arg, OsString::from(&shell_cmd)]); + assert!(command.tmpd.is_some()); + } } From 678a11dcf299a40df324639bb9920080971ab21e Mon Sep 17 00:00:00 2001 From: Yang Hau Date: Mon, 13 Feb 2023 11:40:16 +0800 Subject: [PATCH 03/94] cksum: Implement option -a Implement option -a --algorithm. Move digest to src/uucore/src/lib/features and rename it to hash. fix lint fix Cargo.toml --- Cargo.lock | 27 ++ Cargo.toml | 10 + src/uu/cksum/Cargo.toml | 9 +- src/uu/cksum/src/cksum.rs | 237 +++++++++++------- src/uu/hashsum/Cargo.toml | 21 +- src/uu/hashsum/src/hashsum.rs | 8 +- src/uucore/Cargo.toml | 12 + src/uucore/src/lib/features.rs | 2 + .../src/lib/features/sum.rs} | 232 +++++++++++++++-- src/uucore/src/lib/lib.rs | 2 + tests/by-util/test_cksum.rs | 76 ++++++ .../cksum/bsd_multiple_files.expected | 2 + tests/fixtures/cksum/bsd_single_file.expected | 1 + tests/fixtures/cksum/bsd_stdin.expected | 1 + .../cksum/sysv_multiple_files.expected | 2 + .../fixtures/cksum/sysv_single_file.expected | 1 + tests/fixtures/cksum/sysv_stdin.expected | 1 + 17 files changed, 509 insertions(+), 135 deletions(-) rename src/{uu/hashsum/src/digest.rs => uucore/src/lib/features/sum.rs} (58%) create mode 100644 tests/fixtures/cksum/bsd_multiple_files.expected create mode 100644 tests/fixtures/cksum/bsd_single_file.expected create mode 100644 tests/fixtures/cksum/bsd_stdin.expected create mode 100644 tests/fixtures/cksum/sysv_multiple_files.expected create mode 100644 tests/fixtures/cksum/sysv_single_file.expected create mode 100644 tests/fixtures/cksum/sysv_stdin.expected diff --git a/Cargo.lock b/Cargo.lock index 6d210d303..f08af896a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2053,6 +2053,15 @@ dependencies = [ "autocfg", ] +[[package]] +name = "sm3" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f943a7c5e3089f2bd046221d1e9f4fa59396bf0fe966360983649683086215da" +dependencies = [ + "digest", +] + [[package]] name = "smallvec" version = "1.10.0" @@ -2390,7 +2399,14 @@ dependencies = [ name = "uu_cksum" version = "0.0.17" dependencies = [ + "blake2b_simd", + "blake3", "clap", + "hex", + "md-5", + "sha1", + "sha2", + "sm3", "uucore", ] @@ -2606,6 +2622,7 @@ dependencies = [ "sha1", "sha2", "sha3", + "sm3", "uucore", ] @@ -3299,17 +3316,27 @@ dependencies = [ name = "uucore" version = "0.0.17" dependencies = [ + "blake2b_simd", + "blake3", "clap", "data-encoding", "data-encoding-macro", + "digest", "dns-lookup", "dunce", "glob", + "hex", "itertools", "libc", + "md-5", + "memchr", "nix", "once_cell", "os_display", + "sha1", + "sha2", + "sha3", + "sm3", "thiserror", "time", "uucore_procs", diff --git a/Cargo.toml b/Cargo.toml index ebd8685ca..d84e7daf9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -332,6 +332,16 @@ windows-sys = { version="0.42.0", default-features=false } xattr = "0.2.3" zip = { version = "0.6.3", default_features=false, features=["deflate"] } +hex = "0.4.3" +md-5 = "0.10.5" +sha1 = "0.10.1" +sha2 = "0.10.2" +sha3 = "0.10.6" +blake2b_simd = "1.0.1" +blake3 = "1.3.2" +sm3 = "0.4.1" +digest = "0.10.6" + uucore = { version=">=0.0.17", package="uucore", path="src/uucore" } uucore_procs = { version=">=0.0.17", package="uucore_procs", path="src/uucore_procs" } uu_ls = { version=">=0.0.17", path="src/uu/ls" } diff --git a/src/uu/cksum/Cargo.toml b/src/uu/cksum/Cargo.toml index 4719af074..20406a3f0 100644 --- a/src/uu/cksum/Cargo.toml +++ b/src/uu/cksum/Cargo.toml @@ -16,7 +16,14 @@ path = "src/cksum.rs" [dependencies] clap = { workspace=true } -uucore = { workspace=true } +uucore = { version=">=0.0.17", package="uucore", path="../../uucore", features=["sum"] } +hex = { workspace=true } +md-5 = { workspace=true } +sha1 = { workspace=true } +sha2 = { workspace=true } +blake2b_simd = { workspace=true } +blake3 = { workspace=true } +sm3 = { workspace=true } [[bin]] name = "cksum" diff --git a/src/uu/cksum/src/cksum.rs b/src/uu/cksum/src/cksum.rs index f567d8bf8..146fe0d56 100644 --- a/src/uu/cksum/src/cksum.rs +++ b/src/uu/cksum/src/cksum.rs @@ -5,109 +5,147 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore (ToDO) fname +// spell-checker:ignore (ToDO) fname, algo use clap::{crate_version, Arg, Command}; +use hex::encode; +use md5::Md5; +use sha1::Sha1; +use sha2::{Sha224, Sha256, Sha384, Sha512}; +use std::ffi::OsStr; use std::fs::File; use std::io::{self, stdin, BufReader, Read}; +use std::iter; use std::path::Path; -use uucore::display::Quotable; -use uucore::error::{FromIo, UResult}; -use uucore::{format_usage, show}; - -// NOTE: CRC_TABLE_LEN *must* be <= 256 as we cast 0..CRC_TABLE_LEN to u8 -const CRC_TABLE_LEN: usize = 256; -const CRC_TABLE: [u32; CRC_TABLE_LEN] = generate_crc_table(); +use uucore::{ + error::{FromIo, UResult}, + format_usage, + sum::{div_ceil, Digest, DigestWriter, BSD, CRC, SYSV}, +}; const USAGE: &str = "{} [OPTIONS] [FILE]..."; const ABOUT: &str = "Print CRC and size for each file"; -const fn generate_crc_table() -> [u32; CRC_TABLE_LEN] { - let mut table = [0; CRC_TABLE_LEN]; +fn detect_algo(program: &str) -> (&'static str, Box, usize) { + match program { + "sysv" => ("SYSV", Box::new(SYSV::new()) as Box, 512), + "bsd" => ("BSD", Box::new(BSD::new()) as Box, 1024), + "crc" => ("CRC", Box::new(CRC::new()) as Box, 256), + "md5" => ("MD5", Box::new(Md5::new()) as Box, 128), + "sha1" => ("SHA1", Box::new(Sha1::new()) as Box, 160), + "sha224" => ("SHA224", Box::new(Sha224::new()) as Box, 224), + "sha256" => ("SHA256", Box::new(Sha256::new()) as Box, 256), + "sha384" => ("SHA384", Box::new(Sha384::new()) as Box, 384), + "sha512" => ("SHA512", Box::new(Sha512::new()) as Box, 512), + "blake2b" => ( + "BLAKE2", + Box::new(blake2b_simd::State::new()) as Box, + 512, + ), + "sm3" => ("SM3", Box::new(sm3::Sm3::new()) as Box, 512), + _ => panic!("unknown algorithm"), + } +} - let mut i = 0; - while i < CRC_TABLE_LEN { - table[i] = crc_entry(i as u8); +struct Options { + algo_name: &'static str, + digest: Box, + output_bits: usize, +} - i += 1; +#[allow(clippy::cognitive_complexity)] +fn cksum<'a, I>(mut options: Options, files: I) -> UResult<()> +where + I: Iterator, +{ + for filename in files { + let filename = Path::new(filename); + let stdin_buf; + let file_buf; + let not_file = filename == OsStr::new("-"); + let mut file = BufReader::new(if not_file { + stdin_buf = stdin(); + Box::new(stdin_buf) as Box + } else if filename.is_dir() { + Box::new(BufReader::new(io::empty())) as Box + } else { + file_buf = + File::open(filename).map_err_context(|| filename.to_str().unwrap().to_string())?; + Box::new(file_buf) as Box + }); + let (sum, sz) = digest_read(&mut options.digest, &mut file, options.output_bits) + .map_err_context(|| "failed to read input".to_string())?; + + // Refer to GNU sum.c implementation. The BSD checksum output is 5 digit integer + // https://github.com/coreutils/coreutils/blob/master/src/sum.c + let bsd_width = 5; + match (options.algo_name, not_file) { + ("SYSV", true) => println!( + "{} {}", + sum.parse::().unwrap(), + div_ceil(sz, options.output_bits) + ), + ("SYSV", false) => println!( + "{} {} {}", + sum.parse::().unwrap(), + div_ceil(sz, options.output_bits), + filename.display() + ), + ("BSD", true) => println!( + "{:0bsd_width$} {:bsd_width$}", + sum.parse::().unwrap(), + div_ceil(sz, options.output_bits) + ), + ("BSD", false) => println!( + "{:0bsd_width$} {:bsd_width$} {}", + sum.parse::().unwrap(), + div_ceil(sz, options.output_bits), + filename.display() + ), + (_, true) => println!("{sum} {sz}"), + (_, false) => println!("{sum} {sz} {}", filename.display()), + } } - table + Ok(()) } -const fn crc_entry(input: u8) -> u32 { - let mut crc = (input as u32) << 24; +fn digest_read( + digest: &mut Box, + reader: &mut BufReader, + output_bits: usize, +) -> io::Result<(String, usize)> { + digest.reset(); - let mut i = 0; - while i < 8 { - let if_condition = crc & 0x8000_0000; - let if_body = (crc << 1) ^ 0x04c1_1db7; - let else_body = crc << 1; + // Read bytes from `reader` and write those bytes to `digest`. + // + // If `binary` is `false` and the operating system is Windows, then + // `DigestWriter` replaces "\r\n" with "\n" before it writes the + // bytes into `digest`. Otherwise, it just inserts the bytes as-is. + // + // In order to support replacing "\r\n", we must call `finalize()` + // in order to support the possibility that the last character read + // from the reader was "\r". (This character gets buffered by + // `DigestWriter` and only written if the following character is + // "\n". But when "\r" is the last character read, we need to force + // it to be written.) + let mut digest_writer = DigestWriter::new(digest, true); + let output_size = std::io::copy(reader, &mut digest_writer)? as usize; + digest_writer.finalize(); - // NOTE: i feel like this is easier to understand than emulating an if statement in bitwise - // ops - let condition_table = [else_body, if_body]; - - crc = condition_table[(if_condition != 0) as usize]; - i += 1; - } - - crc -} - -#[inline] -fn crc_update(crc: u32, input: u8) -> u32 { - (crc << 8) ^ CRC_TABLE[((crc >> 24) as usize ^ input as usize) & 0xFF] -} - -#[inline] -fn crc_final(mut crc: u32, mut length: usize) -> u32 { - while length != 0 { - crc = crc_update(crc, length as u8); - length >>= 8; - } - - !crc -} - -fn init_byte_array() -> Vec { - vec![0; 1024 * 1024] -} - -#[inline] -fn cksum(fname: &str) -> io::Result<(u32, usize)> { - let mut crc = 0u32; - let mut size = 0usize; - - let mut rd: Box = match fname { - "-" => Box::new(stdin()), - _ => { - let p = Path::new(fname); - - // Directories should not give an error, but should be interpreted - // as empty files to match GNU semantics. - if p.is_dir() { - Box::new(BufReader::new(io::empty())) as Box - } else { - Box::new(BufReader::new(File::open(p)?)) as Box - } - } - }; - - let mut bytes = init_byte_array(); - loop { - let num_bytes = rd.read(&mut bytes)?; - if num_bytes == 0 { - return Ok((crc_final(crc, size), size)); - } - for &b in bytes[..num_bytes].iter() { - crc = crc_update(crc, b); - } - size += num_bytes; + if digest.output_bits() > 0 { + Ok((digest.result_str(), output_size)) + } else { + // Assume it's SHAKE. result_str() doesn't work with shake (as of 8/30/2016) + let mut bytes = Vec::new(); + bytes.resize((output_bits + 7) / 8, 0); + digest.hash_finalize(&mut bytes); + Ok((encode(bytes), output_size)) } } mod options { pub static FILE: &str = "file"; + pub static ALGORITHM: &str = "algorithm"; } #[uucore::main] @@ -116,23 +154,23 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { let matches = uu_app().try_get_matches_from(args)?; - let files: Vec = match matches.get_many::(options::FILE) { - Some(v) => v.clone().map(|v| v.to_owned()).collect(), - None => vec![], + let algo_name: &str = match matches.get_one::(options::ALGORITHM) { + Some(v) => v, + None => "crc", }; - if files.is_empty() { - let (crc, size) = cksum("-")?; - println!("{crc} {size}"); - return Ok(()); - } + let (name, algo, bits) = detect_algo(algo_name); + let opts = Options { + algo_name: name, + digest: algo, + output_bits: bits, + }; + + match matches.get_many::(options::FILE) { + Some(files) => cksum(opts, files.map(OsStr::new))?, + None => cksum(opts, iter::once(OsStr::new("-")))?, + }; - for fname in &files { - match cksum(fname.as_ref()).map_err_context(|| format!("{}", fname.maybe_quote())) { - Ok((crc, size)) => println!("{crc} {size} {fname}"), - Err(err) => show!(err), - }; - } Ok(()) } @@ -148,4 +186,11 @@ pub fn uu_app() -> Command { .action(clap::ArgAction::Append) .value_hint(clap::ValueHint::FilePath), ) + .arg( + Arg::new(options::ALGORITHM) + .long(options::ALGORITHM) + .short('a') + .help("select the digest type to use. See DIGEST below") + .value_name("ALGORITHM"), + ) } diff --git a/src/uu/hashsum/Cargo.toml b/src/uu/hashsum/Cargo.toml index b037cbcb8..a467dee9f 100644 --- a/src/uu/hashsum/Cargo.toml +++ b/src/uu/hashsum/Cargo.toml @@ -15,18 +15,19 @@ edition = "2021" path = "src/hashsum.rs" [dependencies] -digest = "0.10.6" clap = { workspace=true } -hex = "0.4.3" -memchr = { workspace=true } -md-5 = "0.10.5" -regex = { workspace=true } -sha1 = "0.10.1" -sha2 = "0.10.2" -sha3 = "0.10.6" -blake2b_simd = "1.0.1" -blake3 = "1.3.2" uucore = { workspace=true } +memchr = { workspace=true } +regex = { workspace=true } +hex = { workspace=true } +md-5 = { workspace=true } +sha1 = { workspace=true } +sha2 = { workspace=true } +sha3 = { workspace=true } +blake2b_simd = { workspace=true } +blake3 = { workspace=true } +sm3 = { workspace=true } +digest = { workspace=true } [[bin]] name = "hashsum" diff --git a/src/uu/hashsum/src/hashsum.rs b/src/uu/hashsum/src/hashsum.rs index 3271425bc..a240304be 100644 --- a/src/uu/hashsum/src/hashsum.rs +++ b/src/uu/hashsum/src/hashsum.rs @@ -9,11 +9,6 @@ // spell-checker:ignore (ToDO) algo, algoname, regexes, nread, nonames -mod digest; - -use self::digest::Digest; -use self::digest::DigestWriter; - use clap::builder::ValueParser; use clap::crate_version; use clap::ArgAction; @@ -36,6 +31,7 @@ use uucore::crash; use uucore::display::Quotable; use uucore::error::{FromIo, UError, UResult}; use uucore::show_warning; +use uucore::sum::{Digest, DigestWriter}; const NAME: &str = "hashsum"; @@ -680,7 +676,7 @@ fn digest_reader( // Assume it's SHAKE. result_str() doesn't work with shake (as of 8/30/2016) let mut bytes = Vec::new(); bytes.resize((output_bits + 7) / 8, 0); - digest.result(&mut bytes); + digest.hash_finalize(&mut bytes); Ok(encode(bytes)) } } diff --git a/src/uucore/Cargo.toml b/src/uucore/Cargo.toml index d4d8b8cf2..f5bec9053 100644 --- a/src/uucore/Cargo.toml +++ b/src/uucore/Cargo.toml @@ -36,6 +36,17 @@ libc = { version="0.2.137", optional=true } once_cell = { workspace=true } os_display = "0.1.3" +digest = { workspace=true } +hex = { workspace=true } +memchr = { workspace=true } +md-5 = { workspace=true } +sha1 = { workspace=true } +sha2 = { workspace=true } +sha3 = { workspace=true } +blake2b_simd = { workspace=true } +blake3 = { workspace=true } +sm3 = { workspace=true } + [target.'cfg(unix)'.dependencies] walkdir = { workspace=true, optional=true } nix = { workspace=true, features = ["fs", "uio", "zerocopy"] } @@ -66,3 +77,4 @@ utf8 = [] utmpx = ["time", "time/macros", "libc", "dns-lookup"] wide = [] pipes = [] +sum = [] diff --git a/src/uucore/src/lib/features.rs b/src/uucore/src/lib/features.rs index 2e5aea1e2..f8a8d2d10 100644 --- a/src/uucore/src/lib/features.rs +++ b/src/uucore/src/lib/features.rs @@ -12,6 +12,8 @@ pub mod lines; pub mod memo; #[cfg(feature = "ringbuffer")] pub mod ringbuffer; +#[cfg(feature = "sum")] +pub mod sum; #[cfg(feature = "memo")] mod tokenize; diff --git a/src/uu/hashsum/src/digest.rs b/src/uucore/src/lib/features/sum.rs similarity index 58% rename from src/uu/hashsum/src/digest.rs rename to src/uucore/src/lib/features/sum.rs index df3bfc166..c7cec6e09 100644 --- a/src/uu/hashsum/src/digest.rs +++ b/src/uucore/src/lib/features/sum.rs @@ -16,8 +16,8 @@ pub trait Digest { fn new() -> Self where Self: Sized; - fn input(&mut self, input: &[u8]); - fn result(&mut self, out: &mut [u8]); + fn hash_update(&mut self, input: &[u8]); + fn hash_finalize(&mut self, out: &mut [u8]); fn reset(&mut self); fn output_bits(&self) -> usize; fn output_bytes(&self) -> usize { @@ -25,7 +25,7 @@ pub trait Digest { } fn result_str(&mut self) -> String { let mut buf: Vec = vec![0; self.output_bytes()]; - self.result(&mut buf); + self.hash_finalize(&mut buf); encode(buf) } } @@ -35,11 +35,11 @@ impl Digest for blake2b_simd::State { Self::new() } - fn input(&mut self, input: &[u8]) { + fn hash_update(&mut self, input: &[u8]) { self.update(input); } - fn result(&mut self, out: &mut [u8]) { + fn hash_finalize(&mut self, out: &mut [u8]) { let hash_result = &self.finalize(); out.copy_from_slice(hash_result.as_bytes()); } @@ -58,11 +58,11 @@ impl Digest for blake3::Hasher { Self::new() } - fn input(&mut self, input: &[u8]) { + fn hash_update(&mut self, input: &[u8]) { self.update(input); } - fn result(&mut self, out: &mut [u8]) { + fn hash_finalize(&mut self, out: &mut [u8]) { let hash_result = &self.finalize(); out.copy_from_slice(hash_result.as_bytes()); } @@ -76,6 +76,194 @@ impl Digest for blake3::Hasher { } } +use sm3::{Digest as SM3_D, Sm3}; + +impl Digest for Sm3 { + fn new() -> Self { + ::new() + } + + fn hash_update(&mut self, input: &[u8]) { + self.update(input); + } + + fn hash_finalize(&mut self, out: &mut [u8]) { + out.copy_from_slice(&self.clone().finalize()); + } + + fn reset(&mut self) { + *self = ::new(); + } + + fn output_bits(&self) -> usize { + 256 + } +} + +// NOTE: CRC_TABLE_LEN *must* be <= 256 as we cast 0..CRC_TABLE_LEN to u8 +const CRC_TABLE_LEN: usize = 256; + +pub struct CRC { + state: u32, + size: usize, + crc_table: [u32; CRC_TABLE_LEN], +} + +impl CRC { + fn generate_crc_table() -> [u32; CRC_TABLE_LEN] { + let mut table = [0; CRC_TABLE_LEN]; + + for (i, elt) in table.iter_mut().enumerate().take(CRC_TABLE_LEN) { + *elt = Self::crc_entry(i as u8); + } + + table + } + fn crc_entry(input: u8) -> u32 { + let mut crc = (input as u32) << 24; + + let mut i = 0; + while i < 8 { + let if_condition = crc & 0x8000_0000; + let if_body = (crc << 1) ^ 0x04c1_1db7; + let else_body = crc << 1; + + // NOTE: i feel like this is easier to understand than emulating an if statement in bitwise + // ops + let condition_table = [else_body, if_body]; + + crc = condition_table[(if_condition != 0) as usize]; + i += 1; + } + + crc + } + + fn update(&mut self, input: u8) { + self.state = (self.state << 8) + ^ self.crc_table[((self.state >> 24) as usize ^ input as usize) & 0xFF]; + } +} + +impl Digest for CRC { + fn new() -> Self { + Self { + state: 0, + size: 0, + crc_table: Self::generate_crc_table(), + } + } + + fn hash_update(&mut self, input: &[u8]) { + for &elt in input.iter() { + self.update(elt); + } + self.size += input.len(); + } + + fn hash_finalize(&mut self, out: &mut [u8]) { + let mut sz = self.size; + while sz != 0 { + self.update(sz as u8); + sz >>= 8; + } + self.state = !self.state; + out.copy_from_slice(&self.state.to_ne_bytes()); + } + + fn result_str(&mut self) -> String { + let mut _out: Vec = vec![0; 4]; + self.hash_finalize(&mut _out); + format!("{}", self.state) + } + + fn reset(&mut self) { + *self = Self::new(); + } + + fn output_bits(&self) -> usize { + 256 + } +} + +// This can be replaced with usize::div_ceil once it is stabilized. +// This implementation approach is optimized for when `b` is a constant, +// particularly a power of two. +pub fn div_ceil(a: usize, b: usize) -> usize { + (a + b - 1) / b +} + +pub struct BSD { + state: u16, +} + +impl Digest for BSD { + fn new() -> Self { + Self { state: 0 } + } + + fn hash_update(&mut self, input: &[u8]) { + for &byte in input.iter() { + self.state = (self.state >> 1) + ((self.state & 1) << 15); + self.state = self.state.wrapping_add(u16::from(byte)); + } + } + + fn hash_finalize(&mut self, out: &mut [u8]) { + out.copy_from_slice(&self.state.to_ne_bytes()); + } + + fn result_str(&mut self) -> String { + let mut _out: Vec = vec![0; 2]; + self.hash_finalize(&mut _out); + format!("{}", self.state) + } + + fn reset(&mut self) { + *self = Self::new(); + } + + fn output_bits(&self) -> usize { + 128 + } +} + +pub struct SYSV { + state: u32, +} + +impl Digest for SYSV { + fn new() -> Self { + Self { state: 0 } + } + + fn hash_update(&mut self, input: &[u8]) { + for &byte in input.iter() { + self.state = self.state.wrapping_add(u32::from(byte)); + } + } + + fn hash_finalize(&mut self, out: &mut [u8]) { + self.state = (self.state & 0xffff) + (self.state >> 16); + self.state = (self.state & 0xffff) + (self.state >> 16); + out.copy_from_slice(&(self.state as u16).to_ne_bytes()); + } + + fn result_str(&mut self) -> String { + let mut _out: Vec = vec![0; 2]; + self.hash_finalize(&mut _out); + format!("{}", self.state) + } + + fn reset(&mut self) { + *self = Self::new(); + } + + fn output_bits(&self) -> usize { + 512 + } +} + // Implements the Digest trait for sha2 / sha3 algorithms with fixed output macro_rules! impl_digest_common { ($type: ty, $size: expr) => { @@ -84,16 +272,16 @@ macro_rules! impl_digest_common { Self::default() } - fn input(&mut self, input: &[u8]) { + fn hash_update(&mut self, input: &[u8]) { digest::Digest::update(self, input); } - fn result(&mut self, out: &mut [u8]) { + fn hash_finalize(&mut self, out: &mut [u8]) { digest::Digest::finalize_into_reset(self, out.into()); } fn reset(&mut self) { - *self = Self::new(); + *self = ::new(); } fn output_bits(&self) -> usize { @@ -111,11 +299,11 @@ macro_rules! impl_digest_shake { Self::default() } - fn input(&mut self, input: &[u8]) { + fn hash_update(&mut self, input: &[u8]) { digest::Update::update(self, input); } - fn result(&mut self, out: &mut [u8]) { + fn hash_finalize(&mut self, out: &mut [u8]) { digest::ExtendableOutputReset::finalize_xof_reset_into(self, out); } @@ -148,7 +336,7 @@ impl_digest_shake!(sha3::Shake256); /// /// This struct wraps a [`Digest`] and provides a [`Write`] /// implementation that passes input bytes directly to the -/// [`Digest::input`]. +/// [`Digest::hash_update`]. /// /// On Windows, if `binary` is `false`, then the [`write`] /// implementation replaces instances of "\r\n" with "\n" before passing @@ -182,7 +370,7 @@ impl<'a> DigestWriter<'a> { pub fn finalize(&mut self) -> bool { if self.was_last_character_carriage_return { - self.digest.input(&[b'\r']); + self.digest.hash_update(&[b'\r']); true } else { false @@ -193,14 +381,14 @@ impl<'a> DigestWriter<'a> { impl<'a> Write for DigestWriter<'a> { #[cfg(not(windows))] fn write(&mut self, buf: &[u8]) -> std::io::Result { - self.digest.input(buf); + self.digest.hash_update(buf); Ok(buf.len()) } #[cfg(windows)] fn write(&mut self, buf: &[u8]) -> std::io::Result { if self.binary { - self.digest.input(buf); + self.digest.hash_update(buf); return Ok(buf.len()); } @@ -213,7 +401,7 @@ impl<'a> Write for DigestWriter<'a> { // call to `write()`. let n = buf.len(); if self.was_last_character_carriage_return && n > 0 && buf[0] != b'\n' { - self.digest.input(&[b'\r']); + self.digest.hash_update(&[b'\r']); } // Next, find all occurrences of "\r\n", inputting the slice @@ -221,7 +409,7 @@ impl<'a> Write for DigestWriter<'a> { // the beginning of this "\r\n". let mut i_prev = 0; for i in memmem::find_iter(buf, b"\r\n") { - self.digest.input(&buf[i_prev..i]); + self.digest.hash_update(&buf[i_prev..i]); i_prev = i + 1; } @@ -233,10 +421,10 @@ impl<'a> Write for DigestWriter<'a> { // blocks of the input. if n > 0 && buf[n - 1] == b'\r' { self.was_last_character_carriage_return = true; - self.digest.input(&buf[i_prev..n - 1]); + self.digest.hash_update(&buf[i_prev..n - 1]); } else { self.was_last_character_carriage_return = false; - self.digest.input(&buf[i_prev..n]); + self.digest.hash_update(&buf[i_prev..n]); } // Even though we dropped a "\r" for each "\r\n" we found, we @@ -272,14 +460,14 @@ mod tests { let mut writer_crlf = DigestWriter::new(&mut digest, false); writer_crlf.write_all(&[b'\r']).unwrap(); writer_crlf.write_all(&[b'\n']).unwrap(); - writer_crlf.finalize(); + writer_crlf.hash_finalize(); let result_crlf = digest.result_str(); // We expect "\r\n" to be replaced with "\n" in text mode on Windows. let mut digest = Box::new(md5::Md5::new()) as Box; let mut writer_lf = DigestWriter::new(&mut digest, false); writer_lf.write_all(&[b'\n']).unwrap(); - writer_lf.finalize(); + writer_lf.hash_finalize(); let result_lf = digest.result_str(); assert_eq!(result_crlf, result_lf); diff --git a/src/uucore/src/lib/lib.rs b/src/uucore/src/lib/lib.rs index 66f8881d6..5cbf58faa 100644 --- a/src/uucore/src/lib/lib.rs +++ b/src/uucore/src/lib/lib.rs @@ -46,6 +46,8 @@ pub use crate::features::lines; pub use crate::features::memo; #[cfg(feature = "ringbuffer")] pub use crate::features::ringbuffer; +#[cfg(feature = "sum")] +pub use crate::features::sum; // * (platform-specific) feature-gated modules // ** non-windows (i.e. Unix + Fuchsia) diff --git a/tests/by-util/test_cksum.rs b/tests/by-util/test_cksum.rs index 523715126..361a9c472 100644 --- a/tests/by-util/test_cksum.rs +++ b/tests/by-util/test_cksum.rs @@ -114,3 +114,79 @@ fn test_stdin_larger_than_128_bytes() { assert_eq!(cksum, 945_881_979); assert_eq!(bytes_cnt, 2058); } + +#[test] +fn test_sha1_single_file() { + new_ucmd!() + .arg("-a=sha1") + .arg("lorem_ipsum.txt") + .succeeds() + .stdout_is("ab1dd0bae1d8883a3d18a66de6afbd28252cfbef 772 lorem_ipsum.txt\n"); +} + +#[test] +fn test_sm3_single_file() { + new_ucmd!() + .arg("-a=sm3") + .arg("lorem_ipsum.txt") + .succeeds() + .stdout_is( + "6d296b805d060bfed22808df308dbb9b4317794dd4ed6740a10770a782699bc2 772 lorem_ipsum.txt\n", + ); +} + +#[test] +fn test_bsd_single_file() { + new_ucmd!() + .arg("-a=bsd") + .arg("lorem_ipsum.txt") + .succeeds() + .stdout_only_fixture("bsd_single_file.expected"); +} + +#[test] +fn test_bsd_multiple_files() { + new_ucmd!() + .arg("-a=bsd") + .arg("lorem_ipsum.txt") + .arg("alice_in_wonderland.txt") + .succeeds() + .stdout_only_fixture("bsd_multiple_files.expected"); +} + +#[test] +fn test_bsd_stdin() { + new_ucmd!() + .arg("-a=bsd") + .pipe_in_fixture("lorem_ipsum.txt") + .succeeds() + .stdout_only_fixture("bsd_stdin.expected"); +} + +#[test] +fn test_sysv_single_file() { + new_ucmd!() + .arg("-a=sysv") + .arg("lorem_ipsum.txt") + .succeeds() + .stdout_only_fixture("sysv_single_file.expected"); +} + +#[test] +fn test_sysv_multiple_files() { + new_ucmd!() + .arg("-a=sysv") + .arg("lorem_ipsum.txt") + .arg("alice_in_wonderland.txt") + .succeeds() + .stdout_only_fixture("sysv_multiple_files.expected"); +} + +#[test] +fn test_sysv_stdin() { + new_ucmd!() + .arg("-a=sysv") + .pipe_in_fixture("lorem_ipsum.txt") + .succeeds() + .stdout_only_fixture("sysv_stdin.expected"); +} diff --git a/tests/fixtures/cksum/bsd_multiple_files.expected b/tests/fixtures/cksum/bsd_multiple_files.expected new file mode 100644 index 000000000..941a2a512 --- /dev/null +++ b/tests/fixtures/cksum/bsd_multiple_files.expected @@ -0,0 +1,2 @@ +08109 1 lorem_ipsum.txt +01814 1 alice_in_wonderland.txt diff --git a/tests/fixtures/cksum/bsd_single_file.expected b/tests/fixtures/cksum/bsd_single_file.expected new file mode 100644 index 000000000..293ada3bd --- /dev/null +++ b/tests/fixtures/cksum/bsd_single_file.expected @@ -0,0 +1 @@ +08109 1 lorem_ipsum.txt diff --git a/tests/fixtures/cksum/bsd_stdin.expected b/tests/fixtures/cksum/bsd_stdin.expected new file mode 100644 index 000000000..4843ba082 --- /dev/null +++ b/tests/fixtures/cksum/bsd_stdin.expected @@ -0,0 +1 @@ +08109 1 diff --git a/tests/fixtures/cksum/sysv_multiple_files.expected b/tests/fixtures/cksum/sysv_multiple_files.expected new file mode 100644 index 000000000..83a6d6d83 --- /dev/null +++ b/tests/fixtures/cksum/sysv_multiple_files.expected @@ -0,0 +1,2 @@ +6985 2 lorem_ipsum.txt +27441 1 alice_in_wonderland.txt diff --git a/tests/fixtures/cksum/sysv_single_file.expected b/tests/fixtures/cksum/sysv_single_file.expected new file mode 100644 index 000000000..e0f7252cb --- /dev/null +++ b/tests/fixtures/cksum/sysv_single_file.expected @@ -0,0 +1 @@ +6985 2 lorem_ipsum.txt diff --git a/tests/fixtures/cksum/sysv_stdin.expected b/tests/fixtures/cksum/sysv_stdin.expected new file mode 100644 index 000000000..f0fba8c81 --- /dev/null +++ b/tests/fixtures/cksum/sysv_stdin.expected @@ -0,0 +1 @@ +6985 2 From cf72c990f25c63ccf5525aeb0a39e88d63ebcc35 Mon Sep 17 00:00:00 2001 From: Jeffrey Finkelstein Date: Sat, 12 Nov 2022 15:44:32 -0500 Subject: [PATCH 04/94] dd: create Source enum and simpler Input struct This mirrors a recent commit that introduced the `Dest` enum and a simplified `Output` struct. These changes allow us to add new types of inputs and output more easily. --- src/uu/dd/src/dd.rs | 155 ++++++++++++++++++++++++-------------------- 1 file changed, 86 insertions(+), 69 deletions(-) diff --git a/src/uu/dd/src/dd.rs b/src/uu/dd/src/dd.rs index 813d60ceb..cd6e59157 100644 --- a/src/uu/dd/src/dd.rs +++ b/src/uu/dd/src/dd.rs @@ -27,7 +27,7 @@ use std::cmp; use std::env; use std::ffi::OsString; use std::fs::{File, OpenOptions}; -use std::io::{self, Read, Seek, SeekFrom, Stdout, Write}; +use std::io::{self, Read, Seek, SeekFrom, Stdin, Stdout, Write}; #[cfg(any(target_os = "linux", target_os = "android"))] use std::os::unix::fs::OpenOptionsExt; use std::path::Path; @@ -90,30 +90,84 @@ impl Num { } } -struct Input<'a, R: Read> { - src: R, +/// Data sources. +enum Source { + /// Input from stdin. + Stdin(Stdin), + + /// Input from a file. + File(File), +} + +impl Source { + fn skip(&mut self, n: u64) -> io::Result { + match self { + Self::Stdin(stdin) => match io::copy(&mut stdin.take(n), &mut io::sink()) { + Ok(m) if m < n => { + show_error!("'standard input': cannot skip to specified offset"); + Ok(m) + } + Ok(m) => Ok(m), + Err(e) => Err(e), + }, + Self::File(f) => f.seek(io::SeekFrom::Start(n)), + } + } +} + +impl Read for Source { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match self { + Self::Stdin(stdin) => stdin.read(buf), + Self::File(f) => f.read(buf), + } + } +} + +/// The source of the data, configured with the given settings. +/// +/// Use the [`Input::new_stdin`] or [`Input::new_file`] functions to +/// construct a new instance of this struct. Then pass the instance to +/// the [`Output::dd_out`] function to execute the main copy operation +/// for `dd`. +struct Input<'a> { + /// The source from which bytes will be read. + src: Source, + + /// Configuration settings for how to read the data. settings: &'a Settings, } -impl<'a> Input<'a, io::Stdin> { - fn new(settings: &'a Settings) -> UResult { - let mut input = Self { - src: io::stdin(), - settings, +impl<'a> Input<'a> { + /// Instantiate this struct with stdin as a source. + fn new_stdin(settings: &'a Settings) -> UResult { + let mut src = Source::Stdin(io::stdin()); + if settings.skip > 0 { + src.skip(settings.skip)?; + } + Ok(Self { src, settings }) + } + + /// Instantiate this struct with the named file as a source. + fn new_file(filename: &Path, settings: &'a Settings) -> UResult { + let src = { + let mut opts = OpenOptions::new(); + opts.read(true); + + #[cfg(any(target_os = "linux", target_os = "android"))] + if let Some(libc_flags) = make_linux_iflags(&settings.iflags) { + opts.custom_flags(libc_flags); + } + + opts.open(filename) + .map_err_context(|| format!("failed to open {}", filename.quote()))? }; + let mut src = Source::File(src); if settings.skip > 0 { - if let Err(e) = input.read_skip(settings.skip) { - if let io::ErrorKind::UnexpectedEof = e.kind() { - show_error!("'standard input': cannot skip to specified offset"); - } else { - return io::Result::Err(e) - .map_err_context(|| "I/O error while skipping".to_string()); - } - } + src.skip(settings.skip)?; } - - Ok(input) + Ok(Self { src, settings }) } } @@ -153,31 +207,7 @@ fn make_linux_iflags(iflags: &IFlags) -> Option { } } -impl<'a> Input<'a, File> { - fn new(filename: &Path, settings: &'a Settings) -> UResult { - let mut src = { - let mut opts = OpenOptions::new(); - opts.read(true); - - #[cfg(any(target_os = "linux", target_os = "android"))] - if let Some(libc_flags) = make_linux_iflags(&settings.iflags) { - opts.custom_flags(libc_flags); - } - - opts.open(filename) - .map_err_context(|| format!("failed to open {}", filename.quote()))? - }; - - if settings.skip > 0 { - src.seek(io::SeekFrom::Start(settings.skip)) - .map_err_context(|| "failed to seek in input file".to_string())?; - } - - Ok(Self { src, settings }) - } -} - -impl<'a, R: Read> Read for Input<'a, R> { +impl<'a> Read for Input<'a> { fn read(&mut self, buf: &mut [u8]) -> io::Result { let mut base_idx = 0; let target_len = buf.len(); @@ -200,7 +230,7 @@ impl<'a, R: Read> Read for Input<'a, R> { } } -impl<'a, R: Read> Input<'a, R> { +impl<'a> Input<'a> { /// Fills a given buffer. /// Reads in increments of 'self.ibs'. /// The start of each ibs-sized read follows the previous one. @@ -266,20 +296,6 @@ impl<'a, R: Read> Input<'a, R> { records_truncated: 0, }) } - - /// Skips amount_to_read bytes from the Input by copying into a sink - fn read_skip(&mut self, amount_to_read: u64) -> std::io::Result<()> { - let copy_result = io::copy(&mut self.src.by_ref().take(amount_to_read), &mut io::sink()); - if let Ok(n) = copy_result { - if n != amount_to_read { - io::Result::Err(io::Error::new(io::ErrorKind::UnexpectedEof, "")) - } else { - Ok(()) - } - } else { - io::Result::Err(copy_result.unwrap_err()) - } - } } enum Density { @@ -485,7 +501,7 @@ impl<'a> Output<'a> { /// /// If there is a problem reading from the input or writing to /// this output. - fn dd_out(mut self, mut i: Input) -> std::io::Result<()> { + fn dd_out(mut self, mut i: Input) -> std::io::Result<()> { // The read and write statistics. // // These objects are counters, initialized to zero. After each @@ -645,12 +661,13 @@ fn make_linux_oflags(oflags: &OFlags) -> Option { } } -/// Read helper performs read operations common to all dd reads, and dispatches the buffer to relevant helper functions as dictated by the operations requested by the user. -fn read_helper( - i: &mut Input, - buf: &mut Vec, - bsize: usize, -) -> std::io::Result { +/// Read from an input (that is, a source of bytes) into the given buffer. +/// +/// This function also performs any conversions as specified by +/// `conv=swab` or `conv=block` command-line arguments. This function +/// mutates the `buf` argument in-place. The returned [`ReadStat`] +/// indicates how many blocks were read. +fn read_helper(i: &mut Input, buf: &mut Vec, bsize: usize) -> std::io::Result { // Local Helper Fns ------------------------------------------------- fn perform_swab(buf: &mut [u8]) { for base in (1..buf.len()).step_by(2) { @@ -794,17 +811,17 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { match (&settings.infile, &settings.outfile) { (Some(infile), Some(outfile)) => { - let i = Input::::new(Path::new(&infile), &settings)?; + let i = Input::new_file(Path::new(&infile), &settings)?; let o = Output::new_file(Path::new(&outfile), &settings)?; o.dd_out(i).map_err_context(|| "IO error".to_string()) } (None, Some(outfile)) => { - let i = Input::::new(&settings)?; + let i = Input::new_stdin(&settings)?; let o = Output::new_file(Path::new(&outfile), &settings)?; o.dd_out(i).map_err_context(|| "IO error".to_string()) } (Some(infile), None) => { - let i = Input::::new(Path::new(&infile), &settings)?; + let i = Input::new_file(Path::new(&infile), &settings)?; if is_stdout_redirected_to_seekable_file() { let filename = stdout_canonicalized(); let o = Output::new_file(Path::new(&filename), &settings)?; @@ -815,7 +832,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { } } (None, None) => { - let i = Input::::new(&settings)?; + let i = Input::new_stdin(&settings)?; if is_stdout_redirected_to_seekable_file() { let filename = stdout_canonicalized(); let o = Output::new_file(Path::new(&filename), &settings)?; From 9c3f810f973a1840fbf1ed4d1856a423b52fb166 Mon Sep 17 00:00:00 2001 From: Jeffrey Finkelstein Date: Sun, 13 Nov 2022 08:46:01 -0500 Subject: [PATCH 05/94] dd: add support for seeking in output FIFOs For example, `dd seek=1 of=fifo` will now work. --- src/uu/dd/src/dd.rs | 127 ++++++++++++++++++++++++++++----------- tests/by-util/test_dd.rs | 39 ++++++++++++ 2 files changed, 132 insertions(+), 34 deletions(-) diff --git a/src/uu/dd/src/dd.rs b/src/uu/dd/src/dd.rs index cd6e59157..1a00af8b5 100644 --- a/src/uu/dd/src/dd.rs +++ b/src/uu/dd/src/dd.rs @@ -28,6 +28,8 @@ use std::env; use std::ffi::OsString; use std::fs::{File, OpenOptions}; use std::io::{self, Read, Seek, SeekFrom, Stdin, Stdout, Write}; +#[cfg(unix)] +use std::os::unix::fs::FileTypeExt; #[cfg(any(target_os = "linux", target_os = "android"))] use std::os::unix::fs::OpenOptionsExt; use std::path::Path; @@ -313,6 +315,14 @@ enum Dest { /// The [`Density`] component indicates whether to attempt to /// write a sparse file when all-zero blocks are encountered. File(File, Density), + + /// Output to a named pipe, also known as a FIFO. + #[cfg(unix)] + Fifo(File), + + /// Output to nothing, dropping each byte written to the output. + #[cfg(unix)] + Sink, } impl Dest { @@ -323,6 +333,13 @@ impl Dest { f.flush()?; f.sync_all() } + #[cfg(unix)] + Self::Fifo(f) => { + f.flush()?; + f.sync_all() + } + #[cfg(unix)] + Self::Sink => Ok(()), } } @@ -333,6 +350,13 @@ impl Dest { f.flush()?; f.sync_data() } + #[cfg(unix)] + Self::Fifo(f) => { + f.flush()?; + f.sync_data() + } + #[cfg(unix)] + Self::Sink => Ok(()), } } @@ -340,17 +364,24 @@ impl Dest { match self { Self::Stdout(stdout) => io::copy(&mut io::repeat(0).take(n), stdout), Self::File(f, _) => f.seek(io::SeekFrom::Start(n)), + #[cfg(unix)] + Self::Fifo(f) => { + // Seeking in a named pipe means *reading* from the pipe. + io::copy(&mut f.take(n), &mut io::sink()) + } + #[cfg(unix)] + Self::Sink => Ok(0), } } /// Truncate the underlying file to the current stream position, if possible. fn truncate(&mut self) -> io::Result<()> { match self { - Self::Stdout(_) => Ok(()), Self::File(f, _) => { let pos = f.stream_position()?; f.set_len(pos) } + _ => Ok(()), } } } @@ -373,6 +404,10 @@ impl Write for Dest { } Self::File(f, _) => f.write(buf), Self::Stdout(stdout) => stdout.write(buf), + #[cfg(unix)] + Self::Fifo(f) => f.write(buf), + #[cfg(unix)] + Self::Sink => Ok(buf.len()), } } @@ -380,6 +415,10 @@ impl Write for Dest { match self { Self::Stdout(stdout) => stdout.flush(), Self::File(f, _) => f.flush(), + #[cfg(unix)] + Self::Fifo(f) => f.flush(), + #[cfg(unix)] + Self::Sink => Ok(()), } } } @@ -449,6 +488,35 @@ impl<'a> Output<'a> { Ok(Self { dst, settings }) } + /// Instantiate this struct with the given named pipe as a destination. + #[cfg(unix)] + fn new_fifo(filename: &Path, settings: &'a Settings) -> UResult { + // We simulate seeking in a FIFO by *reading*, so we open the + // file for reading. But then we need to close the file and + // re-open it for writing. + if settings.seek > 0 { + Dest::Fifo(File::open(filename)?).seek(settings.seek)?; + } + // If `count=0`, then we don't bother opening the file for + // writing because that would cause this process to block + // indefinitely. + if let Some(Num::Blocks(0) | Num::Bytes(0)) = settings.count { + let dst = Dest::Sink; + return Ok(Self { dst, settings }); + } + // At this point, we know there is at least one block to write + // to the output, so we open the file for writing. + let mut opts = OpenOptions::new(); + opts.write(true) + .create(!settings.oconv.nocreat) + .create_new(settings.oconv.excl) + .append(settings.oflags.append); + #[cfg(any(target_os = "linux", target_os = "android"))] + opts.custom_flags(make_linux_oflags(&settings.oflags).unwrap_or(0)); + let dst = Dest::Fifo(opts.open(filename)?); + Ok(Self { dst, settings }) + } + /// Write the given bytes one block at a time. /// /// This may write partial blocks (for example, if the underlying @@ -795,6 +863,17 @@ fn is_stdout_redirected_to_seekable_file() -> bool { } } +/// Decide whether the named file is a named pipe, also known as a FIFO. +#[cfg(unix)] +fn is_fifo(filename: &str) -> bool { + if let Ok(metadata) = std::fs::metadata(filename) { + if metadata.file_type().is_fifo() { + return true; + } + } + false +} + #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { let args = args.collect_ignore(); @@ -809,40 +888,20 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { .collect::>()[..], )?; - match (&settings.infile, &settings.outfile) { - (Some(infile), Some(outfile)) => { - let i = Input::new_file(Path::new(&infile), &settings)?; - let o = Output::new_file(Path::new(&outfile), &settings)?; - o.dd_out(i).map_err_context(|| "IO error".to_string()) + let i = match settings.infile { + Some(ref infile) => Input::new_file(Path::new(&infile), &settings)?, + None => Input::new_stdin(&settings)?, + }; + let o = match settings.outfile { + #[cfg(unix)] + Some(ref outfile) if is_fifo(outfile) => Output::new_fifo(Path::new(&outfile), &settings)?, + Some(ref outfile) => Output::new_file(Path::new(&outfile), &settings)?, + None if is_stdout_redirected_to_seekable_file() => { + Output::new_file(Path::new(&stdout_canonicalized()), &settings)? } - (None, Some(outfile)) => { - let i = Input::new_stdin(&settings)?; - let o = Output::new_file(Path::new(&outfile), &settings)?; - o.dd_out(i).map_err_context(|| "IO error".to_string()) - } - (Some(infile), None) => { - let i = Input::new_file(Path::new(&infile), &settings)?; - if is_stdout_redirected_to_seekable_file() { - let filename = stdout_canonicalized(); - let o = Output::new_file(Path::new(&filename), &settings)?; - o.dd_out(i).map_err_context(|| "IO error".to_string()) - } else { - let o = Output::new_stdout(&settings)?; - o.dd_out(i).map_err_context(|| "IO error".to_string()) - } - } - (None, None) => { - let i = Input::new_stdin(&settings)?; - if is_stdout_redirected_to_seekable_file() { - let filename = stdout_canonicalized(); - let o = Output::new_file(Path::new(&filename), &settings)?; - o.dd_out(i).map_err_context(|| "IO error".to_string()) - } else { - let o = Output::new_stdout(&settings)?; - o.dd_out(i).map_err_context(|| "IO error".to_string()) - } - } - } + None => Output::new_stdout(&settings)?, + }; + o.dd_out(i).map_err_context(|| "IO error".to_string()) } pub fn uu_app() -> Command { diff --git a/tests/by-util/test_dd.rs b/tests/by-util/test_dd.rs index 19050965c..554ce785d 100644 --- a/tests/by-util/test_dd.rs +++ b/tests/by-util/test_dd.rs @@ -5,6 +5,8 @@ use crate::common::util::*; use std::fs::{File, OpenOptions}; use std::io::{BufReader, Read, Write}; use std::path::PathBuf; +#[cfg(all(not(windows), not(target_os = "macos")))] +use std::process::{Command, Stdio}; #[cfg(not(windows))] use std::thread::sleep; #[cfg(not(windows))] @@ -1442,3 +1444,40 @@ fn test_sparse() { // number of blocks stored on disk may be zero. assert_eq!(at.metadata("infile").len(), at.metadata("outfile").len()); } + +// TODO These FIFO tests should work on macos, but some issue is +// causing our implementation of dd to wait indefinitely when it +// shouldn't. + +/// Test that a seek on an output FIFO results in a read. +#[test] +#[cfg(all(unix, not(target_os = "macos"), not(target_os = "freebsd")))] +fn test_seek_output_fifo() { + let ts = TestScenario::new(util_name!()); + let at = &ts.fixtures; + at.mkfifo("fifo"); + + // TODO When `dd` is a bit more advanced, we could use the uutils + // version of dd here as well. + let child = Command::new("dd") + .current_dir(&at.subdir) + .args([ + "count=1", + "if=/dev/zero", + &format!("of={}", at.plus_as_string("fifo")), + "status=noxfer", + ]) + .stderr(Stdio::piped()) + .spawn() + .expect("failed to execute child process"); + + ts.ucmd() + .args(&["count=0", "seek=1", "of=fifo", "status=noxfer"]) + .succeeds() + .stderr_only("0+0 records in\n0+0 records out\n"); + + let output = child.wait_with_output().unwrap(); + assert!(output.status.success()); + assert!(output.stdout.is_empty()); + assert_eq!(&output.stderr, b"1+0 records in\n1+0 records out\n"); +} From c52647a6324fd7b73fa42b62672838c1db32bf6b Mon Sep 17 00:00:00 2001 From: Jeffrey Finkelstein Date: Sat, 19 Nov 2022 11:36:08 -0500 Subject: [PATCH 06/94] dd: add support for skipping in input FIFO For example, `dd skip=1 if=fifo` will now work. --- src/uu/dd/src/dd.rs | 24 ++++++++++++++++++++++++ tests/by-util/test_dd.rs | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/src/uu/dd/src/dd.rs b/src/uu/dd/src/dd.rs index 1a00af8b5..fdd7947cf 100644 --- a/src/uu/dd/src/dd.rs +++ b/src/uu/dd/src/dd.rs @@ -99,6 +99,10 @@ enum Source { /// Input from a file. File(File), + + /// Input from a named pipe, also known as a FIFO. + #[cfg(unix)] + Fifo(File), } impl Source { @@ -113,6 +117,8 @@ impl Source { Err(e) => Err(e), }, Self::File(f) => f.seek(io::SeekFrom::Start(n)), + #[cfg(unix)] + Self::Fifo(f) => io::copy(&mut f.take(n), &mut io::sink()), } } } @@ -122,6 +128,8 @@ impl Read for Source { match self { Self::Stdin(stdin) => stdin.read(buf), Self::File(f) => f.read(buf), + #[cfg(unix)] + Self::Fifo(f) => f.read(buf), } } } @@ -171,6 +179,20 @@ impl<'a> Input<'a> { } Ok(Self { src, settings }) } + + /// Instantiate this struct with the named pipe as a source. + #[cfg(unix)] + fn new_fifo(filename: &Path, settings: &'a Settings) -> UResult { + let mut opts = OpenOptions::new(); + opts.read(true); + #[cfg(any(target_os = "linux", target_os = "android"))] + opts.custom_flags(make_linux_iflags(&settings.iflags).unwrap_or(0)); + let mut src = Source::Fifo(opts.open(filename)?); + if settings.skip > 0 { + src.skip(settings.skip)?; + } + Ok(Self { src, settings }) + } } #[cfg(any(target_os = "linux", target_os = "android"))] @@ -889,6 +911,8 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { )?; let i = match settings.infile { + #[cfg(unix)] + Some(ref infile) if is_fifo(infile) => Input::new_fifo(Path::new(&infile), &settings)?, Some(ref infile) => Input::new_file(Path::new(&infile), &settings)?, None => Input::new_stdin(&settings)?, }; diff --git a/tests/by-util/test_dd.rs b/tests/by-util/test_dd.rs index 554ce785d..5deeb12f0 100644 --- a/tests/by-util/test_dd.rs +++ b/tests/by-util/test_dd.rs @@ -1481,3 +1481,36 @@ fn test_seek_output_fifo() { assert!(output.stdout.is_empty()); assert_eq!(&output.stderr, b"1+0 records in\n1+0 records out\n"); } + +/// Test that a skip on an input FIFO results in a read. +#[test] +#[cfg(all(unix, not(target_os = "macos"), not(target_os = "freebsd")))] +fn test_skip_input_fifo() { + let ts = TestScenario::new(util_name!()); + let at = &ts.fixtures; + at.mkfifo("fifo"); + + // TODO When `dd` is a bit more advanced, we could use the uutils + // version of dd here as well. + let child = Command::new("dd") + .current_dir(&at.subdir) + .args([ + "count=1", + "if=/dev/zero", + &format!("of={}", at.plus_as_string("fifo")), + "status=noxfer", + ]) + .stderr(Stdio::piped()) + .spawn() + .expect("failed to execute child process"); + + ts.ucmd() + .args(&["count=0", "skip=1", "if=fifo", "status=noxfer"]) + .succeeds() + .stderr_only("0+0 records in\n0+0 records out\n"); + + let output = child.wait_with_output().unwrap(); + assert!(output.status.success()); + assert!(output.stdout.is_empty()); + assert_eq!(&output.stderr, b"1+0 records in\n1+0 records out\n"); +} From 1c230fd7795932dadd89f5cc29bedfda7b157bb8 Mon Sep 17 00:00:00 2001 From: Joining7943 <111500881+Joining7943@users.noreply.github.com> Date: Wed, 25 Jan 2023 03:40:39 +0100 Subject: [PATCH 07/94] tests/util: Refactor UCommand and TestScenario. Summary of changes in UCommand: * Extend UCommand by builder methods and simplify methods in TestScenario * Simplify code structures where possible. Add documentation. * Store bin_path as PathBuf and util_name as String in all structs * Remove UCommand::util and make bin_path, temp_dir private * Rename UCommand::with_limit -> UCommand::limit Summary of changes in TestScenario: * Rename some parameters in TestScenario methods to be more descriptive * Remove ucmd_keepenv, cmd_keepenv from TestScenario. Use UCommand::keep_env instead. --- tests/by-util/test_cat.rs | 2 +- tests/by-util/test_chmod.rs | 2 +- tests/by-util/test_chown.rs | 14 +- tests/by-util/test_cp.rs | 11 +- tests/by-util/test_env.rs | 3 +- tests/by-util/test_mktemp.rs | 6 +- tests/by-util/test_nproc.rs | 51 +++-- tests/by-util/test_printenv.rs | 6 +- tests/by-util/test_sort.rs | 21 +- tests/by-util/test_uptime.rs | 3 +- tests/by-util/test_users.rs | 3 +- tests/common/util.rs | 339 +++++++++++++++++++-------------- 12 files changed, 276 insertions(+), 185 deletions(-) diff --git a/tests/by-util/test_cat.rs b/tests/by-util/test_cat.rs index bef9e76e4..8b6022c93 100644 --- a/tests/by-util/test_cat.rs +++ b/tests/by-util/test_cat.rs @@ -103,7 +103,7 @@ fn test_closes_file_descriptors() { "alpha.txt", "alpha.txt", ]) - .with_limit(Resource::NOFILE, 9, 9) + .limit(Resource::NOFILE, 9, 9) .succeeds(); } diff --git a/tests/by-util/test_chmod.rs b/tests/by-util/test_chmod.rs index 072756604..52b43d32e 100644 --- a/tests/by-util/test_chmod.rs +++ b/tests/by-util/test_chmod.rs @@ -48,7 +48,7 @@ fn run_single_test(test: &TestCase, at: &AtPath, mut ucmd: UCommand) { let r = ucmd.run(); if !r.succeeded() { println!("{}", r.stderr_str()); - panic!("{}: failed", ucmd); + panic!("{ucmd}: failed"); } let perms = at.metadata(TEST_FILE).permissions().mode(); diff --git a/tests/by-util/test_chown.rs b/tests/by-util/test_chown.rs index 5237a7cf7..c0e210435 100644 --- a/tests/by-util/test_chown.rs +++ b/tests/by-util/test_chown.rs @@ -396,7 +396,7 @@ fn test_chown_only_user_id() { let scene = TestScenario::new(util_name!()); let at = &scene.fixtures; - let result = scene.cmd_keepenv("id").arg("-u").run(); + let result = scene.cmd("id").keep_env().arg("-u").run(); if skipping_test_is_okay(&result, "id: cannot find name for group ID") { return; } @@ -430,7 +430,7 @@ fn test_chown_fail_id() { let scene = TestScenario::new(util_name!()); let at = &scene.fixtures; - let result = scene.cmd_keepenv("id").arg("-u").run(); + let result = scene.cmd("id").keep_env().arg("-u").run(); if skipping_test_is_okay(&result, "id: cannot find name for group ID") { return; } @@ -487,7 +487,7 @@ fn test_chown_only_group_id() { let scene = TestScenario::new(util_name!()); let at = &scene.fixtures; - let result = scene.cmd_keepenv("id").arg("-g").run(); + let result = scene.cmd("id").keep_env().arg("-g").run(); if skipping_test_is_okay(&result, "id: cannot find name for group ID") { return; } @@ -551,14 +551,14 @@ fn test_chown_owner_group_id() { let scene = TestScenario::new(util_name!()); let at = &scene.fixtures; - let result = scene.cmd_keepenv("id").arg("-u").run(); + let result = scene.cmd("id").keep_env().arg("-u").run(); if skipping_test_is_okay(&result, "id: cannot find name for group ID") { return; } let user_id = String::from(result.stdout_str().trim()); assert!(!user_id.is_empty()); - let result = scene.cmd_keepenv("id").arg("-g").run(); + let result = scene.cmd("id").keep_env().arg("-g").run(); if skipping_test_is_okay(&result, "id: cannot find name for group ID") { return; } @@ -612,14 +612,14 @@ fn test_chown_owner_group_mix() { let scene = TestScenario::new(util_name!()); let at = &scene.fixtures; - let result = scene.cmd_keepenv("id").arg("-u").run(); + let result = scene.cmd("id").keep_env().arg("-u").run(); if skipping_test_is_okay(&result, "id: cannot find name for group ID") { return; } let user_id = String::from(result.stdout_str().trim()); assert!(!user_id.is_empty()); - let result = scene.cmd_keepenv("id").arg("-gn").run(); + let result = scene.cmd("id").keep_env().arg("-gn").run(); if skipping_test_is_okay(&result, "id: cannot find name for group ID") { return; } diff --git a/tests/by-util/test_cp.rs b/tests/by-util/test_cp.rs index 790383ded..01eb9bd00 100644 --- a/tests/by-util/test_cp.rs +++ b/tests/by-util/test_cp.rs @@ -1545,7 +1545,7 @@ fn test_closes_file_descriptors() { .arg("--reflink=auto") .arg("dir_with_10_files/") .arg("dir_with_10_files_new/") - .with_limit(Resource::NOFILE, limit_fd, limit_fd) + .limit(Resource::NOFILE, limit_fd, limit_fd) .succeeds(); } @@ -1692,7 +1692,8 @@ fn test_cp_reflink_always_override() { .succeeds(); if !scene - .cmd_keepenv("env") + .cmd("env") + .keep_env() .args(&["mkfs.btrfs", "--rootdir", ROOTDIR, DISK]) .run() .succeeded() @@ -1704,7 +1705,8 @@ fn test_cp_reflink_always_override() { scene.fixtures.mkdir(MOUNTPOINT); let mount = scene - .cmd_keepenv("sudo") + .cmd("sudo") + .keep_env() .args(&["-E", "--non-interactive", "mount", DISK, MOUNTPOINT]) .run(); @@ -1730,7 +1732,8 @@ fn test_cp_reflink_always_override() { .succeeds(); scene - .cmd_keepenv("sudo") + .cmd("sudo") + .keep_env() .args(&["-E", "--non-interactive", "umount", MOUNTPOINT]) .succeeds(); } diff --git a/tests/by-util/test_env.rs b/tests/by-util/test_env.rs index ef10512ab..9b386541c 100644 --- a/tests/by-util/test_env.rs +++ b/tests/by-util/test_env.rs @@ -156,7 +156,8 @@ fn test_unset_variable() { // This test depends on the HOME variable being pre-defined by the // default shell let out = TestScenario::new(util_name!()) - .ucmd_keepenv() + .ucmd() + .keep_env() .arg("-u") .arg("HOME") .succeeds() diff --git a/tests/by-util/test_mktemp.rs b/tests/by-util/test_mktemp.rs index d6926c41b..109963edf 100644 --- a/tests/by-util/test_mktemp.rs +++ b/tests/by-util/test_mktemp.rs @@ -425,7 +425,8 @@ fn test_mktemp_tmpdir_one_arg() { let scene = TestScenario::new(util_name!()); let result = scene - .ucmd_keepenv() + .ucmd() + .keep_env() .arg("--tmpdir") .arg("apt-key-gpghome.XXXXXXXXXX") .succeeds(); @@ -438,7 +439,8 @@ fn test_mktemp_directory_tmpdir() { let scene = TestScenario::new(util_name!()); let result = scene - .ucmd_keepenv() + .ucmd() + .keep_env() .arg("--directory") .arg("--tmpdir") .arg("apt-key-gpghome.XXXXXXXXXX") diff --git a/tests/by-util/test_nproc.rs b/tests/by-util/test_nproc.rs index 3260e46e7..abae40697 100644 --- a/tests/by-util/test_nproc.rs +++ b/tests/by-util/test_nproc.rs @@ -20,7 +20,8 @@ fn test_nproc_all_omp() { assert!(nproc > 0); let result = TestScenario::new(util_name!()) - .ucmd_keepenv() + .ucmd() + .keep_env() .env("OMP_NUM_THREADS", "60") .succeeds(); @@ -28,7 +29,8 @@ fn test_nproc_all_omp() { assert_eq!(nproc_omp, 60); let result = TestScenario::new(util_name!()) - .ucmd_keepenv() + .ucmd() + .keep_env() .env("OMP_NUM_THREADS", "1") // Has no effect .arg("--all") .succeeds(); @@ -37,7 +39,8 @@ fn test_nproc_all_omp() { // If the parsing fails, returns the number of CPU let result = TestScenario::new(util_name!()) - .ucmd_keepenv() + .ucmd() + .keep_env() .env("OMP_NUM_THREADS", "incorrectnumber") // returns the number CPU .succeeds(); let nproc_omp: u8 = result.stdout_str().trim().parse().unwrap(); @@ -51,7 +54,8 @@ fn test_nproc_ignore() { if nproc_total > 1 { // Ignore all CPU but one let result = TestScenario::new(util_name!()) - .ucmd_keepenv() + .ucmd() + .keep_env() .arg("--ignore") .arg((nproc_total - 1).to_string()) .succeeds(); @@ -59,7 +63,8 @@ fn test_nproc_ignore() { assert_eq!(nproc, 1); // Ignore all CPU but one with a string let result = TestScenario::new(util_name!()) - .ucmd_keepenv() + .ucmd() + .keep_env() .arg("--ignore= 1") .succeeds(); let nproc: u8 = result.stdout_str().trim().parse().unwrap(); @@ -70,7 +75,8 @@ fn test_nproc_ignore() { #[test] fn test_nproc_ignore_all_omp() { let result = TestScenario::new(util_name!()) - .ucmd_keepenv() + .ucmd() + .keep_env() .env("OMP_NUM_THREADS", "42") .arg("--ignore=40") .succeeds(); @@ -81,7 +87,8 @@ fn test_nproc_ignore_all_omp() { #[test] fn test_nproc_omp_limit() { let result = TestScenario::new(util_name!()) - .ucmd_keepenv() + .ucmd() + .keep_env() .env("OMP_NUM_THREADS", "42") .env("OMP_THREAD_LIMIT", "0") .succeeds(); @@ -89,7 +96,8 @@ fn test_nproc_omp_limit() { assert_eq!(nproc, 42); let result = TestScenario::new(util_name!()) - .ucmd_keepenv() + .ucmd() + .keep_env() .env("OMP_NUM_THREADS", "42") .env("OMP_THREAD_LIMIT", "2") .succeeds(); @@ -97,7 +105,8 @@ fn test_nproc_omp_limit() { assert_eq!(nproc, 2); let result = TestScenario::new(util_name!()) - .ucmd_keepenv() + .ucmd() + .keep_env() .env("OMP_NUM_THREADS", "42") .env("OMP_THREAD_LIMIT", "2bad") .succeeds(); @@ -109,14 +118,16 @@ fn test_nproc_omp_limit() { assert!(nproc_system > 0); let result = TestScenario::new(util_name!()) - .ucmd_keepenv() + .ucmd() + .keep_env() .env("OMP_THREAD_LIMIT", "1") .succeeds(); let nproc: u8 = result.stdout_str().trim().parse().unwrap(); assert_eq!(nproc, 1); let result = TestScenario::new(util_name!()) - .ucmd_keepenv() + .ucmd() + .keep_env() .env("OMP_NUM_THREADS", "0") .env("OMP_THREAD_LIMIT", "") .succeeds(); @@ -124,7 +135,8 @@ fn test_nproc_omp_limit() { assert_eq!(nproc, nproc_system); let result = TestScenario::new(util_name!()) - .ucmd_keepenv() + .ucmd() + .keep_env() .env("OMP_NUM_THREADS", "") .env("OMP_THREAD_LIMIT", "") .succeeds(); @@ -132,7 +144,8 @@ fn test_nproc_omp_limit() { assert_eq!(nproc, nproc_system); let result = TestScenario::new(util_name!()) - .ucmd_keepenv() + .ucmd() + .keep_env() .env("OMP_NUM_THREADS", "2,2,1") .env("OMP_THREAD_LIMIT", "") .succeeds(); @@ -140,7 +153,8 @@ fn test_nproc_omp_limit() { assert_eq!(2, nproc); let result = TestScenario::new(util_name!()) - .ucmd_keepenv() + .ucmd() + .keep_env() .env("OMP_NUM_THREADS", "2,ignored") .env("OMP_THREAD_LIMIT", "") .succeeds(); @@ -148,7 +162,8 @@ fn test_nproc_omp_limit() { assert_eq!(2, nproc); let result = TestScenario::new(util_name!()) - .ucmd_keepenv() + .ucmd() + .keep_env() .env("OMP_NUM_THREADS", "2,2,1") .env("OMP_THREAD_LIMIT", "0") .succeeds(); @@ -156,7 +171,8 @@ fn test_nproc_omp_limit() { assert_eq!(2, nproc); let result = TestScenario::new(util_name!()) - .ucmd_keepenv() + .ucmd() + .keep_env() .env("OMP_NUM_THREADS", "2,2,1") .env("OMP_THREAD_LIMIT", "1bad") .succeeds(); @@ -164,7 +180,8 @@ fn test_nproc_omp_limit() { assert_eq!(2, nproc); let result = TestScenario::new(util_name!()) - .ucmd_keepenv() + .ucmd() + .keep_env() .env("OMP_NUM_THREADS", "29,2,1") .env("OMP_THREAD_LIMIT", "1bad") .succeeds(); diff --git a/tests/by-util/test_printenv.rs b/tests/by-util/test_printenv.rs index 29ca24857..c4f32705f 100644 --- a/tests/by-util/test_printenv.rs +++ b/tests/by-util/test_printenv.rs @@ -8,7 +8,8 @@ fn test_get_all() { assert_eq!(env::var(key), Ok("VALUE".to_string())); TestScenario::new(util_name!()) - .ucmd_keepenv() + .ucmd() + .keep_env() .succeeds() .stdout_contains("HOME=") .stdout_contains("KEY=VALUE"); @@ -21,7 +22,8 @@ fn test_get_var() { assert_eq!(env::var(key), Ok("VALUE".to_string())); let result = TestScenario::new(util_name!()) - .ucmd_keepenv() + .ucmd() + .keep_env() .arg("KEY") .succeeds(); diff --git a/tests/by-util/test_sort.rs b/tests/by-util/test_sort.rs index 8a03432af..174ac255c 100644 --- a/tests/by-util/test_sort.rs +++ b/tests/by-util/test_sort.rs @@ -31,7 +31,8 @@ fn test_buffer_sizes() { let buffer_sizes = ["0", "50K", "50k", "1M", "100M"]; for buffer_size in &buffer_sizes { TestScenario::new(util_name!()) - .ucmd_keepenv() + .ucmd() + .keep_env() .arg("-n") .arg("-S") .arg(buffer_size) @@ -44,7 +45,8 @@ fn test_buffer_sizes() { let buffer_sizes = ["1000G", "10T"]; for buffer_size in &buffer_sizes { TestScenario::new(util_name!()) - .ucmd_keepenv() + .ucmd() + .keep_env() .arg("-n") .arg("-S") .arg(buffer_size) @@ -918,7 +920,8 @@ fn test_compress_merge() { fn test_compress_fail() { #[cfg(not(windows))] TestScenario::new(util_name!()) - .ucmd_keepenv() + .ucmd() + .keep_env() .args(&[ "ext_sort.txt", "-n", @@ -934,7 +937,8 @@ fn test_compress_fail() { // So, don't check the output #[cfg(windows)] TestScenario::new(util_name!()) - .ucmd_keepenv() + .ucmd() + .keep_env() .args(&[ "ext_sort.txt", "-n", @@ -949,7 +953,8 @@ fn test_compress_fail() { #[test] fn test_merge_batches() { TestScenario::new(util_name!()) - .ucmd_keepenv() + .ucmd() + .keep_env() .timeout(Duration::from_secs(120)) .args(&["ext_sort.txt", "-n", "-S", "150b"]) .succeeds() @@ -959,7 +964,8 @@ fn test_merge_batches() { #[test] fn test_merge_batch_size() { TestScenario::new(util_name!()) - .ucmd_keepenv() + .ucmd() + .keep_env() .arg("--batch-size=2") .arg("-m") .arg("--unique") @@ -1067,7 +1073,8 @@ fn test_output_is_input() { at.touch("file"); at.append("file", input); scene - .ucmd_keepenv() + .ucmd() + .keep_env() .args(&["-m", "-u", "-o", "file", "file", "file", "file"]) .succeeds(); assert_eq!(at.read("file"), input); diff --git a/tests/by-util/test_uptime.rs b/tests/by-util/test_uptime.rs index 946bb30fa..02cba9e8f 100644 --- a/tests/by-util/test_uptime.rs +++ b/tests/by-util/test_uptime.rs @@ -10,7 +10,8 @@ fn test_invalid_arg() { #[test] fn test_uptime() { TestScenario::new(util_name!()) - .ucmd_keepenv() + .ucmd() + .keep_env() .succeeds() .stdout_contains("load average:") .stdout_contains(" up "); diff --git a/tests/by-util/test_users.rs b/tests/by-util/test_users.rs index 747995d99..7fadb2bb2 100644 --- a/tests/by-util/test_users.rs +++ b/tests/by-util/test_users.rs @@ -22,7 +22,8 @@ fn test_users_check_name() { // note: clippy::needless_borrow *false positive* #[allow(clippy::needless_borrow)] let expected = TestScenario::new(&util_name) - .cmd_keepenv(util_name) + .cmd(util_name) + .keep_env() .env("LC_ALL", "C") .succeeds() .stdout_move_str(); diff --git a/tests/common/util.rs b/tests/common/util.rs index 0f3f0d7d8..3097a762f 100644 --- a/tests/common/util.rs +++ b/tests/common/util.rs @@ -3,7 +3,7 @@ // * For the full copyright and license information, please view the LICENSE // * file that was distributed with this source code. -//spell-checker: ignore (linux) rlimit prlimit coreutil ggroups uchild uncaptured scmd SHLVL +//spell-checker: ignore (linux) rlimit prlimit coreutil ggroups uchild uncaptured scmd SHLVL canonicalized #![allow(dead_code)] @@ -13,6 +13,7 @@ use rlimit::prlimit; use rstest::rstest; #[cfg(unix)] use std::borrow::Cow; +use std::collections::VecDeque; #[cfg(not(windows))] use std::ffi::CString; use std::ffi::{OsStr, OsString}; @@ -65,7 +66,7 @@ fn read_scenario_fixture>(tmpd: &Option>, file_rel_p #[derive(Debug, Clone)] pub struct CmdResult { /// bin_path provided by `TestScenario` or `UCommand` - bin_path: String, + bin_path: PathBuf, /// util_name provided by `TestScenario` or `UCommand` util_name: Option, //tmpd is used for convenience functions for asserts against fixtures @@ -79,21 +80,23 @@ pub struct CmdResult { } impl CmdResult { - pub fn new( - bin_path: String, - util_name: Option, + pub fn new( + bin_path: S, + util_name: Option, tmpd: Option>, exit_status: Option, - stdout: T, - stderr: U, + stdout: U, + stderr: V, ) -> Self where - T: Into>, + S: Into, + T: AsRef, U: Into>, + V: Into>, { Self { - bin_path, - util_name, + bin_path: bin_path.into(), + util_name: util_name.map(|s| s.as_ref().into()), tmpd, exit_status, stdout: stdout.into(), @@ -635,7 +638,7 @@ impl CmdResult { self.stderr_only(format!( "{0}: {2}\nTry '{1} {0} --help' for more information.\n", self.util_name.as_ref().unwrap(), // This shouldn't be called using a normal command - self.bin_path, + self.bin_path.display(), msg.as_ref() )) } @@ -1094,18 +1097,21 @@ pub struct TestScenario { } impl TestScenario { - pub fn new(util_name: &str) -> Self { + pub fn new(util_name: T) -> Self + where + T: AsRef, + { let tmpd = Rc::new(TempDir::new().unwrap()); let ts = Self { bin_path: PathBuf::from(TESTS_BINARY), - util_name: String::from(util_name), + util_name: util_name.as_ref().into(), fixtures: AtPath::new(tmpd.as_ref().path()), tmpd, }; let mut fixture_path_builder = env::current_dir().unwrap(); fixture_path_builder.push(TESTS_DIR); fixture_path_builder.push(FIXTURES_DIR); - fixture_path_builder.push(util_name); + fixture_path_builder.push(util_name.as_ref()); if let Ok(m) = fs::metadata(&fixture_path_builder) { if m.is_dir() { recursive_copy(&fixture_path_builder, &ts.fixtures.subdir).unwrap(); @@ -1117,68 +1123,57 @@ impl TestScenario { /// Returns builder for invoking the target uutils binary. Paths given are /// treated relative to the environment's unique temporary test directory. pub fn ucmd(&self) -> UCommand { - self.composite_cmd(&self.bin_path, &self.util_name, true) - } - - /// Returns builder for invoking the target uutils binary. Paths given are - /// treated relative to the environment's unique temporary test directory. - pub fn composite_cmd, T: AsRef>( - &self, - bin: S, - util_name: T, - env_clear: bool, - ) -> UCommand { - UCommand::new_from_tmp(bin, Some(util_name), self.tmpd.clone(), env_clear) + UCommand::from_test_scenario(self) } /// Returns builder for invoking any system command. Paths given are treated /// relative to the environment's unique temporary test directory. - pub fn cmd>(&self, bin: S) -> UCommand { - UCommand::new_from_tmp::(bin, None, self.tmpd.clone(), true) + pub fn cmd>(&self, bin_path: S) -> UCommand { + let mut command = UCommand::new(); + command.bin_path(bin_path); + command.temp_dir(self.tmpd.clone()); + command } /// Returns builder for invoking any uutils command. Paths given are treated /// relative to the environment's unique temporary test directory. - pub fn ccmd>(&self, bin: S) -> UCommand { - self.composite_cmd(&self.bin_path, bin, true) - } - - // different names are used rather than an argument - // because the need to keep the environment is exceedingly rare. - pub fn ucmd_keepenv(&self) -> UCommand { - self.composite_cmd(&self.bin_path, &self.util_name, false) - } - - /// Returns builder for invoking any system command. Paths given are treated - /// relative to the environment's unique temporary test directory. - /// Differs from the builder returned by `cmd` in that `cmd_keepenv` does not call - /// `Command::env_clear` (Clears the entire environment map for the child process.) - pub fn cmd_keepenv>(&self, bin: S) -> UCommand { - UCommand::new_from_tmp::(bin, None, self.tmpd.clone(), false) + pub fn ccmd>(&self, util_name: S) -> UCommand { + UCommand::with_util(util_name, self.tmpd.clone()) } } -/// A `UCommand` is a wrapper around an individual Command that provides several additional features +/// A `UCommand` is a builder wrapping an individual Command that provides several additional features: /// 1. it has convenience functions that are more ergonomic to use for piping in stdin, spawning the command /// and asserting on the results. /// 2. it tracks arguments provided so that in test cases which may provide variations of an arg in loops /// the test failure can display the exact call which preceded an assertion failure. -/// 3. it provides convenience construction arguments to set the Command working directory and/or clear its environment. -#[derive(Debug)] +/// 3. it provides convenience construction methods to set the Command uutils utility and temporary directory. +/// +/// Per default `UCommand` runs a command given as an argument in a shell, platform independently. +/// It does so with safety in mind, so the working directory is set to an individual temporary +/// directory and the environment variables are cleared per default. +/// +/// The default behavior can be changed with builder methods: +/// * [`UCommand::with_util`]: Run `coreutils UTIL_NAME` instead of the shell +/// * [`UCommand::from_test_scenario`]: Run `coreutils UTIL_NAME` instead of the shell in the +/// temporary directory of the [`TestScenario`] +/// * [`UCommand::current_dir`]: Sets the working directory +/// * [`UCommand::keep_env`]: Keep environment variables instead of clearing them +/// * ... +#[derive(Debug, Default)] pub struct UCommand { - args: Vec, + args: VecDeque, env_vars: Vec<(OsString, OsString)>, current_dir: Option, env_clear: bool, bin_path: Option, - util_name: Option, + util_name: Option, has_run: bool, ignore_stdin_write_error: bool, stdin: Option, stdout: Option, stderr: Option, bytes_into_stdin: Option>, - // TODO: Why android? #[cfg(any(target_os = "linux", target_os = "android"))] limits: Vec<(rlimit::Resource, u64, u64)>, stderr_to_stdout: bool, @@ -1187,48 +1182,79 @@ pub struct UCommand { } impl UCommand { + /// Create a new plain [`UCommand`]. + /// + /// Executes a command that must be given as argument (for example with [`UCommand::arg`] in a + /// shell (`sh -c` on unix platforms or `cmd /C` on windows). + /// + /// Per default the environment is cleared and the working directory is set to an individual + /// temporary directory for safety purposes. pub fn new() -> Self { Self { - tmpd: None, - has_run: false, - bin_path: None, - current_dir: None, - args: vec![], env_clear: true, - env_vars: vec![], - util_name: None, - ignore_stdin_write_error: false, - bytes_into_stdin: None, - stdin: None, - stdout: None, - stderr: None, - // TODO: Why android? - #[cfg(any(target_os = "linux", target_os = "android"))] - limits: vec![], - stderr_to_stdout: false, - timeout: Some(Duration::from_secs(30)), + ..Default::default() } } - pub fn new_from_tmp, S: AsRef>( - bin_path: T, - util_name: Option, - tmpd: Rc, - env_clear: bool, - ) -> Self { - let mut ucmd: Self = Self::new(); - ucmd.bin_path = Some(PathBuf::from(bin_path.as_ref())); - ucmd.util_name = util_name.map(|s| s.as_ref().to_os_string()); - ucmd.tmpd = Some(tmpd); - ucmd.env_clear = env_clear; + /// Create a [`UCommand`] for a specific uutils utility. + /// + /// Sets the temporary directory to `tmpd` and the execution binary to the path where + /// `coreutils` is found. + pub fn with_util(util_name: T, tmpd: Rc) -> Self + where + T: AsRef, + { + let mut ucmd = Self::new(); + ucmd.util_name = Some(util_name.as_ref().into()); + ucmd.bin_path(TESTS_BINARY).temp_dir(tmpd); ucmd } + /// Create a [`UCommand`] from a [`TestScenario`]. + /// + /// The temporary directory and uutils utility are inherited from the [`TestScenario`] and the + /// execution binary is set to `coreutils`. + pub fn from_test_scenario(scene: &TestScenario) -> Self { + Self::with_util(&scene.util_name, scene.tmpd.clone()) + } + + /// Set the execution binary. + /// + /// Make sure the binary found at this path is executable. It's safest to provide the + /// canonicalized path instead of just the name of the executable, since path resolution is not + /// guaranteed to work on all platforms. + fn bin_path(&mut self, bin_path: T) -> &mut Self + where + T: Into, + { + self.bin_path = Some(bin_path.into()); + self + } + + /// Set the temporary directory. + /// + /// Per default an individual temporary directory is created for every [`UCommand`]. If not + /// specified otherwise with [`UCommand::current_dir`] the working directory is set to this + /// temporary directory. + fn temp_dir(&mut self, temp_dir: Rc) -> &mut Self { + self.tmpd = Some(temp_dir); + self + } + + /// Keep the environment variables instead of clearing them before running the command. + pub fn keep_env(&mut self) -> &mut Self { + self.env_clear = false; + self + } + + /// Set the working directory for this [`UCommand`] + /// + /// Per default the working directory is set to the [`UCommands`] temporary directory. pub fn current_dir(&mut self, current_dir: T) -> &mut Self where - T: AsRef, + T: Into, { - self.current_dir = Some(current_dir.as_ref().into()); + self.current_dir = Some(current_dir.into()); self } @@ -1255,7 +1281,7 @@ impl UCommand { /// Add a parameter to the invocation. Path arguments are treated relative /// to the test environment directory. pub fn arg>(&mut self, arg: S) -> &mut Self { - self.args.push(arg.as_ref().into()); + self.args.push_back(arg.as_ref().into()); self } @@ -1302,9 +1328,8 @@ impl UCommand { self } - // TODO: Why android? #[cfg(any(target_os = "linux", target_os = "android"))] - pub fn with_limit( + pub fn limit( &mut self, resource: rlimit::Resource, soft_limit: u64, @@ -1326,25 +1351,46 @@ impl UCommand { self } - // TODO: make public? + /// Build the `std::process::Command` and apply the defaults on fields which were not specified + /// by the user. + /// + /// These __defaults__ are: + /// * `bin_path`: Depending on the platform and os, the native shell (unix -> `/bin/sh` etc.). + /// This default also requires to set the first argument to `-c` on unix (`/C` on windows) if + /// this argument wasn't specified explicitly by the user. + /// * `util_name`: `None`. If neither `bin_path` nor `util_name` were given the arguments are + /// run in a shell (See `bin_path` above). + /// * `temp_dir`: If `current_dir` was not set, a new temporary directory will be created in + /// which this command will be run and `current_dir` will be set to this `temp_dir`. + /// * `current_dir`: The temporary directory given by `temp_dir`. + /// * `timeout`: `30 seconds` + /// * `env_clear`: `true`. (Almost) all environment variables will be cleared. + /// * `stdin`: `Stdio::null()` + /// * `ignore_stdin_write_error`: `false` + /// * `stdout`, `stderr`: If not specified the output will be captured with [`CapturedOutput`] + /// * `stderr_to_stdout`: `false` + /// * `bytes_into_stdin`: `None` + /// * `limits`: `None`. fn build(&mut self) -> (Command, Option, Option) { if self.bin_path.is_some() { if let Some(util_name) = &self.util_name { - self.args.insert(0, OsString::from(util_name)); + self.args.push_front(util_name.into()); } } else if let Some(util_name) = &self.util_name { self.bin_path = Some(PathBuf::from(TESTS_BINARY)); - self.args.insert(0, OsString::from(util_name)); + self.args.push_front(util_name.into()); + // neither `bin_path` nor `util_name` was set so we apply the default to run the arguments + // in a platform specific shell } else if cfg!(unix) { - let bin_path = if cfg!(target_os = "android") { - PathBuf::from("/system/bin/sh") - } else { - PathBuf::from("/bin/sh") - }; + #[cfg(target_os = "android")] + let bin_path = PathBuf::from("/system/bin/sh"); + #[cfg(not(target_os = "android"))] + let bin_path = PathBuf::from("/bin/sh"); + self.bin_path = Some(bin_path); let c_arg = OsString::from("-c"); if !self.args.contains(&c_arg) { - self.args.insert(0, c_arg); + self.args.push_front(c_arg); } } else { self.bin_path = Some(PathBuf::from("cmd")); @@ -1355,21 +1401,26 @@ impl UCommand { .iter() .any(|s| s.eq_ignore_ascii_case(&c_arg) || s.eq_ignore_ascii_case(&k_arg)) { - self.args.insert(0, c_arg); + self.args.push_front(c_arg); } }; + // unwrap is safe here because we have set `self.bin_path` before let mut command = Command::new(self.bin_path.as_ref().unwrap()); command.args(&self.args); - if self.tmpd.is_none() { - self.tmpd = Some(Rc::new(tempfile::tempdir().unwrap())); - } - + // We use a temporary directory as working directory if not specified otherwise with + // `current_dir()`. If neither `current_dir` nor a temporary directory is available, then we + // create our own. if let Some(current_dir) = &self.current_dir { command.current_dir(current_dir); + } else if let Some(temp_dir) = &self.tmpd { + command.current_dir(temp_dir.path()); } else { - command.current_dir(self.tmpd.as_ref().unwrap().path()); + let temp_dir = tempfile::tempdir().unwrap(); + self.current_dir = Some(temp_dir.path().into()); + command.current_dir(temp_dir.path()); + self.tmpd = Some(Rc::new(temp_dir)); } if self.env_clear { @@ -1391,8 +1442,10 @@ impl UCommand { } } - for (key, value) in &self.env_vars { - command.env(key, value); + command.envs(self.env_vars.iter().cloned()); + + if self.timeout.is_none() { + self.timeout = Some(Duration::from_secs(30)); } let mut captured_stdout = None; @@ -1436,7 +1489,6 @@ impl UCommand { /// Spawns the command, feeds the stdin if any, and returns the /// child process immediately. pub fn run_no_wait(&mut self) -> UChild { - // TODO: remove? assert!(!self.has_run, "{}", ALREADY_RUN); self.has_run = true; @@ -1509,7 +1561,7 @@ impl std::fmt::Display for UCommand { let mut comm_string: Vec = vec![self .bin_path .as_ref() - .map_or("".to_string(), |p| p.display().to_string())]; + .map_or(String::new(), |p| p.display().to_string())]; comm_string.extend(self.args.iter().map(|s| s.to_string_lossy().to_string())); f.write_str(&comm_string.join(" ")) } @@ -1647,14 +1699,14 @@ impl<'a> UChildAssertion<'a> { self.uchild.stderr_exact_bytes(expected_stderr_size), ), }; - CmdResult { - bin_path: self.uchild.bin_path.clone(), - util_name: self.uchild.util_name.clone(), - tmpd: self.uchild.tmpd.clone(), + CmdResult::new( + self.uchild.bin_path.clone(), + self.uchild.util_name.clone(), + self.uchild.tmpd.clone(), exit_status, stdout, stderr, - } + ) } // Make assertions of [`CmdResult`] with all output from start of the process until now. @@ -1734,7 +1786,7 @@ impl<'a> UChildAssertion<'a> { /// Abstraction for a [`std::process::Child`] to handle the child process. pub struct UChild { raw: Child, - bin_path: String, + bin_path: PathBuf, util_name: Option, captured_stdout: Option, captured_stderr: Option, @@ -1754,11 +1806,8 @@ impl UChild { ) -> Self { Self { raw: child, - bin_path: ucommand.bin_path.as_ref().unwrap().display().to_string(), - util_name: ucommand - .util_name - .clone() - .map(|s| s.to_string_lossy().to_string()), + bin_path: ucommand.bin_path.clone().unwrap(), + util_name: ucommand.util_name.clone(), captured_stdout, captured_stderr, ignore_stdin_write_error: ucommand.ignore_stdin_write_error, @@ -2388,11 +2437,13 @@ fn parse_coreutil_version(version_string: &str) -> f32 { ///``` #[cfg(unix)] pub fn expected_result(ts: &TestScenario, args: &[&str]) -> std::result::Result { - println!("{}", check_coreutil_version(&ts.util_name, VERSION_MIN)?); - let util_name = &host_name_for(&ts.util_name); + let util_name = ts.util_name.as_str(); + println!("{}", check_coreutil_version(util_name, VERSION_MIN)?); + let util_name = host_name_for(util_name); let result = ts - .cmd_keepenv(util_name.as_ref()) + .cmd(util_name.as_ref()) + .keep_env() .env("LC_ALL", "C") .args(args) .run(); @@ -2464,7 +2515,8 @@ pub fn run_ucmd_as_root( // we can run sudo and we're root // run ucmd as root: Ok(ts - .cmd_keepenv("sudo") + .cmd("sudo") + .keep_env() .env("LC_ALL", "C") .arg("-E") .arg("--non-interactive") @@ -2492,30 +2544,8 @@ mod tests { // spell-checker:ignore (tests) asdfsadfa use super::*; - #[cfg(unix)] pub fn run_cmd>(cmd: T) -> CmdResult { - let mut ucmd = UCommand::new_from_tmp::<&str, String>( - "sh", - None, - Rc::new(tempfile::tempdir().unwrap()), - true, - ); - ucmd.arg("-c"); - ucmd.arg(cmd); - ucmd.run() - } - - #[cfg(windows)] - pub fn run_cmd>(cmd: T) -> CmdResult { - let mut ucmd = UCommand::new_from_tmp::<&str, String>( - "cmd", - None, - Rc::new(tempfile::tempdir().unwrap()), - true, - ); - ucmd.arg("/C"); - ucmd.arg(cmd); - ucmd.run() + UCommand::new().arg(cmd).run() } #[test] @@ -3257,7 +3287,7 @@ mod tests { #[cfg(feature = "echo")] #[test] fn test_ucommand_when_default() { - let shell_cmd = format!("{} echo -n hello", TESTS_BINARY); + let shell_cmd = format!("{TESTS_BINARY} echo -n hello"); let mut command = UCommand::new(); command.arg(&shell_cmd).succeeds().stdout_is("hello"); @@ -3274,4 +3304,31 @@ mod tests { std::assert_eq!(command.args, &[expected_arg, OsString::from(&shell_cmd)]); assert!(command.tmpd.is_some()); } + + #[cfg(feature = "echo")] + #[test] + fn test_ucommand_with_util() { + let tmpd = tempfile::tempdir().unwrap(); + let mut command = UCommand::with_util("echo", Rc::new(tmpd)); + + command + .args(&["-n", "hello"]) + .succeeds() + .stdout_only("hello"); + + std::assert_eq!( + &PathBuf::from(TESTS_BINARY), + command.bin_path.as_ref().unwrap() + ); + std::assert_eq!("echo", &command.util_name.unwrap()); + std::assert_eq!( + &[ + OsString::from("echo"), + OsString::from("-n"), + OsString::from("hello") + ], + command.args.make_contiguous() + ); + assert!(command.tmpd.is_some()); + } } From 2cd19522e40ab2c869268348efb0792e7023fc85 Mon Sep 17 00:00:00 2001 From: Yang Hau Date: Sun, 19 Feb 2023 03:22:54 +0800 Subject: [PATCH 08/94] fix: Fix panic in multi-byte characters Closes #4353 --- src/uucore/src/lib/parser/parse_time.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/uucore/src/lib/parser/parse_time.rs b/src/uucore/src/lib/parser/parse_time.rs index e440a3c91..1a7b66e90 100644 --- a/src/uucore/src/lib/parser/parse_time.rs +++ b/src/uucore/src/lib/parser/parse_time.rs @@ -51,7 +51,10 @@ pub fn from_str(string: &str) -> Result { if len == 0 { return Err("empty string".to_owned()); } - let slice = &string[..len - 1]; + let slice = match string.get(..len - 1) { + Some(s) => s, + None => return Err(format!("invalid time interval {}", string.quote())), + }; let (numstr, times) = match string.chars().next_back().unwrap() { 's' => (slice, 1), 'm' => (slice, 60), @@ -112,6 +115,11 @@ mod tests { assert!(from_str("123X").is_err()); } + #[test] + fn test_error_multi_bytes_characters() { + assert!(from_str("10€").is_err()); + } + #[test] fn test_error_invalid_magnitude() { assert!(from_str("12abc3s").is_err()); From 5876cd25814ed0a3d7f034854eeafdff50ad390b Mon Sep 17 00:00:00 2001 From: Kyle Manna Date: Tue, 14 Feb 2023 01:02:31 -0600 Subject: [PATCH 09/94] install: add missing directory chown * Re-factor the copy function chown into a function that can be re-used. * Fix bug where the group overwrote the user. * Add chown compatibility to follow GNU coreutils. * Reduce two chown calls to a single syscall only when needed. * Fixes #4361 --- src/uu/install/src/install.rs | 119 +++++++++++++++++----------------- 1 file changed, 59 insertions(+), 60 deletions(-) diff --git a/src/uu/install/src/install.rs b/src/uu/install/src/install.rs index 597eacc67..c3b64cdd8 100644 --- a/src/uu/install/src/install.rs +++ b/src/uu/install/src/install.rs @@ -466,6 +466,8 @@ fn directory(paths: &[String], b: &Behavior) -> UResult<()> { uucore::error::set_exit_code(1); continue; } + + chown_optional_user_group(path, b)?; } // If the exit code was set, or show! has been called at least once // (which sets the exit code as well), function execution will end after @@ -626,6 +628,62 @@ fn copy_files_into_dir(files: &[PathBuf], target_dir: &Path, b: &Behavior) -> UR Ok(()) } +/// Handle incomplete user/group parings for chown. +/// +/// Returns a Result type with the Err variant containing the error message. +/// +/// # Parameters +/// +/// _path_ must exist. +/// +/// # Errors +/// +/// If the owner or group are invalid or copy system call fails, we print a verbose error and +/// return an empty error value. +/// +fn chown_optional_user_group(path: &Path, b: &Behavior) -> UResult<()> { + let owner = if !b.owner.is_empty() { + match usr2uid(&b.owner) { + Ok(u) => Some(u), + _ => return Err(InstallError::NoSuchUser(b.owner.clone()).into()), + } + } else { + None + }; + + let group = if !b.group.is_empty() { + match grp2gid(&b.group) { + Ok(g) => Some(g), + _ => return Err(InstallError::NoSuchGroup(b.group.clone()).into()), + } + } else { + None + }; + + let meta = match fs::metadata(path) { + Ok(meta) => meta, + Err(e) => return Err(InstallError::MetadataFailed(e).into()), + }; + + let verbosity = Verbosity { + groups_only: owner.is_none(), + level: VerbosityLevel::Normal, + }; + + if owner.is_some() || group.is_some() { + match wrap_chown(path, &meta, owner, group, false, verbosity) { + Ok(n) => { + if !n.is_empty() { + show_error!("{}", n); + } + } + Err(e) => show_error!("{}", e), + } + } + + Ok(()) +} + /// Copy one file to a new location, changing metadata. /// /// Returns a Result type with the Err variant containing the error message. @@ -708,66 +766,7 @@ fn copy(from: &Path, to: &Path, b: &Behavior) -> UResult<()> { return Err(InstallError::ChmodFailed(to.to_path_buf()).into()); } - if !b.owner.is_empty() { - let meta = match fs::metadata(to) { - Ok(meta) => meta, - Err(e) => return Err(InstallError::MetadataFailed(e).into()), - }; - - let owner_id = match usr2uid(&b.owner) { - Ok(g) => g, - _ => return Err(InstallError::NoSuchUser(b.owner.clone()).into()), - }; - let gid = meta.gid(); - match wrap_chown( - to, - &meta, - Some(owner_id), - Some(gid), - false, - Verbosity { - groups_only: false, - level: VerbosityLevel::Normal, - }, - ) { - Ok(n) => { - if !n.is_empty() { - show_error!("{}", n); - } - } - Err(e) => show_error!("{}", e), - } - } - - if !b.group.is_empty() { - let meta = match fs::metadata(to) { - Ok(meta) => meta, - Err(e) => return Err(InstallError::MetadataFailed(e).into()), - }; - - let group_id = match grp2gid(&b.group) { - Ok(g) => g, - _ => return Err(InstallError::NoSuchGroup(b.group.clone()).into()), - }; - match wrap_chown( - to, - &meta, - Some(group_id), - None, - false, - Verbosity { - groups_only: true, - level: VerbosityLevel::Normal, - }, - ) { - Ok(n) => { - if !n.is_empty() { - show_error!("{}", n); - } - } - Err(e) => show_error!("{}", e), - } - } + chown_optional_user_group(to, b)?; if b.preserve_timestamps { let meta = match fs::metadata(from) { From 376f4d90ef7d8ee8b5d0283b6997796e10052719 Mon Sep 17 00:00:00 2001 From: Kyle Manna Date: Wed, 15 Feb 2023 21:28:40 -0600 Subject: [PATCH 10/94] install: add tests for invalid owner and group * Move the user and group resolution to the behavior decoding instead of re-running on every file/directory creation. Simplifies code. * Update error output to match GNU coreutils. * Add tests to verify invalid owner and group. --- src/uu/install/src/install.rs | 97 ++++++++++++++++++----------------- tests/by-util/test_install.rs | 94 +++++++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+), 48 deletions(-) diff --git a/src/uu/install/src/install.rs b/src/uu/install/src/install.rs index c3b64cdd8..ec80709a9 100644 --- a/src/uu/install/src/install.rs +++ b/src/uu/install/src/install.rs @@ -41,8 +41,8 @@ pub struct Behavior { specified_mode: Option, backup_mode: BackupMode, suffix: String, - owner: String, - group: String, + owner_id: Option, + group_id: Option, verbose: bool, preserve_timestamps: bool, compare: bool, @@ -64,8 +64,8 @@ enum InstallError { InstallFailed(PathBuf, PathBuf, std::io::Error), StripProgramFailed(String), MetadataFailed(std::io::Error), - NoSuchUser(String), - NoSuchGroup(String), + InvalidUser(String), + InvalidGroup(String), OmittingDirectory(PathBuf), } @@ -117,8 +117,8 @@ impl Display for InstallError { ), Self::StripProgramFailed(msg) => write!(f, "strip program failed: {msg}"), Self::MetadataFailed(e) => Display::fmt(&uio_error!(e, ""), f), - Self::NoSuchUser(user) => write!(f, "no such user: {}", user.maybe_quote()), - Self::NoSuchGroup(group) => write!(f, "no such group: {}", group.maybe_quote()), + Self::InvalidUser(user) => write!(f, "invalid user: {}", user.quote()), + Self::InvalidGroup(group) => write!(f, "invalid group: {}", group.quote()), Self::OmittingDirectory(dir) => write!(f, "omitting directory {}", dir.quote()), } } @@ -391,21 +391,44 @@ fn behavior(matches: &ArgMatches) -> UResult { show_error!("Options --compare and --strip are mutually exclusive"); return Err(1.into()); } + + let owner = matches + .get_one::(OPT_OWNER) + .map(|s| s.as_str()) + .unwrap_or("") + .to_string(); + + let owner_id = if !owner.is_empty() { + match usr2uid(&owner) { + Ok(u) => Some(u), + _ => return Err(InstallError::InvalidUser(owner.clone()).into()), + } + } else { + None + }; + + let group = matches + .get_one::(OPT_GROUP) + .map(|s| s.as_str()) + .unwrap_or("") + .to_string(); + + let group_id = if !group.is_empty() { + match grp2gid(&group) { + Ok(g) => Some(g), + _ => return Err(InstallError::InvalidGroup(group.clone()).into()), + } + } else { + None + }; + Ok(Behavior { main_function, specified_mode, backup_mode, suffix: backup_control::determine_backup_suffix(matches), - owner: matches - .get_one::(OPT_OWNER) - .map(|s| s.as_str()) - .unwrap_or("") - .to_string(), - group: matches - .get_one::(OPT_GROUP) - .map(|s| s.as_str()) - .unwrap_or("") - .to_string(), + owner_id, + group_id, verbose: matches.get_flag(OPT_VERBOSE), preserve_timestamps, compare, @@ -467,7 +490,11 @@ fn directory(paths: &[String], b: &Behavior) -> UResult<()> { continue; } - chown_optional_user_group(path, b)?; + if chown_optional_user_group(path, b).is_err() { + // Error messages are printed by the chown_optional_user_group function! + uucore::error::set_exit_code(1); + continue; + } } // If the exit code was set, or show! has been called at least once // (which sets the exit code as well), function execution will end after @@ -642,36 +669,18 @@ fn copy_files_into_dir(files: &[PathBuf], target_dir: &Path, b: &Behavior) -> UR /// return an empty error value. /// fn chown_optional_user_group(path: &Path, b: &Behavior) -> UResult<()> { - let owner = if !b.owner.is_empty() { - match usr2uid(&b.owner) { - Ok(u) => Some(u), - _ => return Err(InstallError::NoSuchUser(b.owner.clone()).into()), - } - } else { - None - }; - - let group = if !b.group.is_empty() { - match grp2gid(&b.group) { - Ok(g) => Some(g), - _ => return Err(InstallError::NoSuchGroup(b.group.clone()).into()), - } - } else { - None - }; - let meta = match fs::metadata(path) { Ok(meta) => meta, Err(e) => return Err(InstallError::MetadataFailed(e).into()), }; let verbosity = Verbosity { - groups_only: owner.is_none(), + groups_only: b.owner_id.is_none(), level: VerbosityLevel::Normal, }; - if owner.is_some() || group.is_some() { - match wrap_chown(path, &meta, owner, group, false, verbosity) { + if b.owner_id.is_some() || b.group_id.is_some() { + match wrap_chown(path, &meta, b.owner_id, b.group_id, false, verbosity) { Ok(n) => { if !n.is_empty() { show_error!("{}", n); @@ -846,19 +855,11 @@ fn need_copy(from: &Path, to: &Path, b: &Behavior) -> UResult { // TODO: if -P (#1809) and from/to contexts mismatch, return true. - if !b.owner.is_empty() { - let owner_id = match usr2uid(&b.owner) { - Ok(id) => id, - _ => return Err(InstallError::NoSuchUser(b.owner.clone()).into()), - }; + if let Some(owner_id) = b.owner_id { if owner_id != to_meta.uid() { return Ok(true); } - } else if !b.group.is_empty() { - let group_id = match grp2gid(&b.group) { - Ok(id) => id, - _ => return Err(InstallError::NoSuchGroup(b.group.clone()).into()), - }; + } else if let Some(group_id) = b.group_id { if group_id != to_meta.gid() { return Ok(true); } diff --git a/tests/by-util/test_install.rs b/tests/by-util/test_install.rs index 1e6cd7606..342824fdb 100644 --- a/tests/by-util/test_install.rs +++ b/tests/by-util/test_install.rs @@ -1369,6 +1369,100 @@ fn test_install_dir_req_verbose() { .stdout_contains("install: creating directory 'sub5/a'\ninstall: creating directory 'sub5/a/b'\ninstall: creating directory 'sub5/a/b/c'\n'source_file1' -> 'sub5/a/b/c/file'"); } +#[test] +fn test_install_chown_file_invalid() { + let scene = TestScenario::new(util_name!()); + let at = &scene.fixtures; + + let file_1 = "source_file1"; + at.touch(file_1); + + scene + .ucmd() + .arg("-o") + .arg("test_invalid_user") + .arg(file_1) + .arg("target_file1") + .fails() + .stderr_contains("install: invalid user: 'test_invalid_user'"); + + scene + .ucmd() + .arg("-g") + .arg("test_invalid_group") + .arg(file_1) + .arg("target_file1") + .fails() + .stderr_contains("install: invalid group: 'test_invalid_group'"); + + scene + .ucmd() + .arg("-o") + .arg("test_invalid_user") + .arg("-g") + .arg("test_invalid_group") + .arg(file_1) + .arg("target_file1") + .fails() + .stderr_contains("install: invalid user: 'test_invalid_user'"); + + scene + .ucmd() + .arg("-g") + .arg("test_invalid_group") + .arg("-o") + .arg("test_invalid_user") + .arg(file_1) + .arg("target_file1") + .fails() + .stderr_contains("install: invalid user: 'test_invalid_user'"); +} + +#[test] +fn test_install_chown_directory_invalid() { + let scene = TestScenario::new(util_name!()); + + scene + .ucmd() + .arg("-o") + .arg("test_invalid_user") + .arg("-d") + .arg("dir1/dir2") + .fails() + .stderr_contains("install: invalid user: 'test_invalid_user'"); + + scene + .ucmd() + .arg("-g") + .arg("test_invalid_group") + .arg("-d") + .arg("dir1/dir2") + .fails() + .stderr_contains("install: invalid group: 'test_invalid_group'"); + + scene + .ucmd() + .arg("-o") + .arg("test_invalid_user") + .arg("-g") + .arg("test_invalid_group") + .arg("-d") + .arg("dir1/dir2") + .fails() + .stderr_contains("install: invalid user: 'test_invalid_user'"); + + scene + .ucmd() + .arg("-g") + .arg("test_invalid_group") + .arg("-o") + .arg("test_invalid_user") + .arg("-d") + .arg("dir1/dir2") + .fails() + .stderr_contains("install: invalid user: 'test_invalid_user'"); +} + #[test] fn test_install_compare_option() { let scene = TestScenario::new(util_name!()); From 109553436dcd89e59b38eeb6c6166c3c6506b450 Mon Sep 17 00:00:00 2001 From: Yang Hau Date: Sun, 19 Feb 2023 18:09:34 +0800 Subject: [PATCH 11/94] add function profile --- src/uu/cksum/src/cksum.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/uu/cksum/src/cksum.rs b/src/uu/cksum/src/cksum.rs index 146fe0d56..d86c44819 100644 --- a/src/uu/cksum/src/cksum.rs +++ b/src/uu/cksum/src/cksum.rs @@ -52,6 +52,12 @@ struct Options { output_bits: usize, } +/// Calculate checksum +/// +/// # Arguments +/// +/// * `options` - CLI options for the assigning checksum algorithm +/// * `files` - A iterator of OsStr which is a bunch of files that are using for calculating checksum #[allow(clippy::cognitive_complexity)] fn cksum<'a, I>(mut options: Options, files: I) -> UResult<()> where From 05db5f744239d81bdfff00ef8f7c81981dc92a7f Mon Sep 17 00:00:00 2001 From: Kyle Manna Date: Sat, 18 Feb 2023 23:45:39 -0600 Subject: [PATCH 12/94] install: address merge request feedback * Explicitly handle Err() in match. * Move metdata functions deeper into `chown_optional_user_group()`. * Add `ChownFailed` to propagate errors. * Use `show_if_err!()` to wrap `chown_optional_user_group`. * Simplify chown verbose message handling. --- src/uu/install/src/install.rs | 58 ++++++++++++++++------------------- 1 file changed, 27 insertions(+), 31 deletions(-) diff --git a/src/uu/install/src/install.rs b/src/uu/install/src/install.rs index ec80709a9..5e6efd150 100644 --- a/src/uu/install/src/install.rs +++ b/src/uu/install/src/install.rs @@ -58,6 +58,7 @@ enum InstallError { DirNeedsArg(), CreateDirFailed(PathBuf, std::io::Error), ChmodFailed(PathBuf), + ChownFailed(PathBuf, String), InvalidTarget(PathBuf), TargetDirIsntDir(PathBuf), BackupFailed(PathBuf, PathBuf, std::io::Error), @@ -99,6 +100,7 @@ impl Display for InstallError { Display::fmt(&uio_error!(e, "failed to create {}", dir.quote()), f) } Self::ChmodFailed(file) => write!(f, "failed to chmod {}", file.quote()), + Self::ChownFailed(file, msg) => write!(f, "failed to chown {}: {}", file.quote(), msg), Self::InvalidTarget(target) => write!( f, "invalid target {}: No such file or directory", @@ -393,30 +395,30 @@ fn behavior(matches: &ArgMatches) -> UResult { } let owner = matches - .get_one::(OPT_OWNER) - .map(|s| s.as_str()) - .unwrap_or("") - .to_string(); + .get_one::(OPT_OWNER) + .map(|s| s.as_str()) + .unwrap_or("") + .to_string(); let owner_id = if !owner.is_empty() { match usr2uid(&owner) { Ok(u) => Some(u), - _ => return Err(InstallError::InvalidUser(owner.clone()).into()), + Err(_) => return Err(InstallError::InvalidUser(owner.clone()).into()), } } else { None }; let group = matches - .get_one::(OPT_GROUP) - .map(|s| s.as_str()) - .unwrap_or("") - .to_string(); + .get_one::(OPT_GROUP) + .map(|s| s.as_str()) + .unwrap_or("") + .to_string(); let group_id = if !group.is_empty() { match grp2gid(&group) { Ok(g) => Some(g), - _ => return Err(InstallError::InvalidGroup(group.clone()).into()), + Err(_) => return Err(InstallError::InvalidGroup(group.clone()).into()), } } else { None @@ -490,11 +492,7 @@ fn directory(paths: &[String], b: &Behavior) -> UResult<()> { continue; } - if chown_optional_user_group(path, b).is_err() { - // Error messages are printed by the chown_optional_user_group function! - uucore::error::set_exit_code(1); - continue; - } + show_if_err!(chown_optional_user_group(path, b)); } // If the exit code was set, or show! has been called at least once // (which sets the exit code as well), function execution will end after @@ -669,24 +667,22 @@ fn copy_files_into_dir(files: &[PathBuf], target_dir: &Path, b: &Behavior) -> UR /// return an empty error value. /// fn chown_optional_user_group(path: &Path, b: &Behavior) -> UResult<()> { - let meta = match fs::metadata(path) { - Ok(meta) => meta, - Err(e) => return Err(InstallError::MetadataFailed(e).into()), - }; - - let verbosity = Verbosity { - groups_only: b.owner_id.is_none(), - level: VerbosityLevel::Normal, - }; - if b.owner_id.is_some() || b.group_id.is_some() { + let meta = match fs::metadata(path) { + Ok(meta) => meta, + Err(e) => return Err(InstallError::MetadataFailed(e).into()), + }; + + // GNU coreutils doesn't print chown operations during install with verbose flag. + let verbosity = Verbosity { + groups_only: b.owner_id.is_none(), + level: VerbosityLevel::Normal, + }; + match wrap_chown(path, &meta, b.owner_id, b.group_id, false, verbosity) { - Ok(n) => { - if !n.is_empty() { - show_error!("{}", n); - } - } - Err(e) => show_error!("{}", e), + Ok(msg) if b.verbose && !msg.is_empty() => println!("chown: {msg}"), + Ok(_) => {} + Err(e) => return Err(InstallError::ChownFailed(path.to_path_buf(), e).into()), } } From ff6f61276e0a077a6cc608dc616f71e9e0294ecf Mon Sep 17 00:00:00 2001 From: curtain Date: Mon, 20 Feb 2023 14:25:00 +0800 Subject: [PATCH 13/94] basename: move help strings to markdown file --- src/uu/basename/basename.md | 12 ++++++++++++ src/uu/basename/src/basename.rs | 8 +++----- 2 files changed, 15 insertions(+), 5 deletions(-) create mode 100644 src/uu/basename/basename.md diff --git a/src/uu/basename/basename.md b/src/uu/basename/basename.md new file mode 100644 index 000000000..6af001323 --- /dev/null +++ b/src/uu/basename/basename.md @@ -0,0 +1,12 @@ +# basename + +## Usage +``` +basename NAME [SUFFIX] +basename OPTION... NAME... +``` + +## About + +Print NAME with any leading directory components removed +If specified, also remove a trailing SUFFIX diff --git a/src/uu/basename/src/basename.rs b/src/uu/basename/src/basename.rs index 613c4d67c..bbc8abf75 100644 --- a/src/uu/basename/src/basename.rs +++ b/src/uu/basename/src/basename.rs @@ -11,13 +11,11 @@ use clap::{crate_version, Arg, ArgAction, Command}; use std::path::{is_separator, PathBuf}; use uucore::display::Quotable; use uucore::error::{UResult, UUsageError}; -use uucore::format_usage; +use uucore::{format_usage, help_usage, help_section}; -static ABOUT: &str = r#"Print NAME with any leading directory components removed -If specified, also remove a trailing SUFFIX"#; +static ABOUT: &str = help_section!("about","basename.md"); -const USAGE: &str = "{} NAME [SUFFIX] - {} OPTION... NAME..."; +const USAGE: &str = help_usage!("basename.md"); pub mod options { pub static MULTIPLE: &str = "multiple"; From f70d23b0182dafb87c9b47e1ac82b42aeba8cacc Mon Sep 17 00:00:00 2001 From: curtain Date: Mon, 20 Feb 2023 18:13:36 +0800 Subject: [PATCH 14/94] basename: update format according to #4385 --- src/uu/basename/basename.md | 3 --- src/uu/basename/src/basename.rs | 4 ++-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/uu/basename/basename.md b/src/uu/basename/basename.md index 6af001323..b17cac74a 100644 --- a/src/uu/basename/basename.md +++ b/src/uu/basename/basename.md @@ -1,12 +1,9 @@ # basename -## Usage ``` basename NAME [SUFFIX] basename OPTION... NAME... ``` -## About - Print NAME with any leading directory components removed If specified, also remove a trailing SUFFIX diff --git a/src/uu/basename/src/basename.rs b/src/uu/basename/src/basename.rs index bbc8abf75..48f00f4b8 100644 --- a/src/uu/basename/src/basename.rs +++ b/src/uu/basename/src/basename.rs @@ -11,9 +11,9 @@ use clap::{crate_version, Arg, ArgAction, Command}; use std::path::{is_separator, PathBuf}; use uucore::display::Quotable; use uucore::error::{UResult, UUsageError}; -use uucore::{format_usage, help_usage, help_section}; +use uucore::{format_usage, help_usage, help_about}; -static ABOUT: &str = help_section!("about","basename.md"); +static ABOUT: &str = help_about!("basename.md"); const USAGE: &str = help_usage!("basename.md"); From e0470c02940ea93e8980aae225bbed52766e2bd1 Mon Sep 17 00:00:00 2001 From: Koki Ueha Date: Mon, 20 Feb 2023 11:30:33 +0000 Subject: [PATCH 15/94] tac: move help strings to markdown file --- src/uu/tac/src/tac.rs | 6 +++--- src/uu/tac/tac.md | 7 +++++++ 2 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 src/uu/tac/tac.md diff --git a/src/uu/tac/src/tac.rs b/src/uu/tac/src/tac.rs index b0d79e6da..aef9932a2 100644 --- a/src/uu/tac/src/tac.rs +++ b/src/uu/tac/src/tac.rs @@ -19,12 +19,12 @@ use std::{ use uucore::display::Quotable; use uucore::error::UError; use uucore::error::UResult; -use uucore::{format_usage, show}; +use uucore::{format_usage, help_about, help_usage, show}; use crate::error::TacError; -static USAGE: &str = "{} [OPTION]... [FILE]..."; -static ABOUT: &str = "Write each file to standard output, last line first."; +static USAGE: &str = help_usage!("tac.md"); +static ABOUT: &str = help_about!("tac.md"); mod options { pub static BEFORE: &str = "before"; diff --git a/src/uu/tac/tac.md b/src/uu/tac/tac.md new file mode 100644 index 000000000..6787b3f49 --- /dev/null +++ b/src/uu/tac/tac.md @@ -0,0 +1,7 @@ +# tac + +``` +tac [OPTION]... [FILE]... +``` + +Write each file to standard output, last line first. From 9bf9bba7c319ee79f5fd5e5c8c2ec343488e1a20 Mon Sep 17 00:00:00 2001 From: Koki Ueha Date: Mon, 20 Feb 2023 11:56:39 +0000 Subject: [PATCH 16/94] kill: move help strings to markdown file --- src/uu/kill/kill.md | 7 +++++++ src/uu/kill/src/kill.rs | 6 +++--- 2 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 src/uu/kill/kill.md diff --git a/src/uu/kill/kill.md b/src/uu/kill/kill.md new file mode 100644 index 000000000..1e3e2b3d0 --- /dev/null +++ b/src/uu/kill/kill.md @@ -0,0 +1,7 @@ +# kill + +``` +kill [OPTIONS]... PID... +``` + +Send signal to processes or list information about signals. diff --git a/src/uu/kill/src/kill.rs b/src/uu/kill/src/kill.rs index c14023e82..d18a483fd 100644 --- a/src/uu/kill/src/kill.rs +++ b/src/uu/kill/src/kill.rs @@ -14,10 +14,10 @@ use std::io::Error; use uucore::display::Quotable; use uucore::error::{FromIo, UError, UResult, USimpleError}; use uucore::signals::{signal_by_name_or_value, ALL_SIGNALS}; -use uucore::{format_usage, show}; +use uucore::{format_usage, help_about, help_usage, show}; -static ABOUT: &str = "Send signal to processes or list information about signals."; -const USAGE: &str = "{} [OPTIONS]... PID..."; +static ABOUT: &str = help_about!("kill.md"); +const USAGE: &str = help_usage!("kill.md"); pub mod options { pub static PIDS_OR_SIGNALS: &str = "pids_or_signals"; From 72f13369aa5f8483160bec439ae280539c13cb54 Mon Sep 17 00:00:00 2001 From: pkubaj Date: Mon, 20 Feb 2023 15:20:31 +0000 Subject: [PATCH 17/94] Add FreeBSD to documentation --- docs/src/installation.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/src/installation.md b/docs/src/installation.md index e2e72699b..063935ed6 100644 --- a/docs/src/installation.md +++ b/docs/src/installation.md @@ -107,6 +107,13 @@ brew install uutils-coreutils port install coreutils-uutils ``` +## FreeBSD +[![FreeBSD port](https://repology.org/badge/version-for-repo/freebsd/uutils-coreutils.svg)](https://repology.org/project/uutils-coreutils/versions) + +```sh +pkg install uutils +``` + ## Windows ### Scoop From fe4da2b8e55f7a45085cf7456f65b055b52139d8 Mon Sep 17 00:00:00 2001 From: David Matos Date: Sun, 18 Sep 2022 20:25:14 +0200 Subject: [PATCH 18/94] cp: modify archive flag to copy dir contents rather than dir --- src/uu/cp/src/copydir.rs | 2 +- tests/by-util/test_cp.rs | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/uu/cp/src/copydir.rs b/src/uu/cp/src/copydir.rs index a89bd8537..c312b7cbb 100644 --- a/src/uu/cp/src/copydir.rs +++ b/src/uu/cp/src/copydir.rs @@ -93,7 +93,7 @@ impl<'a> Context<'a> { fn new(root: &'a Path, target: &'a Path) -> std::io::Result { let current_dir = env::current_dir()?; let root_path = current_dir.join(root); - let root_parent = if target.exists() { + let root_parent = if target.exists() && !root.to_str().unwrap().ends_with("/.") { root_path.parent().map(|p| p.to_path_buf()) } else { Some(root_path) diff --git a/tests/by-util/test_cp.rs b/tests/by-util/test_cp.rs index e6e8a6ba2..eec6b4536 100644 --- a/tests/by-util/test_cp.rs +++ b/tests/by-util/test_cp.rs @@ -2532,3 +2532,14 @@ fn test_src_base_dot() { .no_stdout(); assert!(!at.dir_exists("y/x")); } + +#[test] +#[cfg(not(windows))] +fn test_cp_archive_on_directory_ending_dot() { + let (at, mut ucmd) = at_and_ucmd!(); + at.mkdir("dir1"); + at.mkdir("dir2"); + at.touch("dir1/file"); + ucmd.args(&["-a", "dir1/.", "dir2"]).succeeds(); + assert!(at.file_exists("dir2/file")); +} From dd4299c32ebef81a450f3434fe186b8faeb18171 Mon Sep 17 00:00:00 2001 From: David Matos Date: Mon, 20 Feb 2023 22:18:07 +0100 Subject: [PATCH 19/94] chmod: supress verbose output when not verbose --- src/uu/chmod/src/chmod.rs | 10 ++++++---- tests/by-util/test_chmod.rs | 15 +++++++++++++-- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/uu/chmod/src/chmod.rs b/src/uu/chmod/src/chmod.rs index 374f22874..eb8db104f 100644 --- a/src/uu/chmod/src/chmod.rs +++ b/src/uu/chmod/src/chmod.rs @@ -190,16 +190,18 @@ impl Chmoder { let file = Path::new(filename); if !file.exists() { if file.is_symlink() { - println!( - "failed to change mode of {} from 0000 (---------) to 0000 (---------)", - filename.quote() - ); if !self.quiet { show!(USimpleError::new( 1, format!("cannot operate on dangling symlink {}", filename.quote()), )); } + if self.verbose { + println!( + "failed to change mode of {} from 0000 (---------) to 1500 (r-x-----T)", + filename.quote() + ); + } } else if !self.quiet { show!(USimpleError::new( 1, diff --git a/tests/by-util/test_chmod.rs b/tests/by-util/test_chmod.rs index feb756632..c5f731d7d 100644 --- a/tests/by-util/test_chmod.rs +++ b/tests/by-util/test_chmod.rs @@ -414,7 +414,7 @@ fn test_chmod_symlink_non_existing_file() { let non_existing = "test_chmod_symlink_non_existing_file"; let test_symlink = "test_chmod_symlink_non_existing_file_symlink"; let expected_stdout = &format!( - "failed to change mode of '{test_symlink}' from 0000 (---------) to 0000 (---------)" + "failed to change mode of '{test_symlink}' from 0000 (---------) to 1500 (r-x-----T)" ); let expected_stderr = &format!("cannot operate on dangling symlink '{test_symlink}'"); @@ -442,6 +442,17 @@ fn test_chmod_symlink_non_existing_file() { .code_is(1) .no_stderr() .stdout_contains(expected_stdout); + + // this should only include the dangling symlink message + // NOT the failure to change mode + scene + .ucmd() + .arg("755") + .arg(test_symlink) + .run() + .code_is(1) + .no_stdout() + .stderr_contains(expected_stderr); } #[test] @@ -616,7 +627,7 @@ fn test_chmod_file_symlink_after_non_existing_file() { let non_existing = "test_chmod_symlink_non_existing_file"; let test_dangling_symlink = "test_chmod_symlink_non_existing_file_symlink"; let expected_stdout = &format!( - "failed to change mode of '{test_dangling_symlink}' from 0000 (---------) to 0000 (---------)" + "failed to change mode of '{test_dangling_symlink}' from 0000 (---------) to 1500 (r-x-----T)" ); let expected_stderr = &format!("cannot operate on dangling symlink '{test_dangling_symlink}'"); From 7cfeba0d22e60a8b69ad690635cef02fcdd8f6d2 Mon Sep 17 00:00:00 2001 From: Cheng-Hao Date: Mon, 20 Feb 2023 16:29:20 -0800 Subject: [PATCH 20/94] mkdir: move help strings to markdown file --- src/uu/mkdir/mkdir.md | 11 +++++++++++ src/uu/mkdir/src/mkdir.rs | 9 ++++----- 2 files changed, 15 insertions(+), 5 deletions(-) create mode 100644 src/uu/mkdir/mkdir.md diff --git a/src/uu/mkdir/mkdir.md b/src/uu/mkdir/mkdir.md new file mode 100644 index 000000000..79deafc46 --- /dev/null +++ b/src/uu/mkdir/mkdir.md @@ -0,0 +1,11 @@ +# mkdir + +``` +mkdir [OPTION]... [USER] +``` + +Create the given DIRECTORY(ies) if they do not exist + +## Long Usage + +Each MODE is of the form '[ugoa]*([-+=]([rwxXst]*|[ugo]))+|[-+=]?[0-7]+'. diff --git a/src/uu/mkdir/src/mkdir.rs b/src/uu/mkdir/src/mkdir.rs index 9aced8ba5..ab92e7e74 100644 --- a/src/uu/mkdir/src/mkdir.rs +++ b/src/uu/mkdir/src/mkdir.rs @@ -18,14 +18,13 @@ use uucore::error::{UResult, USimpleError}; #[cfg(not(windows))] use uucore::mode; use uucore::{display::Quotable, fs::dir_strip_dot_for_creation}; -use uucore::{format_usage, show, show_if_err}; +use uucore::{format_usage, help_about, help_section, help_usage, show, show_if_err}; static DEFAULT_PERM: u32 = 0o755; -const ABOUT: &str = "Create the given DIRECTORY(ies) if they do not exist"; -const USAGE: &str = "{} [OPTION]... [USER]"; -const LONG_USAGE: &str = - "Each MODE is of the form '[ugoa]*([-+=]([rwxXst]*|[ugo]))+|[-+=]?[0-7]+'."; +const ABOUT: &str = help_about!("mkdir.md"); +const USAGE: &str = help_usage!("mkdir.md"); +const LONG_USAGE: &str = help_section!("long usage", "mkdir.md"); mod options { pub const MODE: &str = "mode"; From 9322580967195a74a91d127fffb0cfb21036c6fc Mon Sep 17 00:00:00 2001 From: Yang Hau Date: Tue, 21 Feb 2023 16:23:18 +0800 Subject: [PATCH 21/94] Add checksum algorithm abstractions --- Cargo.lock | 14 ---- src/uu/cksum/Cargo.toml | 8 +-- src/uu/cksum/src/cksum.rs | 19 ++---- src/uu/hashsum/Cargo.toml | 8 --- src/uu/hashsum/src/hashsum.rs | 29 +++------ src/uucore/src/lib/features/sum.rs | 101 +++++++++++++++++------------ 6 files changed, 77 insertions(+), 102 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f08af896a..2d68c19e4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2399,14 +2399,8 @@ dependencies = [ name = "uu_cksum" version = "0.0.17" dependencies = [ - "blake2b_simd", - "blake3", "clap", "hex", - "md-5", - "sha1", - "sha2", - "sm3", "uucore", ] @@ -2611,18 +2605,10 @@ dependencies = [ name = "uu_hashsum" version = "0.0.17" dependencies = [ - "blake2b_simd", - "blake3", "clap", - "digest", "hex", - "md-5", "memchr", "regex", - "sha1", - "sha2", - "sha3", - "sm3", "uucore", ] diff --git a/src/uu/cksum/Cargo.toml b/src/uu/cksum/Cargo.toml index 20406a3f0..9b7f70f8f 100644 --- a/src/uu/cksum/Cargo.toml +++ b/src/uu/cksum/Cargo.toml @@ -16,14 +16,8 @@ path = "src/cksum.rs" [dependencies] clap = { workspace=true } -uucore = { version=">=0.0.17", package="uucore", path="../../uucore", features=["sum"] } +uucore = { workspace=true, features=["sum"] } hex = { workspace=true } -md-5 = { workspace=true } -sha1 = { workspace=true } -sha2 = { workspace=true } -blake2b_simd = { workspace=true } -blake3 = { workspace=true } -sm3 = { workspace=true } [[bin]] name = "cksum" diff --git a/src/uu/cksum/src/cksum.rs b/src/uu/cksum/src/cksum.rs index d86c44819..cadf523c2 100644 --- a/src/uu/cksum/src/cksum.rs +++ b/src/uu/cksum/src/cksum.rs @@ -8,9 +8,6 @@ // spell-checker:ignore (ToDO) fname, algo use clap::{crate_version, Arg, Command}; use hex::encode; -use md5::Md5; -use sha1::Sha1; -use sha2::{Sha224, Sha256, Sha384, Sha512}; use std::ffi::OsStr; use std::fs::File; use std::io::{self, stdin, BufReader, Read}; @@ -19,7 +16,10 @@ use std::path::Path; use uucore::{ error::{FromIo, UResult}, format_usage, - sum::{div_ceil, Digest, DigestWriter, BSD, CRC, SYSV}, + sum::{ + div_ceil, Blake2b, Digest, DigestWriter, Md5, Sha1, Sha224, Sha256, Sha384, Sha512, Sm3, + BSD, CRC, SYSV, + }, }; const USAGE: &str = "{} [OPTIONS] [FILE]..."; @@ -36,12 +36,8 @@ fn detect_algo(program: &str) -> (&'static str, Box, usize "sha256" => ("SHA256", Box::new(Sha256::new()) as Box, 256), "sha384" => ("SHA384", Box::new(Sha384::new()) as Box, 384), "sha512" => ("SHA512", Box::new(Sha512::new()) as Box, 512), - "blake2b" => ( - "BLAKE2", - Box::new(blake2b_simd::State::new()) as Box, - 512, - ), - "sm3" => ("SM3", Box::new(sm3::Sm3::new()) as Box, 512), + "blake2b" => ("BLAKE2", Box::new(Blake2b::new()) as Box, 512), + "sm3" => ("SM3", Box::new(Sm3::new()) as Box, 512), _ => panic!("unknown algorithm"), } } @@ -81,8 +77,7 @@ where let (sum, sz) = digest_read(&mut options.digest, &mut file, options.output_bits) .map_err_context(|| "failed to read input".to_string())?; - // Refer to GNU sum.c implementation. The BSD checksum output is 5 digit integer - // https://github.com/coreutils/coreutils/blob/master/src/sum.c + // The BSD checksum output is 5 digit integer let bsd_width = 5; match (options.algo_name, not_file) { ("SYSV", true) => println!( diff --git a/src/uu/hashsum/Cargo.toml b/src/uu/hashsum/Cargo.toml index a467dee9f..f335211b8 100644 --- a/src/uu/hashsum/Cargo.toml +++ b/src/uu/hashsum/Cargo.toml @@ -20,14 +20,6 @@ uucore = { workspace=true } memchr = { workspace=true } regex = { workspace=true } hex = { workspace=true } -md-5 = { workspace=true } -sha1 = { workspace=true } -sha2 = { workspace=true } -sha3 = { workspace=true } -blake2b_simd = { workspace=true } -blake3 = { workspace=true } -sm3 = { workspace=true } -digest = { workspace=true } [[bin]] name = "hashsum" diff --git a/src/uu/hashsum/src/hashsum.rs b/src/uu/hashsum/src/hashsum.rs index a240304be..c49c530e4 100644 --- a/src/uu/hashsum/src/hashsum.rs +++ b/src/uu/hashsum/src/hashsum.rs @@ -14,11 +14,7 @@ use clap::crate_version; use clap::ArgAction; use clap::{Arg, ArgMatches, Command}; use hex::encode; -use md5::Md5; use regex::Regex; -use sha1::Sha1; -use sha2::{Sha224, Sha256, Sha384, Sha512}; -use sha3::{Sha3_224, Sha3_256, Sha3_384, Sha3_512, Shake128, Shake256}; use std::cmp::Ordering; use std::error::Error; use std::ffi::{OsStr, OsString}; @@ -27,11 +23,12 @@ use std::io::{self, stdin, BufRead, BufReader, Read}; use std::iter; use std::num::ParseIntError; use std::path::Path; -use uucore::crash; -use uucore::display::Quotable; use uucore::error::{FromIo, UError, UResult}; -use uucore::show_warning; -use uucore::sum::{Digest, DigestWriter}; +use uucore::sum::{ + Blake2b, Blake3, Digest, DigestWriter, Md5, Sha1, Sha224, Sha256, Sha384, Sha3_224, Sha3_256, + Sha3_384, Sha3_512, Sha512, Shake128, Shake256, +}; +use uucore::{crash, display::Quotable, show_warning}; const NAME: &str = "hashsum"; @@ -64,16 +61,8 @@ fn detect_algo( "sha256sum" => ("SHA256", Box::new(Sha256::new()) as Box, 256), "sha384sum" => ("SHA384", Box::new(Sha384::new()) as Box, 384), "sha512sum" => ("SHA512", Box::new(Sha512::new()) as Box, 512), - "b2sum" => ( - "BLAKE2", - Box::new(blake2b_simd::State::new()) as Box, - 512, - ), - "b3sum" => ( - "BLAKE3", - Box::new(blake3::Hasher::new()) as Box, - 256, - ), + "b2sum" => ("BLAKE2", Box::new(Blake2b::new()) as Box, 512), + "b3sum" => ("BLAKE3", Box::new(Blake3::new()) as Box, 256), "sha3sum" => match matches.get_one::("bits") { Some(224) => ( "SHA3-224", @@ -166,10 +155,10 @@ fn detect_algo( set_or_crash("SHA512", Box::new(Sha512::new()), 512); } if matches.get_flag("b2sum") { - set_or_crash("BLAKE2", Box::new(blake2b_simd::State::new()), 512); + set_or_crash("BLAKE2", Box::new(Blake2b::new()), 512); } if matches.get_flag("b3sum") { - set_or_crash("BLAKE3", Box::new(blake3::Hasher::new()), 256); + set_or_crash("BLAKE3", Box::new(Blake3::new()), 256); } if matches.get_flag("sha3") { match matches.get_one::("bits") { diff --git a/src/uucore/src/lib/features/sum.rs b/src/uucore/src/lib/features/sum.rs index c7cec6e09..339414630 100644 --- a/src/uucore/src/lib/features/sum.rs +++ b/src/uucore/src/lib/features/sum.rs @@ -1,4 +1,12 @@ -// spell-checker:ignore memmem +// This file is part of the uutils coreutils package. +// +// (c) Yuan YangHao +// +// For the full copyright and license information, please view the LICENSE file +// that was distributed with this source code. + +// spell-checker:ignore memmem algo + //! Implementations of digest functions, like md5 and sha1. //! //! The [`Digest`] trait represents the interface for providing inputs @@ -30,17 +38,18 @@ pub trait Digest { } } -impl Digest for blake2b_simd::State { +pub struct Blake2b(blake2b_simd::State); +impl Digest for Blake2b { fn new() -> Self { - Self::new() + Self(blake2b_simd::State::new()) } fn hash_update(&mut self, input: &[u8]) { - self.update(input); + self.0.update(input); } fn hash_finalize(&mut self, out: &mut [u8]) { - let hash_result = &self.finalize(); + let hash_result = &self.0.finalize(); out.copy_from_slice(hash_result.as_bytes()); } @@ -53,17 +62,18 @@ impl Digest for blake2b_simd::State { } } -impl Digest for blake3::Hasher { +pub struct Blake3(blake3::Hasher); +impl Digest for Blake3 { fn new() -> Self { - Self::new() + Self(blake3::Hasher::new()) } fn hash_update(&mut self, input: &[u8]) { - self.update(input); + self.0.update(input); } fn hash_finalize(&mut self, out: &mut [u8]) { - let hash_result = &self.finalize(); + let hash_result = &self.0.finalize(); out.copy_from_slice(hash_result.as_bytes()); } @@ -76,23 +86,22 @@ impl Digest for blake3::Hasher { } } -use sm3::{Digest as SM3_D, Sm3}; - +pub struct Sm3(sm3::Sm3); impl Digest for Sm3 { fn new() -> Self { - ::new() + Self(::new()) } fn hash_update(&mut self, input: &[u8]) { - self.update(input); + ::update(&mut self.0, input); } fn hash_finalize(&mut self, out: &mut [u8]) { - out.copy_from_slice(&self.clone().finalize()); + out.copy_from_slice(&::finalize(self.0.clone())); } fn reset(&mut self) { - *self = ::new(); + *self = Self::new(); } fn output_bits(&self) -> usize { @@ -108,7 +117,6 @@ pub struct CRC { size: usize, crc_table: [u32; CRC_TABLE_LEN], } - impl CRC { fn generate_crc_table() -> [u32; CRC_TABLE_LEN] { let mut table = [0; CRC_TABLE_LEN]; @@ -196,7 +204,6 @@ pub fn div_ceil(a: usize, b: usize) -> usize { pub struct BSD { state: u16, } - impl Digest for BSD { fn new() -> Self { Self { state: 0 } @@ -231,7 +238,6 @@ impl Digest for BSD { pub struct SYSV { state: u32, } - impl Digest for SYSV { fn new() -> Self { Self { state: 0 } @@ -266,22 +272,22 @@ impl Digest for SYSV { // Implements the Digest trait for sha2 / sha3 algorithms with fixed output macro_rules! impl_digest_common { - ($type: ty, $size: expr) => { - impl Digest for $type { + ($algo_type: ty, $size: expr) => { + impl Digest for $algo_type { fn new() -> Self { - Self::default() + Self(Default::default()) } fn hash_update(&mut self, input: &[u8]) { - digest::Digest::update(self, input); + digest::Digest::update(&mut self.0, input); } fn hash_finalize(&mut self, out: &mut [u8]) { - digest::Digest::finalize_into_reset(self, out.into()); + digest::Digest::finalize_into_reset(&mut self.0, out.into()); } fn reset(&mut self) { - *self = ::new(); + *self = Self::new(); } fn output_bits(&self) -> usize { @@ -293,18 +299,18 @@ macro_rules! impl_digest_common { // Implements the Digest trait for sha2 / sha3 algorithms with variable output macro_rules! impl_digest_shake { - ($type: ty) => { - impl Digest for $type { + ($algo_type: ty) => { + impl Digest for $algo_type { fn new() -> Self { - Self::default() + Self(Default::default()) } fn hash_update(&mut self, input: &[u8]) { - digest::Update::update(self, input); + digest::Update::update(&mut self.0, input); } fn hash_finalize(&mut self, out: &mut [u8]) { - digest::ExtendableOutputReset::finalize_xof_reset_into(self, out); + digest::ExtendableOutputReset::finalize_xof_reset_into(&mut self.0, out); } fn reset(&mut self) { @@ -318,19 +324,32 @@ macro_rules! impl_digest_shake { }; } -impl_digest_common!(md5::Md5, 128); -impl_digest_common!(sha1::Sha1, 160); -impl_digest_common!(sha2::Sha224, 224); -impl_digest_common!(sha2::Sha256, 256); -impl_digest_common!(sha2::Sha384, 384); -impl_digest_common!(sha2::Sha512, 512); +pub struct Md5(md5::Md5); +pub struct Sha1(sha1::Sha1); +pub struct Sha224(sha2::Sha224); +pub struct Sha256(sha2::Sha256); +pub struct Sha384(sha2::Sha384); +pub struct Sha512(sha2::Sha512); +impl_digest_common!(Md5, 128); +impl_digest_common!(Sha1, 160); +impl_digest_common!(Sha224, 224); +impl_digest_common!(Sha256, 256); +impl_digest_common!(Sha384, 384); +impl_digest_common!(Sha512, 512); -impl_digest_common!(sha3::Sha3_224, 224); -impl_digest_common!(sha3::Sha3_256, 256); -impl_digest_common!(sha3::Sha3_384, 384); -impl_digest_common!(sha3::Sha3_512, 512); -impl_digest_shake!(sha3::Shake128); -impl_digest_shake!(sha3::Shake256); +pub struct Sha3_224(sha3::Sha3_224); +pub struct Sha3_256(sha3::Sha3_256); +pub struct Sha3_384(sha3::Sha3_384); +pub struct Sha3_512(sha3::Sha3_512); +impl_digest_common!(Sha3_224, 224); +impl_digest_common!(Sha3_256, 256); +impl_digest_common!(Sha3_384, 384); +impl_digest_common!(Sha3_512, 512); + +pub struct Shake128(sha3::Shake128); +pub struct Shake256(sha3::Shake256); +impl_digest_shake!(Shake128); +impl_digest_shake!(Shake256); /// A struct that writes to a digest. /// From 4e7ae2d46a8550e7bd372a80449662fd646e895e Mon Sep 17 00:00:00 2001 From: papparapa <37232476+papparapa@users.noreply.github.com> Date: Wed, 22 Feb 2023 00:51:59 +0900 Subject: [PATCH 22/94] du: move help strings to markdown file (#4384) --- src/uu/du/du.md | 24 ++++++++++++++++++++++++ src/uu/du/src/du.rs | 28 +++++++--------------------- 2 files changed, 31 insertions(+), 21 deletions(-) create mode 100644 src/uu/du/du.md diff --git a/src/uu/du/du.md b/src/uu/du/du.md new file mode 100644 index 000000000..6f154d047 --- /dev/null +++ b/src/uu/du/du.md @@ -0,0 +1,24 @@ +# du + +``` +du [OPTION]... [FILE]... +du [OPTION]... --files0-from=F +``` + +Estimate file space usage + +## After Help + +Display values are in units of the first available SIZE from --block-size, +and the DU_BLOCK_SIZE, BLOCK_SIZE and BLOCKSIZE environment variables. +Otherwise, units default to 1024 bytes (or 512 if POSIXLY_CORRECT is set). + +SIZE is an integer and optional unit (example: 10M is 10*1024*1024). +Units are K, M, G, T, P, E, Z, Y (powers of 1024) or KB, MB,... (powers +of 1000). + +PATTERN allows some advanced exclusions. For example, the following syntaxes +are supported: +? will match only one character +* will match zero or more characters +{a,b} will match a or b diff --git a/src/uu/du/src/du.rs b/src/uu/du/src/du.rs index a204adf4d..c35bbf119 100644 --- a/src/uu/du/src/du.rs +++ b/src/uu/du/src/du.rs @@ -36,7 +36,9 @@ use uucore::error::FromIo; use uucore::error::{UError, UResult}; use uucore::parse_glob; use uucore::parse_size::{parse_size, ParseSizeError}; -use uucore::{crash, format_usage, show, show_error, show_warning}; +use uucore::{ + crash, format_usage, help_about, help_section, help_usage, show, show_error, show_warning, +}; #[cfg(windows)] use windows_sys::Win32::Foundation::HANDLE; #[cfg(windows)] @@ -73,25 +75,9 @@ mod options { pub const FILE: &str = "FILE"; } -const ABOUT: &str = "Estimate file space usage"; -const LONG_HELP: &str = " -Display values are in units of the first available SIZE from --block-size, -and the DU_BLOCK_SIZE, BLOCK_SIZE and BLOCKSIZE environment variables. -Otherwise, units default to 1024 bytes (or 512 if POSIXLY_CORRECT is set). - -SIZE is an integer and optional unit (example: 10M is 10*1024*1024). -Units are K, M, G, T, P, E, Z, Y (powers of 1024) or KB, MB,... (powers -of 1000). - -PATTERN allows some advanced exclusions. For example, the following syntaxes -are supported: -? will match only one character -* will match zero or more characters -{a,b} will match a or b -"; -const USAGE: &str = "\ - {} [OPTION]... [FILE]... - {} [OPTION]... --files0-from=F"; +const ABOUT: &str = help_about!("du.md"); +const AFTER_HELP: &str = help_section!("after help", "du.md"); +const USAGE: &str = help_usage!("du.md"); // TODO: Support Z & Y (currently limited by size of u64) const UNITS: [(char, u32); 6] = [('E', 6), ('P', 5), ('T', 4), ('G', 3), ('M', 2), ('K', 1)]; @@ -705,7 +691,7 @@ pub fn uu_app() -> Command { Command::new(uucore::util_name()) .version(crate_version!()) .about(ABOUT) - .after_help(LONG_HELP) + .after_help(AFTER_HELP) .override_usage(format_usage(USAGE)) .infer_long_args(true) .disable_help_flag(true) From e6fec39db136014d64bc0ddecf464424ecb764a8 Mon Sep 17 00:00:00 2001 From: Cheng-Hao Date: Tue, 21 Feb 2023 11:05:35 -0800 Subject: [PATCH 23/94] mkdir: update markdown naming for consistency --- src/uu/mkdir/mkdir.md | 2 +- src/uu/mkdir/src/mkdir.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/uu/mkdir/mkdir.md b/src/uu/mkdir/mkdir.md index 79deafc46..1a6041a7e 100644 --- a/src/uu/mkdir/mkdir.md +++ b/src/uu/mkdir/mkdir.md @@ -6,6 +6,6 @@ mkdir [OPTION]... [USER] Create the given DIRECTORY(ies) if they do not exist -## Long Usage +## After Help Each MODE is of the form '[ugoa]*([-+=]([rwxXst]*|[ugo]))+|[-+=]?[0-7]+'. diff --git a/src/uu/mkdir/src/mkdir.rs b/src/uu/mkdir/src/mkdir.rs index ab92e7e74..9f490ecf9 100644 --- a/src/uu/mkdir/src/mkdir.rs +++ b/src/uu/mkdir/src/mkdir.rs @@ -24,7 +24,7 @@ static DEFAULT_PERM: u32 = 0o755; const ABOUT: &str = help_about!("mkdir.md"); const USAGE: &str = help_usage!("mkdir.md"); -const LONG_USAGE: &str = help_section!("long usage", "mkdir.md"); +const AFTER_HELP: &str = help_section!("after help", "mkdir.md"); mod options { pub const MODE: &str = "mode"; @@ -89,7 +89,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { // Linux-specific options, not implemented // opts.optflag("Z", "context", "set SELinux security context" + // " of each created directory to CTX"), - let matches = uu_app().after_help(LONG_USAGE).try_get_matches_from(args)?; + let matches = uu_app().after_help(AFTER_HELP).try_get_matches_from(args)?; let dirs = matches .get_many::(options::DIRS) From 7d7b9eb49ac634c9758b1143d64643fc463c96bd Mon Sep 17 00:00:00 2001 From: "Guilherme A. de Souza" <125218612+rgasnix@users.noreply.github.com> Date: Tue, 21 Feb 2023 17:52:18 -0300 Subject: [PATCH 24/94] Migrate from `atty` to `is-terminal` (#4382) --- Cargo.lock | 162 ++++++++++++++++++++++++++----------- Cargo.toml | 4 +- deny.toml | 14 +++- src/uu/cat/Cargo.toml | 2 +- src/uu/cat/src/cat.rs | 3 +- src/uu/cut/Cargo.toml | 2 +- src/uu/cut/src/cut.rs | 3 +- src/uu/ls/Cargo.toml | 2 +- src/uu/ls/src/ls.rs | 9 ++- src/uu/more/Cargo.toml | 2 +- src/uu/more/src/more.rs | 3 +- src/uu/nohup/Cargo.toml | 2 +- src/uu/nohup/src/nohup.rs | 7 +- src/uu/tail/Cargo.toml | 2 +- src/uu/tail/src/args.rs | 4 +- src/uu/tty/Cargo.toml | 2 +- src/uu/tty/src/tty.rs | 3 +- tests/by-util/test_more.rs | 5 +- 18 files changed, 161 insertions(+), 70 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fd2385a64..8742a8092 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -67,7 +67,7 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ - "hermit-abi", + "hermit-abi 0.1.19", "libc", "winapi", ] @@ -293,7 +293,7 @@ dependencies = [ "lazy_static", "libc", "unicode-width", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -321,7 +321,6 @@ checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" name = "coreutils" version = "0.0.17" dependencies = [ - "atty", "chrono", "clap", "clap_complete", @@ -329,6 +328,7 @@ dependencies = [ "filetime", "glob", "hex-literal", + "is-terminal", "libc", "nix", "once_cell", @@ -635,7 +635,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1631ca6e3c59112501a9d87fd86f21591ff77acd31331e8a73f8d80a65bbdd71" dependencies = [ "nix", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -834,7 +834,7 @@ dependencies = [ "cfg-if", "libc", "redox_syscall", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -1045,6 +1045,12 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" + [[package]] name = "hex" version = "0.4.3" @@ -1139,6 +1145,28 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59ce5ef949d49ee85593fc4d3f3f95ad61657076395cbbce23e2121fc5542074" +[[package]] +name = "io-lifetimes" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1abeb7a0dd0f8181267ff8adc397075586500b81b28a73e8a0208b00fc170fb3" +dependencies = [ + "libc", + "windows-sys 0.45.0", +] + +[[package]] +name = "is-terminal" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0a45d56fe973d6db23972bf5bc46f988a4a2385deac9cc29572f09daef" +dependencies = [ + "hermit-abi 0.3.1", + "io-lifetimes 1.0.5", + "rustix 0.36.8", + "windows-sys 0.45.0", +] + [[package]] name = "itertools" version = "0.10.5" @@ -1235,6 +1263,12 @@ version = "0.0.46" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4d2456c373231a208ad294c33dc5bff30051eafd954cd4caae83a712b12854d" +[[package]] +name = "linux-raw-sys" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f051f77a7c8e6957c0696eac88f26b0117e54f52d3fc682ab19397a8812846a4" + [[package]] name = "lock_api" version = "0.4.9" @@ -1326,7 +1360,7 @@ dependencies = [ "libc", "log", "wasi", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -1415,7 +1449,7 @@ version = "1.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6058e64324c71e02bc2b150e4f3bc8286db6c83092132ffa3f6b1eab0f9def5" dependencies = [ - "hermit-abi", + "hermit-abi 0.1.19", "libc", ] @@ -1545,7 +1579,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -1693,7 +1727,7 @@ dependencies = [ "byteorder", "hex", "lazy_static", - "rustix", + "rustix 0.35.13", ] [[package]] @@ -1899,10 +1933,24 @@ checksum = "727a1a6d65f786ec22df8a81ca3121107f235970dc1705ed681d3e6e8b9cd5f9" dependencies = [ "bitflags", "errno", - "io-lifetimes", + "io-lifetimes 0.7.5", "libc", - "linux-raw-sys", - "windows-sys", + "linux-raw-sys 0.0.46", + "windows-sys 0.42.0", +] + +[[package]] +name = "rustix" +version = "0.36.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f43abb88211988493c1abb44a70efa56ff0ce98f233b7b276146f1f3f7ba9644" +dependencies = [ + "bitflags", + "errno", + "io-lifetimes 1.0.5", + "libc", + "linux-raw-sys 0.1.4", + "windows-sys 0.45.0", ] [[package]] @@ -2161,8 +2209,8 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40ca90c434fd12083d1a6bdcbe9f92a14f96c8a1ba600ba451734ac334521f7a" dependencies = [ - "rustix", - "windows-sys", + "rustix 0.35.13", + "windows-sys 0.42.0", ] [[package]] @@ -2334,8 +2382,8 @@ dependencies = [ name = "uu_cat" version = "0.0.17" dependencies = [ - "atty", "clap", + "is-terminal", "nix", "thiserror", "uucore", @@ -2432,9 +2480,9 @@ dependencies = [ name = "uu_cut" version = "0.0.17" dependencies = [ - "atty", "bstr", "clap", + "is-terminal", "memchr", "uucore", ] @@ -2447,7 +2495,7 @@ dependencies = [ "clap", "libc", "uucore", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -2503,7 +2551,7 @@ dependencies = [ "clap", "glob", "uucore", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -2634,7 +2682,7 @@ dependencies = [ "clap", "hostname", "uucore", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -2705,10 +2753,10 @@ dependencies = [ name = "uu_ls" version = "0.0.17" dependencies = [ - "atty", "chrono", "clap", "glob", + "is-terminal", "lscolors", "number_prefix", "once_cell", @@ -2759,9 +2807,9 @@ dependencies = [ name = "uu_more" version = "0.0.17" dependencies = [ - "atty", "clap", "crossterm", + "is-terminal", "nix", "unicode-segmentation", "unicode-width", @@ -2801,8 +2849,8 @@ dependencies = [ name = "uu_nohup" version = "0.0.17" dependencies = [ - "atty", "clap", + "is-terminal", "libc", "uucore", ] @@ -2936,7 +2984,7 @@ dependencies = [ "libc", "uucore", "walkdir", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -3079,7 +3127,7 @@ dependencies = [ "libc", "nix", "uucore", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -3097,16 +3145,16 @@ dependencies = [ name = "uu_tail" version = "0.0.17" dependencies = [ - "atty", "clap", "fundu", + "is-terminal", "libc", "memchr", "notify", "same-file", "uucore", "winapi-util", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -3146,7 +3194,7 @@ dependencies = [ "filetime", "time", "uucore", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -3186,8 +3234,8 @@ dependencies = [ name = "uu_tty" version = "0.0.17" dependencies = [ - "atty", "clap", + "is-terminal", "nix", "uucore", ] @@ -3282,7 +3330,7 @@ dependencies = [ "clap", "libc", "uucore", - "windows-sys", + "windows-sys 0.42.0", ] [[package]] @@ -3316,7 +3364,7 @@ dependencies = [ "walkdir", "wild", "winapi-util", - "windows-sys", + "windows-sys 0.42.0", "z85", ] @@ -3478,46 +3526,70 @@ dependencies = [ ] [[package]] -name = "windows_aarch64_gnullvm" -version = "0.42.0" +name = "windows-sys" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41d2aa71f6f0cbe00ae5167d90ef3cfe66527d6f613ca78ac8024c3ccab9a19e" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" [[package]] name = "windows_aarch64_msvc" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd0f252f5a35cac83d6311b2e795981f5ee6e67eb1f9a7f64eb4500fbc4dcdb4" +checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" [[package]] name = "windows_i686_gnu" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbeae19f6716841636c28d695375df17562ca208b2b7d0dc47635a50ae6c5de7" +checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" [[package]] name = "windows_i686_msvc" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84c12f65daa39dd2babe6e442988fc329d6243fdce47d7d2d155b8d874862246" +checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" [[package]] name = "windows_x86_64_gnu" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf7b1b21b5362cbc318f686150e5bcea75ecedc74dd157d874d754a2ca44b0ed" +checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d525d2ba30eeb3297665bd434a54297e4170c7f1a44cad4ef58095b4cd2028" +checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" [[package]] name = "windows_x86_64_msvc" -version = "0.42.0" +version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40009d85759725a34da6d89a94e63d7bdc50a862acf0dbc7c8e488f1edcb6f5" +checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" [[package]] name = "xattr" diff --git a/Cargo.toml b/Cargo.toml index ebd8685ca..61d6db205 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -263,7 +263,6 @@ feat_os_windows_legacy = [ test = [ "uu_test" ] [workspace.dependencies] -atty = "0.2" bigdecimal = "0.3" binary-heap-plus = "0.5.0" bstr = "1.0" @@ -287,6 +286,7 @@ gcd = "2.2" glob = "0.3.0" half = "2.1" indicatif = "0.17" +is-terminal = "0.4.3" itertools = "0.10.0" libc = "0.2.139" lscolors = { version = "0.13.0", default-features=false, features = ["nu-ansi-term"] } @@ -476,7 +476,7 @@ time = { workspace=true, features=["local-offset"] } unindent = "0.1" uucore = { workspace=true, features=["entries", "process", "signals"] } walkdir = { workspace=true } -atty = { workspace=true } +is-terminal = { workspace=true } hex-literal = "0.3.1" rstest = "0.16.0" diff --git a/deny.toml b/deny.toml index 20995c936..aa05cdf3e 100644 --- a/deny.toml +++ b/deny.toml @@ -58,12 +58,24 @@ highlight = "all" # For each duplicate dependency, indicate the name of the dependency which # introduces it. # spell-checker: disable -skip = [] +skip = [ + # is-terminal + { name = "hermit-abi", version = "0.3.1" }, + # is-terminal + { name = "rustix", version = "0.36.8" }, + # is-terminal (via rustix) + { name = "io-lifetimes", version = "1.0.5" }, + # is-terminal + { name = "linux-raw-sys", version = "0.1.4" }, + # is-terminal + { name = "windows-sys", version = "0.45.0" }, +] # spell-checker: enable # This section is considered when running `cargo deny check sources`. # More documentation about the 'sources' section can be found here: # https://embarkstudios.github.io/cargo-deny/checks/sources/cfg.html + [sources] unknown-registry = "warn" unknown-git = "warn" diff --git a/src/uu/cat/Cargo.toml b/src/uu/cat/Cargo.toml index e707c6db0..fbf91c249 100644 --- a/src/uu/cat/Cargo.toml +++ b/src/uu/cat/Cargo.toml @@ -17,7 +17,7 @@ path = "src/cat.rs" [dependencies] clap = { workspace=true } thiserror = { workspace = true } -atty = { workspace=true } +is-terminal = { workspace = true } uucore = { workspace=true, features=["fs", "pipes"] } [target.'cfg(unix)'.dependencies] diff --git a/src/uu/cat/src/cat.rs b/src/uu/cat/src/cat.rs index 85a626d5b..e90bdd116 100644 --- a/src/uu/cat/src/cat.rs +++ b/src/uu/cat/src/cat.rs @@ -12,6 +12,7 @@ // last synced with: cat (GNU coreutils) 8.13 use clap::{crate_version, Arg, ArgAction, Command}; +use is_terminal::IsTerminal; use std::fs::{metadata, File}; use std::io::{self, Read, Write}; use thiserror::Error; @@ -332,7 +333,7 @@ fn cat_path( let stdin = io::stdin(); let mut handle = InputHandle { reader: stdin, - is_interactive: atty::is(atty::Stream::Stdin), + is_interactive: std::io::stdin().is_terminal(), }; cat_handle(&mut handle, options, state) } diff --git a/src/uu/cut/Cargo.toml b/src/uu/cut/Cargo.toml index f680b67e5..8a5e8f1d2 100644 --- a/src/uu/cut/Cargo.toml +++ b/src/uu/cut/Cargo.toml @@ -19,7 +19,7 @@ clap = { workspace=true } uucore = { workspace=true } memchr = { workspace=true } bstr = { workspace=true } -atty = { workspace=true } +is-terminal = { workspace=true } [[bin]] name = "cut" diff --git a/src/uu/cut/src/cut.rs b/src/uu/cut/src/cut.rs index 9db114fb2..105449cbc 100644 --- a/src/uu/cut/src/cut.rs +++ b/src/uu/cut/src/cut.rs @@ -9,6 +9,7 @@ use bstr::io::BufReadExt; use clap::{crate_version, Arg, ArgAction, Command}; +use is_terminal::IsTerminal; use std::fs::File; use std::io::{stdin, stdout, BufReader, BufWriter, Read, Write}; use std::path::Path; @@ -136,7 +137,7 @@ enum Mode { } fn stdout_writer() -> Box { - if atty::is(atty::Stream::Stdout) { + if std::io::stdout().is_terminal() { Box::new(stdout()) } else { Box::new(BufWriter::new(stdout())) as Box diff --git a/src/uu/ls/Cargo.toml b/src/uu/ls/Cargo.toml index ade9c5e29..7c7add294 100644 --- a/src/uu/ls/Cargo.toml +++ b/src/uu/ls/Cargo.toml @@ -25,7 +25,7 @@ glob = { workspace=true } lscolors = { workspace=true } uucore = { workspace=true, features = ["entries", "fs"] } once_cell = { workspace=true } -atty = { workspace=true } +is-terminal = { workspace=true } selinux = { workspace=true, optional = true } [[bin]] diff --git a/src/uu/ls/src/ls.rs b/src/uu/ls/src/ls.rs index 965248495..abf05676f 100644 --- a/src/uu/ls/src/ls.rs +++ b/src/uu/ls/src/ls.rs @@ -12,6 +12,7 @@ use clap::{ crate_version, Arg, ArgAction, Command, }; use glob::{MatchOptions, Pattern}; +use is_terminal::IsTerminal; use lscolors::LsColors; use number_prefix::NumberPrefix; use once_cell::unsync::OnceCell; @@ -451,7 +452,7 @@ impl Config { (Format::Commas, Some(options::format::COMMAS)) } else if options.get_flag(options::format::COLUMNS) { (Format::Columns, Some(options::format::COLUMNS)) - } else if atty::is(atty::Stream::Stdout) { + } else if std::io::stdout().is_terminal() { (Format::Columns, None) } else { (Format::OneLine, None) @@ -557,7 +558,7 @@ impl Config { None => options.contains_id(options::COLOR), Some(val) => match val.as_str() { "" | "always" | "yes" | "force" => true, - "auto" | "tty" | "if-tty" => atty::is(atty::Stream::Stdout), + "auto" | "tty" | "if-tty" => std::io::stdout().is_terminal(), /* "never" | "no" | "none" | */ _ => false, }, }; @@ -678,7 +679,7 @@ impl Config { } else if options.get_flag(options::SHOW_CONTROL_CHARS) { true } else { - !atty::is(atty::Stream::Stdout) + !std::io::stdout().is_terminal() }; let opt_quoting_style = options @@ -750,7 +751,7 @@ impl Config { "never" | "no" | "none" => IndicatorStyle::None, "always" | "yes" | "force" => IndicatorStyle::Classify, "auto" | "tty" | "if-tty" => { - if atty::is(atty::Stream::Stdout) { + if std::io::stdout().is_terminal() { IndicatorStyle::Classify } else { IndicatorStyle::None diff --git a/src/uu/more/Cargo.toml b/src/uu/more/Cargo.toml index 416a2359b..01bb96183 100644 --- a/src/uu/more/Cargo.toml +++ b/src/uu/more/Cargo.toml @@ -18,7 +18,7 @@ path = "src/more.rs" clap = { workspace=true } uucore = { workspace=true } crossterm = { workspace=true } -atty = { workspace=true } +is-terminal = { workspace=true } unicode-width = { workspace=true } unicode-segmentation = { workspace=true } diff --git a/src/uu/more/src/more.rs b/src/uu/more/src/more.rs index 0309ad33e..31b1640a2 100644 --- a/src/uu/more/src/more.rs +++ b/src/uu/more/src/more.rs @@ -23,6 +23,7 @@ use crossterm::{ terminal, }; +use is_terminal::IsTerminal; use unicode_segmentation::UnicodeSegmentation; use unicode_width::UnicodeWidthStr; use uucore::display::Quotable; @@ -83,7 +84,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { buff.clear(); } reset_term(&mut stdout); - } else if atty::isnt(atty::Stream::Stdin) { + } else if !std::io::stdin().is_terminal() { stdin().read_to_string(&mut buff).unwrap(); let mut stdout = setup_term(); more(&buff, &mut stdout, None, silent)?; diff --git a/src/uu/nohup/Cargo.toml b/src/uu/nohup/Cargo.toml index 9a6b4af43..f35869506 100644 --- a/src/uu/nohup/Cargo.toml +++ b/src/uu/nohup/Cargo.toml @@ -17,7 +17,7 @@ path = "src/nohup.rs" [dependencies] clap = { workspace=true } libc = { workspace=true } -atty = { workspace=true } +is-terminal = { workspace=true } uucore = { workspace=true, features=["fs"] } [[bin]] diff --git a/src/uu/nohup/src/nohup.rs b/src/uu/nohup/src/nohup.rs index 789888ea4..d00ef4931 100644 --- a/src/uu/nohup/src/nohup.rs +++ b/src/uu/nohup/src/nohup.rs @@ -8,6 +8,7 @@ // spell-checker:ignore (ToDO) execvp SIGHUP cproc vprocmgr cstrs homeout use clap::{crate_version, Arg, ArgAction, Command}; +use is_terminal::IsTerminal; use libc::{c_char, dup2, execvp, signal}; use libc::{SIGHUP, SIG_IGN}; use std::env; @@ -129,7 +130,7 @@ pub fn uu_app() -> Command { } fn replace_fds() -> UResult<()> { - if atty::is(atty::Stream::Stdin) { + if std::io::stdin().is_terminal() { let new_stdin = File::open(Path::new("/dev/null")) .map_err(|e| NohupError::CannotReplace("STDIN", e))?; if unsafe { dup2(new_stdin.as_raw_fd(), 0) } != 0 { @@ -137,7 +138,7 @@ fn replace_fds() -> UResult<()> { } } - if atty::is(atty::Stream::Stdout) { + if std::io::stdout().is_terminal() { let new_stdout = find_stdout()?; let fd = new_stdout.as_raw_fd(); @@ -146,7 +147,7 @@ fn replace_fds() -> UResult<()> { } } - if atty::is(atty::Stream::Stderr) && unsafe { dup2(1, 2) } != 2 { + if std::io::stderr().is_terminal() && unsafe { dup2(1, 2) } != 2 { return Err(NohupError::CannotReplace("STDERR", Error::last_os_error()).into()); } Ok(()) diff --git a/src/uu/tail/Cargo.toml b/src/uu/tail/Cargo.toml index bd16132ea..88b82a8e6 100644 --- a/src/uu/tail/Cargo.toml +++ b/src/uu/tail/Cargo.toml @@ -22,7 +22,7 @@ memchr = { workspace=true } notify = { workspace=true } uucore = { workspace=true } same-file = { workspace=true } -atty = { workspace=true } +is-terminal = { workspace=true } fundu = { workspace=true } [target.'cfg(windows)'.dependencies] diff --git a/src/uu/tail/src/args.rs b/src/uu/tail/src/args.rs index bfc314185..0807aa912 100644 --- a/src/uu/tail/src/args.rs +++ b/src/uu/tail/src/args.rs @@ -7,10 +7,10 @@ use crate::paths::Input; use crate::{parse, platform, Quotable}; -use atty::Stream; use clap::crate_version; use clap::{parser::ValueSource, Arg, ArgAction, ArgMatches, Command}; use fundu::DurationParser; +use is_terminal::IsTerminal; use same_file::Handle; use std::collections::VecDeque; use std::ffi::OsString; @@ -274,7 +274,7 @@ impl Settings { .map_or(false, |meta| !meta.is_file()) }); - if !blocking_stdin && atty::is(Stream::Stdin) { + if !blocking_stdin && std::io::stdin().is_terminal() { show_warning!("following standard input indefinitely is ineffective"); } } diff --git a/src/uu/tty/Cargo.toml b/src/uu/tty/Cargo.toml index 34eb84570..ee7fe559b 100644 --- a/src/uu/tty/Cargo.toml +++ b/src/uu/tty/Cargo.toml @@ -17,7 +17,7 @@ path = "src/tty.rs" [dependencies] clap = { workspace=true } nix = { workspace=true, features=["term"] } -atty = { workspace=true } +is-terminal = { workspace=true } uucore = { workspace=true, features=["fs"] } [[bin]] diff --git a/src/uu/tty/src/tty.rs b/src/uu/tty/src/tty.rs index ad41c323c..e3fc0451e 100644 --- a/src/uu/tty/src/tty.rs +++ b/src/uu/tty/src/tty.rs @@ -10,6 +10,7 @@ // spell-checker:ignore (ToDO) ttyname filedesc use clap::{crate_version, Arg, ArgAction, Command}; +use is_terminal::IsTerminal; use std::io::Write; use std::os::unix::io::AsRawFd; use uucore::error::{set_exit_code, UResult}; @@ -30,7 +31,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { // If silent, we don't need the name, only whether or not stdin is a tty. if silent { - return if atty::is(atty::Stream::Stdin) { + return if std::io::stdin().is_terminal() { Ok(()) } else { Err(1.into()) diff --git a/tests/by-util/test_more.rs b/tests/by-util/test_more.rs index 5c2b3c0f6..b3a1522f6 100644 --- a/tests/by-util/test_more.rs +++ b/tests/by-util/test_more.rs @@ -1,9 +1,10 @@ use crate::common::util::*; +use is_terminal::IsTerminal; #[test] fn test_more_no_arg() { // Reading from stdin is now supported, so this must succeed - if atty::is(atty::Stream::Stdout) { + if std::io::stdout().is_terminal() { new_ucmd!().succeeds(); } else { } @@ -14,7 +15,7 @@ fn test_more_dir_arg() { // Run the test only if there's a valid terminal, else do nothing // Maybe we could capture the error, i.e. "Device not found" in that case // but I am leaving this for later - if atty::is(atty::Stream::Stdout) { + if std::io::stdout().is_terminal() { new_ucmd!() .arg(".") .fails() From ddd40fda6b89f4bb20e7bfdaf607e7d4356aeb22 Mon Sep 17 00:00:00 2001 From: Chen Chi <114895836+ChenChiii@users.noreply.github.com> Date: Tue, 21 Feb 2023 15:56:52 -0800 Subject: [PATCH 25/94] sleep: move help strings to markdown file --- src/uu/sleep/sleep.md | 16 ++++++++++++++++ src/uu/sleep/src/sleep.rs | 16 +++++----------- 2 files changed, 21 insertions(+), 11 deletions(-) create mode 100644 src/uu/sleep/sleep.md diff --git a/src/uu/sleep/sleep.md b/src/uu/sleep/sleep.md new file mode 100644 index 000000000..0a9a39a19 --- /dev/null +++ b/src/uu/sleep/sleep.md @@ -0,0 +1,16 @@ +# sleep + +``` +sleep NUMBER[SUFFIX]... +sleep OPTION +``` + +Pause for NUMBER seconds. + +## After Help + +Pause for NUMBER seconds. SUFFIX may be 's' for seconds (the default), +'m' for minutes, 'h' for hours or 'd' for days. Unlike most implementations +that require NUMBER be an integer, here NUMBER may be an arbitrary floating +point number. Given two or more arguments, pause for the amount of time +specified by the sum of their values. \ No newline at end of file diff --git a/src/uu/sleep/src/sleep.rs b/src/uu/sleep/src/sleep.rs index eecdc0512..517fda098 100644 --- a/src/uu/sleep/src/sleep.rs +++ b/src/uu/sleep/src/sleep.rs @@ -10,20 +10,14 @@ use std::time::Duration; use uucore::{ error::{UResult, USimpleError, UUsageError}, - format_usage, show, + format_usage, help_about, help_section, help_usage, show, }; use clap::{crate_version, Arg, ArgAction, Command}; -static ABOUT: &str = "Pause for NUMBER seconds."; -const USAGE: &str = "\ - {} NUMBER[SUFFIX]... - {} OPTION"; -static LONG_HELP: &str = "Pause for NUMBER seconds. SUFFIX may be 's' for seconds (the default), -'m' for minutes, 'h' for hours or 'd' for days. Unlike most implementations -that require NUMBER be an integer, here NUMBER may be an arbitrary floating -point number. Given two or more arguments, pause for the amount of time -specified by the sum of their values."; +static ABOUT: &str = help_about!("sleep.md"); +const USAGE: &str = help_usage!("sleep.md"); +static AFTER_HELP: &str = help_section!("after help", "sleep.md"); mod options { pub const NUMBER: &str = "NUMBER"; @@ -54,7 +48,7 @@ pub fn uu_app() -> Command { Command::new(uucore::util_name()) .version(crate_version!()) .about(ABOUT) - .after_help(LONG_HELP) + .after_help(AFTER_HELP) .override_usage(format_usage(USAGE)) .infer_long_args(true) .arg( From ab5d0b6b9defe2160f5d4cdfd115fb3334c162f4 Mon Sep 17 00:00:00 2001 From: Cheng-Hao Date: Tue, 21 Feb 2023 18:28:35 -0800 Subject: [PATCH 26/94] rmdir: move help strings to markdown file --- src/uu/rmdir/rmdir.md | 7 +++++++ src/uu/rmdir/src/rmdir.rs | 6 +++--- 2 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 src/uu/rmdir/rmdir.md diff --git a/src/uu/rmdir/rmdir.md b/src/uu/rmdir/rmdir.md new file mode 100644 index 000000000..16f7bf82b --- /dev/null +++ b/src/uu/rmdir/rmdir.md @@ -0,0 +1,7 @@ +# rmdir + +``` +rmdir [OPTION]... DIRECTORY... +``` + +Remove the DIRECTORY(ies), if they are empty. diff --git a/src/uu/rmdir/src/rmdir.rs b/src/uu/rmdir/src/rmdir.rs index 51240b5ad..9a864266a 100644 --- a/src/uu/rmdir/src/rmdir.rs +++ b/src/uu/rmdir/src/rmdir.rs @@ -16,10 +16,10 @@ use std::path::Path; use uucore::display::Quotable; use uucore::error::{set_exit_code, strip_errno, UResult}; -use uucore::{format_usage, show_error, util_name}; +use uucore::{format_usage, help_about, help_usage, show_error, util_name}; -static ABOUT: &str = "Remove the DIRECTORY(ies), if they are empty."; -const USAGE: &str = "{} [OPTION]... DIRECTORY..."; +static ABOUT: &str = help_about!("rmdir.md"); +const USAGE: &str = help_usage!("rmdir.md"); static OPT_IGNORE_FAIL_NON_EMPTY: &str = "ignore-fail-on-non-empty"; static OPT_PARENTS: &str = "parents"; static OPT_VERBOSE: &str = "verbose"; From 4cbf9a63987f84aeb7759016b4c36592d6e9a4b4 Mon Sep 17 00:00:00 2001 From: Cheng-Hao Date: Tue, 21 Feb 2023 18:50:47 -0800 Subject: [PATCH 27/94] rm: move help strings to markdown file --- src/uu/rm/rm.md | 22 ++++++++++++++++++++++ src/uu/rm/src/rm.rs | 22 +++++----------------- 2 files changed, 27 insertions(+), 17 deletions(-) create mode 100644 src/uu/rm/rm.md diff --git a/src/uu/rm/rm.md b/src/uu/rm/rm.md new file mode 100644 index 000000000..7acc46363 --- /dev/null +++ b/src/uu/rm/rm.md @@ -0,0 +1,22 @@ +# rm + +``` +rm [OPTION]... FILE... +``` + +Remove (unlink) the FILE(s) + +## After Help + +By default, rm does not remove directories. Use the --recursive (-r or -R) +option to remove each listed directory, too, along with all of its contents + +To remove a file whose name starts with a '-', for example '-foo', +use one of these commands: +rm -- -foo + +rm ./-foo + +Note that if you use rm to remove a file, it might be possible to recover +some of its contents, given sufficient expertise and/or time. For greater +assurance that the contents are truly unrecoverable, consider using shred. diff --git a/src/uu/rm/src/rm.rs b/src/uu/rm/src/rm.rs index 0d621355d..05d03cee2 100644 --- a/src/uu/rm/src/rm.rs +++ b/src/uu/rm/src/rm.rs @@ -15,7 +15,7 @@ use std::ops::BitOr; use std::path::{Path, PathBuf}; use uucore::display::Quotable; use uucore::error::{UResult, USimpleError, UUsageError}; -use uucore::{format_usage, prompt_yes, show_error}; +use uucore::{format_usage, help_about, help_section, help_usage, prompt_yes, show_error}; use walkdir::{DirEntry, WalkDir}; #[derive(Eq, PartialEq, Clone, Copy)] @@ -37,21 +37,9 @@ struct Options { verbose: bool, } -const ABOUT: &str = "Remove (unlink) the FILE(s)"; -const USAGE: &str = "{} [OPTION]... FILE..."; -const LONG_USAGE: &str = "\ -By default, rm does not remove directories. Use the --recursive (-r or -R) -option to remove each listed directory, too, along with all of its contents - -To remove a file whose name starts with a '-', for example '-foo', -use one of these commands: -rm -- -foo - -rm ./-foo - -Note that if you use rm to remove a file, it might be possible to recover -some of its contents, given sufficient expertise and/or time. For greater -assurance that the contents are truly unrecoverable, consider using shred."; +const ABOUT: &str = help_about!("rm.md"); +const USAGE: &str = help_usage!("rm.md"); +const AFTER_HELP: &str = help_section!("after help", "rm.md"); static OPT_DIR: &str = "dir"; static OPT_INTERACTIVE: &str = "interactive"; @@ -69,7 +57,7 @@ static ARG_FILES: &str = "files"; #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { - let matches = uu_app().after_help(LONG_USAGE).try_get_matches_from(args)?; + let matches = uu_app().after_help(AFTER_HELP).try_get_matches_from(args)?; let files: Vec = matches .get_many::(ARG_FILES) From 52902f6e99b978468fca7dcc9aff2414ca2f8d1f Mon Sep 17 00:00:00 2001 From: Koki Ueha Date: Wed, 22 Feb 2023 10:41:49 +0000 Subject: [PATCH 28/94] nohup: move help strings to markdown file --- src/uu/nohup/nohup.md | 15 +++++++++++++++ src/uu/nohup/src/nohup.rs | 17 +++++------------ 2 files changed, 20 insertions(+), 12 deletions(-) create mode 100644 src/uu/nohup/nohup.md diff --git a/src/uu/nohup/nohup.md b/src/uu/nohup/nohup.md new file mode 100644 index 000000000..cc35ef9f4 --- /dev/null +++ b/src/uu/nohup/nohup.md @@ -0,0 +1,15 @@ +# nohup + +``` +nohup COMMAND [ARG]... +nohup FLAG +``` + +Run COMMAND ignoring hangup signals. + +## After Help + +If standard input is terminal, it'll be replaced with /dev/null. +If standard output is terminal, it'll be appended to nohup.out instead, +or $HOME/nohup.out, if nohup.out open failed. +If standard error is terminal, it'll be redirected to stdout. diff --git a/src/uu/nohup/src/nohup.rs b/src/uu/nohup/src/nohup.rs index d00ef4931..8247cdb3e 100644 --- a/src/uu/nohup/src/nohup.rs +++ b/src/uu/nohup/src/nohup.rs @@ -20,18 +20,11 @@ use std::os::unix::prelude::*; use std::path::{Path, PathBuf}; use uucore::display::Quotable; use uucore::error::{set_exit_code, UClapError, UError, UResult}; -use uucore::{format_usage, show_error}; +use uucore::{format_usage, help_about, help_section, help_usage, show_error}; -static ABOUT: &str = "Run COMMAND ignoring hangup signals."; -static LONG_HELP: &str = " -If standard input is terminal, it'll be replaced with /dev/null. -If standard output is terminal, it'll be appended to nohup.out instead, -or $HOME/nohup.out, if nohup.out open failed. -If standard error is terminal, it'll be redirected to stdout. -"; -const USAGE: &str = "\ - {} COMMAND [ARG]... - {} FLAG"; +const ABOUT: &str = help_about!("nohup.md"); +const AFTER_HELP: &str = help_section!("after help", "nohup.md"); +const USAGE: &str = help_usage!("nohup.md"); static NOHUP_OUT: &str = "nohup.out"; // exit codes that match the GNU implementation static EXIT_CANCELED: i32 = 125; @@ -116,7 +109,7 @@ pub fn uu_app() -> Command { Command::new(uucore::util_name()) .version(crate_version!()) .about(ABOUT) - .after_help(LONG_HELP) + .after_help(AFTER_HELP) .override_usage(format_usage(USAGE)) .arg( Arg::new(options::CMD) From ed30869b45d8f7a9149a8ec85c8d78f50f619701 Mon Sep 17 00:00:00 2001 From: Terts Diepraam Date: Wed, 22 Feb 2023 16:23:27 +0100 Subject: [PATCH 29/94] fix spell checker error in mkdir.md --- src/uu/mkdir/mkdir.md | 1 + 1 file changed, 1 insertion(+) diff --git a/src/uu/mkdir/mkdir.md b/src/uu/mkdir/mkdir.md index 1a6041a7e..8773d15f4 100644 --- a/src/uu/mkdir/mkdir.md +++ b/src/uu/mkdir/mkdir.md @@ -1,3 +1,4 @@ + # mkdir ``` From 215803ec8d6a879331630924cd9f8d152e330ec4 Mon Sep 17 00:00:00 2001 From: Arvid Norlander Date: Wed, 22 Feb 2023 16:02:55 +0100 Subject: [PATCH 30/94] stty: Support for --file/-F Fixes issue #3863 --- src/uu/stty/src/stty.rs | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/uu/stty/src/stty.rs b/src/uu/stty/src/stty.rs index 6bd3c2f65..928c3f177 100644 --- a/src/uu/stty/src/stty.rs +++ b/src/uu/stty/src/stty.rs @@ -3,19 +3,20 @@ // * For the full copyright and license information, please view the LICENSE file // * that was distributed with this source code. -// spell-checker:ignore tcgetattr tcsetattr tcsanow tiocgwinsz tiocswinsz cfgetospeed ushort +// spell-checker:ignore clocal tcgetattr tcsetattr tcsanow tiocgwinsz tiocswinsz cfgetospeed ushort mod flags; use clap::{crate_version, Arg, ArgAction, ArgMatches, Command}; -use nix::libc::{c_ushort, TIOCGWINSZ, TIOCSWINSZ}; +use nix::libc::{c_ushort, O_NONBLOCK, TIOCGWINSZ, TIOCSWINSZ}; use nix::sys::termios::{ cfgetospeed, tcgetattr, tcsetattr, ControlFlags, InputFlags, LocalFlags, OutputFlags, Termios, }; use nix::{ioctl_read_bad, ioctl_write_ptr_bad}; use std::io::{self, stdout}; use std::ops::ControlFlow; -use std::os::unix::io::{AsRawFd, RawFd}; +use std::os::unix::fs::OpenOptionsExt; +use std::os::unix::io::{AsRawFd, IntoRawFd, RawFd}; use uucore::error::{UResult, USimpleError}; use uucore::format_usage; @@ -102,7 +103,20 @@ impl<'a> Options<'a> { all: matches.get_flag(options::ALL), save: matches.get_flag(options::SAVE), file: match matches.get_one::(options::FILE) { - Some(_f) => todo!(), + // Two notes here: + // 1. O_NONBLOCK is needed because according to GNU docs, a + // POSIX tty can block waiting for carrier-detect if the + // "clocal" flag is not set. If your TTY is not connected + // to a modem, it is probably not relevant though. + // 2. We never close the FD that we open here, but the OS + // will clean up the FD for us on exit, so it doesn't + // matter. The alternative would be to have an enum of + // BorrowedFd/OwnedFd to handle both cases. + Some(f) => std::fs::OpenOptions::new() + .read(true) + .custom_flags(O_NONBLOCK) + .open(f)? + .into_raw_fd(), None => stdout().as_raw_fd(), }, settings: matches From 29a25a6c8527ac5861a29504702228bb8eef32b8 Mon Sep 17 00:00:00 2001 From: papparapa <37232476+papparapa@users.noreply.github.com> Date: Thu, 23 Feb 2023 00:44:55 +0900 Subject: [PATCH 31/94] od: move help strings to markdown file (#4406) * od: move help strings to markdown file * od: fix the commented points --- src/uu/od/od.md | 49 ++++++++++++++++++++++++++++++++++++++++++ src/uu/od/src/od.rs | 52 +++++---------------------------------------- 2 files changed, 54 insertions(+), 47 deletions(-) create mode 100644 src/uu/od/od.md diff --git a/src/uu/od/od.md b/src/uu/od/od.md new file mode 100644 index 000000000..5cd9ac3b5 --- /dev/null +++ b/src/uu/od/od.md @@ -0,0 +1,49 @@ +# od + +``` +od [OPTION]... [--] [FILENAME]... +od [-abcdDefFhHiIlLoOsxX] [FILENAME] [[+][0x]OFFSET[.][b]] +od --traditional [OPTION]... [FILENAME] [[+][0x]OFFSET[.][b] [[+][0x]LABEL[.][b]]] +``` + +Dump files in octal and other formats + +## After Help + +Displays data in various human-readable formats. If multiple formats are +specified, the output will contain all formats in the order they appear on the +command line. Each format will be printed on a new line. Only the line +containing the first format will be prefixed with the offset. + +If no filename is specified, or it is "-", stdin will be used. After a "--", no +more options will be recognized. This allows for filenames starting with a "-". + +If a filename is a valid number which can be used as an offset in the second +form, you can force it to be recognized as a filename if you include an option +like "-j0", which is only valid in the first form. + +RADIX is one of o,d,x,n for octal, decimal, hexadecimal or none. + +BYTES is decimal by default, octal if prefixed with a "0", or hexadecimal if +prefixed with "0x". The suffixes b, KB, K, MB, M, GB, G, will multiply the +number with 512, 1000, 1024, 1000^2, 1024^2, 1000^3, 1024^3, 1000^2, 1024^2. + +OFFSET and LABEL are octal by default, hexadecimal if prefixed with "0x" or +decimal if a "." suffix is added. The "b" suffix will multiply with 512. + +TYPE contains one or more format specifications consisting of: + a for printable 7-bits ASCII + c for utf-8 characters or octal for undefined characters + d[SIZE] for signed decimal + f[SIZE] for floating point + o[SIZE] for octal + u[SIZE] for unsigned decimal + x[SIZE] for hexadecimal +SIZE is the number of bytes which can be the number 1, 2, 4, 8 or 16, + or C, I, S, L for 1, 2, 4, 8 bytes for integer types, + or F, D, L for 4, 8, 16 bytes for floating point. +Any type specification can have a "z" suffix, which will add a ASCII dump at + the end of the line. + +If an error occurred, a diagnostic message will be printed to stderr, and the +exit code will be non-zero. diff --git a/src/uu/od/src/od.rs b/src/uu/od/src/od.rs index ebcaa50ba..e41776ac4 100644 --- a/src/uu/od/src/od.rs +++ b/src/uu/od/src/od.rs @@ -44,57 +44,15 @@ use clap::ArgAction; use clap::{crate_version, parser::ValueSource, Arg, ArgMatches, Command}; use uucore::display::Quotable; use uucore::error::{UResult, USimpleError}; -use uucore::format_usage; use uucore::parse_size::ParseSizeError; -use uucore::show_error; -use uucore::show_warning; +use uucore::{format_usage, help_about, help_section, help_usage, show_error, show_warning}; const PEEK_BUFFER_SIZE: usize = 4; // utf-8 can be 4 bytes -static ABOUT: &str = "Dump files in octal and other formats"; +const ABOUT: &str = help_about!("od.md"); -static USAGE: &str = "\ - {} [OPTION]... [--] [FILENAME]... - {} [-abcdDefFhHiIlLoOsxX] [FILENAME] [[+][0x]OFFSET[.][b]] - {} --traditional [OPTION]... [FILENAME] [[+][0x]OFFSET[.][b] [[+][0x]LABEL[.][b]]]"; +const USAGE: &str = help_usage!("od.md"); -static LONG_HELP: &str = r#" -Displays data in various human-readable formats. If multiple formats are -specified, the output will contain all formats in the order they appear on the -command line. Each format will be printed on a new line. Only the line -containing the first format will be prefixed with the offset. - -If no filename is specified, or it is "-", stdin will be used. After a "--", no -more options will be recognized. This allows for filenames starting with a "-". - -If a filename is a valid number which can be used as an offset in the second -form, you can force it to be recognized as a filename if you include an option -like "-j0", which is only valid in the first form. - -RADIX is one of o,d,x,n for octal, decimal, hexadecimal or none. - -BYTES is decimal by default, octal if prefixed with a "0", or hexadecimal if -prefixed with "0x". The suffixes b, KB, K, MB, M, GB, G, will multiply the -number with 512, 1000, 1024, 1000^2, 1024^2, 1000^3, 1024^3, 1000^2, 1024^2. - -OFFSET and LABEL are octal by default, hexadecimal if prefixed with "0x" or -decimal if a "." suffix is added. The "b" suffix will multiply with 512. - -TYPE contains one or more format specifications consisting of: - a for printable 7-bits ASCII - c for utf-8 characters or octal for undefined characters - d[SIZE] for signed decimal - f[SIZE] for floating point - o[SIZE] for octal - u[SIZE] for unsigned decimal - x[SIZE] for hexadecimal -SIZE is the number of bytes which can be the number 1, 2, 4, 8 or 16, - or C, I, S, L for 1, 2, 4, 8 bytes for integer types, - or F, D, L for 4, 8, 16 bytes for floating point. -Any type specification can have a "z" suffix, which will add a ASCII dump at - the end of the line. - -If an error occurred, a diagnostic message will be printed to stderr, and the -exitcode will be non-zero."#; +const AFTER_HELP: &str = help_section!("after help", "od.md"); pub(crate) mod options { pub const HELP: &str = "help"; @@ -295,7 +253,7 @@ pub fn uu_app() -> Command { .version(crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) - .after_help(LONG_HELP) + .after_help(AFTER_HELP) .trailing_var_arg(true) .dont_delimit_trailing_values(true) .infer_long_args(true) From b70131f14891137d82dde824cf2832db189679ea Mon Sep 17 00:00:00 2001 From: zleyyij <75810274+zleyyij@users.noreply.github.com> Date: Wed, 22 Feb 2023 08:50:27 -0700 Subject: [PATCH 32/94] arch: move help strings to a markdown file (#4373) Co-authored-by: zleyyij --- src/uu/arch/arch.md | 14 ++++++++++++++ src/uu/arch/src/arch.rs | 5 +++-- 2 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 src/uu/arch/arch.md diff --git a/src/uu/arch/arch.md b/src/uu/arch/arch.md new file mode 100644 index 000000000..7225ede2d --- /dev/null +++ b/src/uu/arch/arch.md @@ -0,0 +1,14 @@ +# arch + +``` +arch +``` + + +Display machine architecture + + +## After Help + +Determine architecture name for current machine. + diff --git a/src/uu/arch/src/arch.rs b/src/uu/arch/src/arch.rs index 62a07159e..a2208d8b0 100644 --- a/src/uu/arch/src/arch.rs +++ b/src/uu/arch/src/arch.rs @@ -10,9 +10,10 @@ use platform_info::*; use clap::{crate_version, Command}; use uucore::error::{FromIo, UResult}; +use uucore::{help_about, help_section}; -static ABOUT: &str = "Display machine architecture"; -static SUMMARY: &str = "Determine architecture name for current machine."; +static ABOUT: &str = help_about!("arch.md"); +static SUMMARY: &str = help_section!("after help", "arch.md"); #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { From dd216ee23eaee199165aac62902280d60265614c Mon Sep 17 00:00:00 2001 From: Benjamin Bara Date: Tue, 21 Feb 2023 07:05:44 +0100 Subject: [PATCH 33/94] utmpx: cast timeval fields to i64 --- src/uucore/src/lib/features/utmpx.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src/uucore/src/lib/features/utmpx.rs b/src/uucore/src/lib/features/utmpx.rs index ee8744721..599a02778 100644 --- a/src/uucore/src/lib/features/utmpx.rs +++ b/src/uucore/src/lib/features/utmpx.rs @@ -192,13 +192,9 @@ impl Utmpx { } /// A.K.A. ut.ut_tv pub fn login_time(&self) -> time::OffsetDateTime { - #[cfg(all(not(target_os = "freebsd"), not(target_vendor = "apple")))] - let ts_nanos: i128 = (self.inner.ut_tv.tv_sec as i64 * 1_000_000_000_i64 - + self.inner.ut_tv.tv_usec as i64 * 1_000_i64) - .into(); - #[cfg(any(target_os = "freebsd", target_vendor = "apple"))] - let ts_nanos: i128 = (self.inner.ut_tv.tv_sec * 1_000_000_000_i64 - + self.inner.ut_tv.tv_usec as i64 * 1_000_i64) + #[allow(clippy::unnecessary_cast)] + let ts_nanos: i128 = (1_000_000_000_i64 * self.inner.ut_tv.tv_sec as i64 + + 1_000_i64 * self.inner.ut_tv.tv_usec as i64) .into(); let local_offset = time::OffsetDateTime::now_local().unwrap().offset(); time::OffsetDateTime::from_unix_timestamp_nanos(ts_nanos) From 66893402289a3c472d07c3542c97e655ed5e333c Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Wed, 22 Feb 2023 11:22:34 +0100 Subject: [PATCH 34/94] gentoo has renamed the package See: https://bugs.gentoo.org/895754 --- docs/src/installation.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/installation.md b/docs/src/installation.md index 063935ed6..ada343ae9 100644 --- a/docs/src/installation.md +++ b/docs/src/installation.md @@ -55,10 +55,10 @@ export PATH=/usr/lib/cargo/bin/coreutils:$PATH ### Gentoo -[![Gentoo package](https://repology.org/badge/version-for-repo/gentoo/uutils-coreutils.svg)](https://packages.gentoo.org/packages/sys-apps/uutils) +[![Gentoo package](https://repology.org/badge/version-for-repo/gentoo/uutils-coreutils.svg)](https://packages.gentoo.org/packages/sys-apps/uutils-coreutils) ```bash -emerge -pv sys-apps/uutils +emerge -pv sys-apps/uutils-coreutils ``` ### Manjaro From a24da1d57539e68998d62c475f9c9a1119b28523 Mon Sep 17 00:00:00 2001 From: zleyyij <75810274+zleyyij@users.noreply.github.com> Date: Wed, 22 Feb 2023 13:02:39 -0700 Subject: [PATCH 35/94] basenc: Move strings to markdown file (#4417) basenc: moved help strings to markdown file --- src/uu/basenc/basenc.md | 12 ++++++++++++ src/uu/basenc/src/basenc.rs | 12 +++--------- 2 files changed, 15 insertions(+), 9 deletions(-) create mode 100644 src/uu/basenc/basenc.md diff --git a/src/uu/basenc/basenc.md b/src/uu/basenc/basenc.md new file mode 100644 index 000000000..17916bd4a --- /dev/null +++ b/src/uu/basenc/basenc.md @@ -0,0 +1,12 @@ +# basenc + +``` +basenc [OPTION]... [FILE]" +``` + +Encode/decode data and print to standard output +With no FILE, or when FILE is -, read standard input. + +When decoding, the input may contain newlines in addition to the bytes of +the formal alphabet. Use --ignore-garbage to attempt to recover +from any other non-alphabet bytes in the encoded stream. diff --git a/src/uu/basenc/src/basenc.rs b/src/uu/basenc/src/basenc.rs index 1bc2aae44..afc25c736 100644 --- a/src/uu/basenc/src/basenc.rs +++ b/src/uu/basenc/src/basenc.rs @@ -19,14 +19,10 @@ use uucore::{ use std::io::{stdin, Read}; use uucore::error::UClapError; -static ABOUT: &str = "\ -Encode/decode data and print to standard output -With no FILE, or when FILE is -, read standard input. +use uucore::{help_about, help_usage}; -When decoding, the input may contain newlines in addition to the bytes of -the formal alphabet. Use --ignore-garbage to attempt to recover -from any other non-alphabet bytes in the encoded stream. -"; +const ABOUT: &str = help_about!("basenc.md"); +const USAGE: &str = help_usage!("basenc.md"); const ENCODINGS: &[(&str, Format)] = &[ ("base64", Format::Base64), @@ -39,8 +35,6 @@ const ENCODINGS: &[(&str, Format)] = &[ ("z85", Format::Z85), ]; -const USAGE: &str = "{} [OPTION]... [FILE]"; - pub fn uu_app() -> Command { let mut command = base_common::base_app(ABOUT, USAGE); for encoding in ENCODINGS { From 74419366fd817cc921bbdbb4197ec21f12c72804 Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Wed, 22 Feb 2023 21:24:08 +0100 Subject: [PATCH 36/94] timeout: add a test to verify that it fails as expected Follow up of PR #4388 --- tests/by-util/test_timeout.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/by-util/test_timeout.rs b/tests/by-util/test_timeout.rs index 79d128d51..deedea6c7 100644 --- a/tests/by-util/test_timeout.rs +++ b/tests/by-util/test_timeout.rs @@ -121,6 +121,14 @@ fn test_invalid_signal() { .usage_error("'invalid': invalid signal"); } +#[test] +fn test_invalid_multi_byte_characters() { + new_ucmd!() + .args(&["10€", "sleep", "0"]) + .fails() + .usage_error("invalid time interval '10€'"); +} + /// Test that the long form of the `--kill-after` argument is recognized. #[test] fn test_kill_after_long() { From e37e5ad9159ca4ed8154076b16f3b883942e9fed Mon Sep 17 00:00:00 2001 From: curtain Date: Thu, 23 Feb 2023 09:43:54 +0800 Subject: [PATCH 37/94] basename: fix rustfmt problem --- src/uu/basename/src/basename.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/uu/basename/src/basename.rs b/src/uu/basename/src/basename.rs index 48f00f4b8..ed7faee65 100644 --- a/src/uu/basename/src/basename.rs +++ b/src/uu/basename/src/basename.rs @@ -11,7 +11,7 @@ use clap::{crate_version, Arg, ArgAction, Command}; use std::path::{is_separator, PathBuf}; use uucore::display::Quotable; use uucore::error::{UResult, UUsageError}; -use uucore::{format_usage, help_usage, help_about}; +use uucore::{format_usage, help_about, help_usage}; static ABOUT: &str = help_about!("basename.md"); From b1d4951c34e1efd3571cfe4a25506e970d8ae022 Mon Sep 17 00:00:00 2001 From: Yang Hau Date: Wed, 22 Feb 2023 11:23:39 +0800 Subject: [PATCH 38/94] add ALGORITHM_HELP_DESC --- src/uu/cksum/src/cksum.rs | 121 ++++++++++++++++++++++++++++++++------ 1 file changed, 103 insertions(+), 18 deletions(-) diff --git a/src/uu/cksum/src/cksum.rs b/src/uu/cksum/src/cksum.rs index cadf523c2..e66d5f029 100644 --- a/src/uu/cksum/src/cksum.rs +++ b/src/uu/cksum/src/cksum.rs @@ -25,20 +25,76 @@ use uucore::{ const USAGE: &str = "{} [OPTIONS] [FILE]..."; const ABOUT: &str = "Print CRC and size for each file"; +const ALGORITHM_OPTIONS_SYSV: &str = "sysv"; +const ALGORITHM_OPTIONS_BSD: &str = "bsd"; +const ALGORITHM_OPTIONS_CRC: &str = "crc"; +const ALGORITHM_OPTIONS_MD5: &str = "md5"; +const ALGORITHM_OPTIONS_SHA1: &str = "sha1"; +const ALGORITHM_OPTIONS_SHA224: &str = "sha224"; +const ALGORITHM_OPTIONS_SHA256: &str = "sha256"; +const ALGORITHM_OPTIONS_SHA384: &str = "sha384"; +const ALGORITHM_OPTIONS_SHA512: &str = "sha512"; +const ALGORITHM_OPTIONS_BLAKE2B: &str = "blake2b"; +const ALGORITHM_OPTIONS_SM3: &str = "sm3"; + fn detect_algo(program: &str) -> (&'static str, Box, usize) { match program { - "sysv" => ("SYSV", Box::new(SYSV::new()) as Box, 512), - "bsd" => ("BSD", Box::new(BSD::new()) as Box, 1024), - "crc" => ("CRC", Box::new(CRC::new()) as Box, 256), - "md5" => ("MD5", Box::new(Md5::new()) as Box, 128), - "sha1" => ("SHA1", Box::new(Sha1::new()) as Box, 160), - "sha224" => ("SHA224", Box::new(Sha224::new()) as Box, 224), - "sha256" => ("SHA256", Box::new(Sha256::new()) as Box, 256), - "sha384" => ("SHA384", Box::new(Sha384::new()) as Box, 384), - "sha512" => ("SHA512", Box::new(Sha512::new()) as Box, 512), - "blake2b" => ("BLAKE2", Box::new(Blake2b::new()) as Box, 512), - "sm3" => ("SM3", Box::new(Sm3::new()) as Box, 512), - _ => panic!("unknown algorithm"), + ALGORITHM_OPTIONS_SYSV => ( + ALGORITHM_OPTIONS_SYSV, + Box::new(SYSV::new()) as Box, + 512, + ), + ALGORITHM_OPTIONS_BSD => ( + ALGORITHM_OPTIONS_BSD, + Box::new(BSD::new()) as Box, + 1024, + ), + ALGORITHM_OPTIONS_CRC => ( + ALGORITHM_OPTIONS_CRC, + Box::new(CRC::new()) as Box, + 256, + ), + ALGORITHM_OPTIONS_MD5 => ( + ALGORITHM_OPTIONS_MD5, + Box::new(Md5::new()) as Box, + 128, + ), + ALGORITHM_OPTIONS_SHA1 => ( + ALGORITHM_OPTIONS_SHA1, + Box::new(Sha1::new()) as Box, + 160, + ), + ALGORITHM_OPTIONS_SHA224 => ( + ALGORITHM_OPTIONS_SHA224, + Box::new(Sha224::new()) as Box, + 224, + ), + ALGORITHM_OPTIONS_SHA256 => ( + ALGORITHM_OPTIONS_SHA256, + Box::new(Sha256::new()) as Box, + 256, + ), + ALGORITHM_OPTIONS_SHA384 => ( + ALGORITHM_OPTIONS_SHA384, + Box::new(Sha384::new()) as Box, + 384, + ), + ALGORITHM_OPTIONS_SHA512 => ( + ALGORITHM_OPTIONS_SHA512, + Box::new(Sha512::new()) as Box, + 512, + ), + ALGORITHM_OPTIONS_BLAKE2B => ( + ALGORITHM_OPTIONS_BLAKE2B, + Box::new(Blake2b::new()) as Box, + 512, + ), + ALGORITHM_OPTIONS_SM3 => ( + ALGORITHM_OPTIONS_SM3, + Box::new(Sm3::new()) as Box, + 512, + ), + _ => unreachable!("unknown algorithm: clap should have prevented this case"), } } @@ -80,23 +136,23 @@ where // The BSD checksum output is 5 digit integer let bsd_width = 5; match (options.algo_name, not_file) { - ("SYSV", true) => println!( + (ALGORITHM_OPTIONS_SYSV, true) => println!( "{} {}", sum.parse::().unwrap(), div_ceil(sz, options.output_bits) ), - ("SYSV", false) => println!( + (ALGORITHM_OPTIONS_SYSV, false) => println!( "{} {} {}", sum.parse::().unwrap(), div_ceil(sz, options.output_bits), filename.display() ), - ("BSD", true) => println!( + (ALGORITHM_OPTIONS_BSD, true) => println!( "{:0bsd_width$} {:bsd_width$}", sum.parse::().unwrap(), div_ceil(sz, options.output_bits) ), - ("BSD", false) => println!( + (ALGORITHM_OPTIONS_BSD, false) => println!( "{:0bsd_width$} {:bsd_width$} {}", sum.parse::().unwrap(), div_ceil(sz, options.output_bits), @@ -149,6 +205,21 @@ mod options { pub static ALGORITHM: &str = "algorithm"; } +const ALGORITHM_HELP_DESC: &str = + "DIGEST determines the digest algorithm and default output format:\n\ +\n\ +-a=sysv: (equivalent to sum -s)\n\ +-a=bsd: (equivalent to sum -r)\n\ +-a=crc: (equivalent to cksum)\n\ +-a=md5: (equivalent to md5sum)\n\ +-a=sha1: (equivalent to sha1sum)\n\ +-a=sha224: (equivalent to sha224sum)\n\ +-a=sha256: (equivalent to sha256sum)\n\ +-a=sha384: (equivalent to sha384sum)\n\ +-a=sha512: (equivalent to sha512sum)\n\ +-a=blake2b: (equivalent to b2sum)\n\ +-a=sm3: (only available through cksum)\n"; + #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { let args = args.collect_ignore(); @@ -157,7 +228,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { let algo_name: &str = match matches.get_one::(options::ALGORITHM) { Some(v) => v, - None => "crc", + None => ALGORITHM_OPTIONS_CRC, }; let (name, algo, bits) = detect_algo(algo_name); @@ -192,6 +263,20 @@ pub fn uu_app() -> Command { .long(options::ALGORITHM) .short('a') .help("select the digest type to use. See DIGEST below") - .value_name("ALGORITHM"), + .value_name("ALGORITHM") + .value_parser([ + ALGORITHM_OPTIONS_SYSV, + ALGORITHM_OPTIONS_BSD, + ALGORITHM_OPTIONS_CRC, + ALGORITHM_OPTIONS_MD5, + ALGORITHM_OPTIONS_SHA1, + ALGORITHM_OPTIONS_SHA224, + ALGORITHM_OPTIONS_SHA256, + ALGORITHM_OPTIONS_SHA384, + ALGORITHM_OPTIONS_SHA512, + ALGORITHM_OPTIONS_BLAKE2B, + ALGORITHM_OPTIONS_SM3, + ]), ) + .after_help(ALGORITHM_HELP_DESC) } From e54c0063b01194a7de38b27f4c2a8b1f7ea0dbd6 Mon Sep 17 00:00:00 2001 From: Chen Chi <114895836+ChenChiii@users.noreply.github.com> Date: Wed, 22 Feb 2023 21:01:38 -0800 Subject: [PATCH 39/94] shuf: move help strings to markdown file --- src/uu/shuf/shuf.md | 11 +++++++++++ src/uu/shuf/src/shuf.rs | 12 +++--------- 2 files changed, 14 insertions(+), 9 deletions(-) create mode 100644 src/uu/shuf/shuf.md diff --git a/src/uu/shuf/shuf.md b/src/uu/shuf/shuf.md new file mode 100644 index 000000000..41a275035 --- /dev/null +++ b/src/uu/shuf/shuf.md @@ -0,0 +1,11 @@ +# shuf + +``` +shuf [OPTION]... [FILE] +shuf -e [OPTION]... [ARG]... +shuf -i LO-HI [OPTION]...; +``` + +Shuffle the input by outputting a random permutation of input lines. +Each output permutation is equally likely. +With no FILE, or when FILE is -, read standard input. \ No newline at end of file diff --git a/src/uu/shuf/src/shuf.rs b/src/uu/shuf/src/shuf.rs index d0022f5f5..2481baf3d 100644 --- a/src/uu/shuf/src/shuf.rs +++ b/src/uu/shuf/src/shuf.rs @@ -15,7 +15,7 @@ use std::fs::File; use std::io::{stdin, stdout, BufReader, BufWriter, Read, Write}; use uucore::display::Quotable; use uucore::error::{FromIo, UResult, USimpleError}; -use uucore::format_usage; +use uucore::{format_usage, help_about, help_usage}; mod rand_read_adapter; @@ -25,14 +25,8 @@ enum Mode { InputRange((usize, usize)), } -static USAGE: &str = "\ - {} [OPTION]... [FILE] - {} -e [OPTION]... [ARG]... - {} -i LO-HI [OPTION]..."; -static ABOUT: &str = "\ - Shuffle the input by outputting a random permutation of input lines. \ - Each output permutation is equally likely. \ - With no FILE, or when FILE is -, read standard input."; +static USAGE: &str = help_usage!("shuf.md"); +static ABOUT: &str = help_about!("shuf.md"); struct Options { head_count: usize, From 0c4bde342cba65a6a90078596e09a09cd59e770f Mon Sep 17 00:00:00 2001 From: Cheng-Hao Date: Wed, 22 Feb 2023 23:14:54 -0800 Subject: [PATCH 40/94] link: move help strings to markdown file --- src/uu/link/link.md | 7 +++++++ src/uu/link/src/link.rs | 6 +++--- 2 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 src/uu/link/link.md diff --git a/src/uu/link/link.md b/src/uu/link/link.md new file mode 100644 index 000000000..ea6a531b9 --- /dev/null +++ b/src/uu/link/link.md @@ -0,0 +1,7 @@ +# link + +``` +link FILE1 FILE2 +``` + +Call the link function to create a link named FILE2 to an existing FILE1. diff --git a/src/uu/link/src/link.rs b/src/uu/link/src/link.rs index 80b2b1f9b..6688003a9 100644 --- a/src/uu/link/src/link.rs +++ b/src/uu/link/src/link.rs @@ -11,10 +11,10 @@ use std::fs::hard_link; use std::path::Path; use uucore::display::Quotable; use uucore::error::{FromIo, UResult}; -use uucore::format_usage; +use uucore::{format_usage, help_about, help_usage}; -static ABOUT: &str = "Call the link function to create a link named FILE2 to an existing FILE1."; -const USAGE: &str = "{} FILE1 FILE2"; +static ABOUT: &str = help_about!("link.md"); +const USAGE: &str = help_usage!("link.md"); pub mod options { pub static FILES: &str = "FILES"; From 79ee0dd58d970ee64420261af69e7ae79c11d4b2 Mon Sep 17 00:00:00 2001 From: Cheng-Hao Date: Wed, 22 Feb 2023 23:55:51 -0800 Subject: [PATCH 41/94] ln: move help strings to markdown file --- src/uu/ln/ln.md | 21 +++++++++++++++++++++ src/uu/ln/src/ln.rs | 26 +++++++------------------- 2 files changed, 28 insertions(+), 19 deletions(-) create mode 100644 src/uu/ln/ln.md diff --git a/src/uu/ln/ln.md b/src/uu/ln/ln.md new file mode 100644 index 000000000..b2320d6c4 --- /dev/null +++ b/src/uu/ln/ln.md @@ -0,0 +1,21 @@ +# ln + +``` +ln [OPTION]... [-T] TARGET LINK_NAME +ln [OPTION]... TARGET +ln [OPTION]... TARGET... DIRECTORY +ln [OPTION]... -t DIRECTORY TARGET... +``` + +Change file owner and group + +## After Help + +In the 1st form, create a link to TARGET with the name LINK_NAME. +In the 2nd form, create a link to TARGET in the current directory. +In the 3rd and 4th forms, create links to each TARGET in DIRECTORY. +Create hard links by default, symbolic links with --symbolic. +By default, each destination (name of new link) should not already exist. +When creating hard links, each TARGET must exist. Symbolic links +can hold arbitrary text; if later resolved, a relative link is +interpreted in relation to its parent directory. diff --git a/src/uu/ln/src/ln.rs b/src/uu/ln/src/ln.rs index dac7fd556..c583aac1e 100644 --- a/src/uu/ln/src/ln.rs +++ b/src/uu/ln/src/ln.rs @@ -11,7 +11,7 @@ use clap::{crate_version, Arg, ArgAction, Command}; use uucore::display::Quotable; use uucore::error::{FromIo, UError, UResult}; use uucore::fs::{make_path_relative_to, paths_refer_to_same_file}; -use uucore::{format_usage, prompt_yes, show_error}; +use uucore::{format_usage, help_about, help_section, help_usage, prompt_yes, show_error}; use std::borrow::Cow; use std::error::Error; @@ -85,21 +85,9 @@ impl UError for LnError { } } -const ABOUT: &str = "Change file owner and group"; -const USAGE: &str = "\ - {} [OPTION]... [-T] TARGET LINK_NAME - {} [OPTION]... TARGET - {} [OPTION]... TARGET... DIRECTORY - {} [OPTION]... -t DIRECTORY TARGET..."; -const LONG_USAGE: &str = "\ - In the 1st form, create a link to TARGET with the name LINK_NAME.\n\ - In the 2nd form, create a link to TARGET in the current directory.\n\ - In the 3rd and 4th forms, create links to each TARGET in DIRECTORY.\n\ - Create hard links by default, symbolic links with --symbolic.\n\ - By default, each destination (name of new link) should not already exist.\n\ - When creating hard links, each TARGET must exist. Symbolic links\n\ - can hold arbitrary text; if later resolved, a relative link is\n\ - interpreted in relation to its parent directory."; +const ABOUT: &str = help_about!("ln.md"); +const USAGE: &str = help_usage!("ln.md"); +const AFTER_HELP: &str = help_section!("after help", "ln.md"); mod options { pub const FORCE: &str = "force"; @@ -119,13 +107,13 @@ static ARG_FILES: &str = "files"; #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { - let long_usage = format!( + let after_help = format!( "{}\n\n{}", - LONG_USAGE, + AFTER_HELP, backup_control::BACKUP_CONTROL_LONG_HELP ); - let matches = uu_app().after_help(long_usage).try_get_matches_from(args)?; + let matches = uu_app().after_help(after_help).try_get_matches_from(args)?; /* the list of files */ From c3817e535957c311188a622203ce412514342731 Mon Sep 17 00:00:00 2001 From: Koki Ueha Date: Thu, 23 Feb 2023 12:17:00 +0000 Subject: [PATCH 42/94] tee: move help strings to markdown file --- src/uu/tee/src/tee.rs | 9 +++++---- src/uu/tee/tee.md | 11 +++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 src/uu/tee/tee.md diff --git a/src/uu/tee/src/tee.rs b/src/uu/tee/src/tee.rs index 71b5edc5c..96c318afd 100644 --- a/src/uu/tee/src/tee.rs +++ b/src/uu/tee/src/tee.rs @@ -11,15 +11,16 @@ use std::io::{copy, sink, stdin, stdout, Error, ErrorKind, Read, Result, Write}; use std::path::PathBuf; use uucore::display::Quotable; use uucore::error::UResult; -use uucore::{format_usage, show_error}; +use uucore::{format_usage, help_about, help_section, help_usage, show_error}; // spell-checker:ignore nopipe #[cfg(unix)] use uucore::libc; -static ABOUT: &str = "Copy standard input to each FILE, and also to standard output."; -const USAGE: &str = "{} [OPTION]... [FILE]..."; +const ABOUT: &str = help_about!("tee.md"); +const USAGE: &str = help_usage!("tee.md"); +const AFTER_HELP: &str = help_section!("after help", "tee.md"); mod options { pub const APPEND: &str = "append"; @@ -88,7 +89,7 @@ pub fn uu_app() -> Command { .version(crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) - .after_help("If a FILE is -, it refers to a file named - .") + .after_help(AFTER_HELP) .infer_long_args(true) .arg( Arg::new(options::APPEND) diff --git a/src/uu/tee/tee.md b/src/uu/tee/tee.md new file mode 100644 index 000000000..8bf097cec --- /dev/null +++ b/src/uu/tee/tee.md @@ -0,0 +1,11 @@ +# tee + +``` +tee [OPTION]... [FILE]... +``` + +Copy standard input to each FILE, and also to standard output. + +## After Help + +If a FILE is -, it refers to a file named - . From 58dae0a36d6755cd72188d3307a7bfd5fa5bf0c4 Mon Sep 17 00:00:00 2001 From: Koki Ueha Date: Thu, 23 Feb 2023 12:37:01 +0000 Subject: [PATCH 43/94] tail: move help strings to markdown file --- src/uu/tail/src/args.rs | 12 +++--------- src/uu/tail/tail.md | 11 +++++++++++ 2 files changed, 14 insertions(+), 9 deletions(-) create mode 100644 src/uu/tail/tail.md diff --git a/src/uu/tail/src/args.rs b/src/uu/tail/src/args.rs index 0807aa912..6cb77757c 100644 --- a/src/uu/tail/src/args.rs +++ b/src/uu/tail/src/args.rs @@ -17,16 +17,10 @@ use std::ffi::OsString; use std::time::Duration; use uucore::error::{UResult, USimpleError, UUsageError}; use uucore::parse_size::{parse_size, ParseSizeError}; -use uucore::{format_usage, show_warning}; +use uucore::{format_usage, help_about, help_usage, show_warning}; -const ABOUT: &str = "\ - Print the last 10 lines of each FILE to standard output.\n\ - With more than one FILE, precede each with a header giving the file name.\n\ - With no FILE, or when FILE is -, read standard input.\n\ - \n\ - Mandatory arguments to long flags are mandatory for short flags too.\ - "; -const USAGE: &str = "{} [FLAG]... [FILE]..."; +const ABOUT: &str = help_about!("tail.md"); +const USAGE: &str = help_usage!("tail.md"); pub mod options { pub mod verbosity { diff --git a/src/uu/tail/tail.md b/src/uu/tail/tail.md new file mode 100644 index 000000000..fefe7f6ee --- /dev/null +++ b/src/uu/tail/tail.md @@ -0,0 +1,11 @@ +# tail + +``` +tail [FLAG]... [FILE]... +``` + +Print the last 10 lines of each FILE to standard output. +With more than one FILE, precede each with a header giving the file name. +With no FILE, or when FILE is -, read standard input. + +Mandatory arguments to long flags are mandatory for short flags too. From d9a21ff8f0b33a41e8897f3b411e8677b307975e Mon Sep 17 00:00:00 2001 From: David Matos Date: Tue, 21 Feb 2023 19:28:41 +0100 Subject: [PATCH 44/94] chmod: allow verbose and quiet flags to be used more than once --- src/uu/chmod/src/chmod.rs | 1 + tests/by-util/test_chmod.rs | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/src/uu/chmod/src/chmod.rs b/src/uu/chmod/src/chmod.rs index eb8db104f..dee26ab69 100644 --- a/src/uu/chmod/src/chmod.rs +++ b/src/uu/chmod/src/chmod.rs @@ -110,6 +110,7 @@ pub fn uu_app() -> Command { .version(crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) + .args_override_self(true) .infer_long_args(true) .arg( Arg::new(options::CHANGES) diff --git a/tests/by-util/test_chmod.rs b/tests/by-util/test_chmod.rs index a34a3b3c0..b5f8ca5c5 100644 --- a/tests/by-util/test_chmod.rs +++ b/tests/by-util/test_chmod.rs @@ -650,3 +650,24 @@ fn test_chmod_file_symlink_after_non_existing_file() { 0o100764 ); } + +#[test] +fn test_quiet_n_verbose_used_multiple_times() { + let scene = TestScenario::new(util_name!()); + let at = &scene.fixtures; + at.touch("file"); + scene + .ucmd() + .arg("u+x") + .arg("--verbose") + .arg("--verbose") + .arg("file") + .succeeds(); + scene + .ucmd() + .arg("u+x") + .arg("--quiet") + .arg("--quiet") + .arg("file") + .succeeds(); +} From e6027f82d6600a3f65c17f1e29a50a111b6d16cd Mon Sep 17 00:00:00 2001 From: Koutheir Attouchi Date: Thu, 23 Feb 2023 18:41:34 -0500 Subject: [PATCH 45/94] Updated selinux dependency to version 0.4. --- Cargo.lock | 8 ++++---- Cargo.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0e57ea98a..89d67bf81 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1470,9 +1470,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" [[package]] name = "once_cell" -version = "1.16.0" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f0b0d4bf799edbc74508c1e8bf170ff5f41238e5f8225603ca7caaae2b7860" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "onig" @@ -1982,9 +1982,9 @@ checksum = "9c8132065adcfd6e02db789d9285a0deb2f3fcb04002865ab67d5fb103533898" [[package]] name = "selinux" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "966a861c0b329c3078d82b404f7086009487123fd0cc905a9caac55d8b13bee1" +checksum = "a00576725d21b588213fbd4af84cd7e4cc4304e8e9bd6c0f5a1498a3e2ca6a51" dependencies = [ "bitflags", "libc", diff --git a/Cargo.toml b/Cargo.toml index f5a2ceff7..61a461deb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -312,7 +312,7 @@ redox_syscall = "0.2" regex = "1.7.1" rust-ini = "0.18.0" same-file = "1.0.6" -selinux = "0.3" +selinux = "0.4" signal-hook = "0.3.14" smallvec = { version = "1.10", features = ["union"] } strum = "0.24.1" From 3faf64280de61e469f906de39401556ea15a1f59 Mon Sep 17 00:00:00 2001 From: Jeffrey Finkelstein Date: Wed, 7 Dec 2022 17:42:34 -0500 Subject: [PATCH 46/94] dd: move finalize() function to module level Move the `finalize()` function out of the `impl Output` block and up to the module level so that it can be accessed by other module-level functions. --- src/uu/dd/src/dd.rs | 70 ++++++++++++++++++++++----------------------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/src/uu/dd/src/dd.rs b/src/uu/dd/src/dd.rs index 2325e462c..7aa5d4ba5 100644 --- a/src/uu/dd/src/dd.rs +++ b/src/uu/dd/src/dd.rs @@ -632,7 +632,7 @@ impl<'a> Output<'a> { // Optimization: if no blocks are to be written, then don't // bother allocating any buffers. if let Some(Num::Blocks(0) | Num::Bytes(0)) = i.settings.count { - return self.finalize(rstat, wstat, start, &prog_tx, output_thread); + return finalize(&mut self, rstat, wstat, start, &prog_tx, output_thread); }; // Create a common buffer with a capacity of the block size. @@ -673,42 +673,42 @@ impl<'a> Output<'a> { prog_tx.send(prog_update).unwrap_or(()); } } - self.finalize(rstat, wstat, start, &prog_tx, output_thread) + finalize(&mut self, rstat, wstat, start, &prog_tx, output_thread) + } +} + +/// Flush output, print final stats, and join with the progress thread. +fn finalize( + output: &mut Output, + rstat: ReadStat, + wstat: WriteStat, + start: time::Instant, + prog_tx: &mpsc::Sender, + output_thread: thread::JoinHandle, +) -> std::io::Result<()> { + // Flush the output, if configured to do so. + output.sync()?; + + // Truncate the file to the final cursor location. + // + // Calling `set_len()` may result in an error (for example, + // when calling it on `/dev/null`), but we don't want to + // terminate the process when that happens. Instead, we + // suppress the error by calling `Result::ok()`. This matches + // the behavior of GNU `dd` when given the command-line + // argument `of=/dev/null`. + if !output.settings.oconv.notrunc { + output.dst.truncate().ok(); } - /// Flush output, print final stats, and join with the progress thread. - fn finalize( - &mut self, - rstat: ReadStat, - wstat: WriteStat, - start: time::Instant, - prog_tx: &mpsc::Sender, - output_thread: thread::JoinHandle, - ) -> std::io::Result<()> { - // Flush the output, if configured to do so. - self.sync()?; - - // Truncate the file to the final cursor location. - // - // Calling `set_len()` may result in an error (for example, - // when calling it on `/dev/null`), but we don't want to - // terminate the process when that happens. Instead, we - // suppress the error by calling `Result::ok()`. This matches - // the behavior of GNU `dd` when given the command-line - // argument `of=/dev/null`. - if !self.settings.oconv.notrunc { - self.dst.truncate().ok(); - } - - // Print the final read/write statistics. - let prog_update = ProgUpdate::new(rstat, wstat, start.elapsed(), true); - prog_tx.send(prog_update).unwrap_or(()); - // Wait for the output thread to finish - output_thread - .join() - .expect("Failed to join with the output thread."); - Ok(()) - } + // Print the final read/write statistics. + let prog_update = ProgUpdate::new(rstat, wstat, start.elapsed(), true); + prog_tx.send(prog_update).unwrap_or(()); + // Wait for the output thread to finish + output_thread + .join() + .expect("Failed to join with the output thread."); + Ok(()) } #[cfg(any(target_os = "linux", target_os = "android"))] From b9003d19edbfb569c13aaa0f08ac608d53879c30 Mon Sep 17 00:00:00 2001 From: Jeffrey Finkelstein Date: Sun, 27 Nov 2022 13:31:20 -0500 Subject: [PATCH 47/94] dd: move dd_out() function up to module level Move the `dd_out()` function out of the `impl Output` and up to the module level, renaming it to `dd_copy()`. This change aligns it with the pattern set by `std::io::copy()`, which similarly takes an input reader and an output writer. This does not change the behavior of `dd`, just the code organization to make it more closely match the idioms in the Rust standard library. --- src/uu/dd/src/dd.rs | 189 ++++++++++++++++++++++---------------------- 1 file changed, 94 insertions(+), 95 deletions(-) diff --git a/src/uu/dd/src/dd.rs b/src/uu/dd/src/dd.rs index 7aa5d4ba5..eaf89ca55 100644 --- a/src/uu/dd/src/dd.rs +++ b/src/uu/dd/src/dd.rs @@ -138,7 +138,7 @@ impl Read for Source { /// /// Use the [`Input::new_stdin`] or [`Input::new_file`] functions to /// construct a new instance of this struct. Then pass the instance to -/// the [`Output::dd_out`] function to execute the main copy operation +/// the [`dd_copy`] function to execute the main copy operation /// for `dd`. struct Input<'a> { /// The source from which bytes will be read. @@ -449,7 +449,7 @@ impl Write for Dest { /// /// Use the [`Output::new_stdout`] or [`Output::new_file`] functions /// to construct a new instance of this struct. Then use the -/// [`Output::dd_out`] function to execute the main copy operation for +/// [`dd_copy`] function to execute the main copy operation for /// `dd`. struct Output<'a> { /// The destination to which bytes will be written. @@ -579,102 +579,101 @@ impl<'a> Output<'a> { Ok(()) } } +} - /// Copy the given input data to this output, consuming both. - /// - /// This method contains the main loop for the `dd` program. Bytes - /// are read in blocks from `i` and written in blocks to this - /// output. Read/write statistics are reported to stderr as - /// configured by the `status` command-line argument. - /// - /// # Errors - /// - /// If there is a problem reading from the input or writing to - /// this output. - fn dd_out(mut self, mut i: Input) -> std::io::Result<()> { - // The read and write statistics. +/// Copy the given input data to this output, consuming both. +/// +/// This method contains the main loop for the `dd` program. Bytes +/// are read in blocks from `i` and written in blocks to this +/// output. Read/write statistics are reported to stderr as +/// configured by the `status` command-line argument. +/// +/// # Errors +/// +/// If there is a problem reading from the input or writing to +/// this output. +fn dd_copy(mut i: Input, mut o: Output) -> std::io::Result<()> { + // The read and write statistics. + // + // These objects are counters, initialized to zero. After each + // iteration of the main loop, each will be incremented by the + // number of blocks read and written, respectively. + let mut rstat = ReadStat::default(); + let mut wstat = WriteStat::default(); + + // The time at which the main loop starts executing. + // + // When `status=progress` is given on the command-line, the + // `dd` program reports its progress every second or so. Part + // of its report includes the throughput in bytes per second, + // which requires knowing how long the process has been + // running. + let start = time::Instant::now(); + + // A good buffer size for reading. + // + // This is an educated guess about a good buffer size based on + // the input and output block sizes. + let bsize = calc_bsize(i.settings.ibs, o.settings.obs); + + // Start a thread that reports transfer progress. + // + // The `dd` program reports its progress after every block is written, + // at most every 1 second, and only if `status=progress` is given on + // the command-line or a SIGUSR1 signal is received. We + // perform this reporting in a new thread so as not to take + // any CPU time away from the actual reading and writing of + // data. We send a `ProgUpdate` from the transmitter `prog_tx` + // to the receives `rx`, and the receiver prints the transfer + // information. + let (prog_tx, rx) = mpsc::channel(); + let output_thread = thread::spawn(gen_prog_updater(rx, i.settings.status)); + let mut progress_as_secs = 0; + + // Optimization: if no blocks are to be written, then don't + // bother allocating any buffers. + if let Some(Num::Blocks(0) | Num::Bytes(0)) = i.settings.count { + return finalize(&mut o, rstat, wstat, start, &prog_tx, output_thread); + }; + + // Create a common buffer with a capacity of the block size. + // This is the max size needed. + let mut buf = vec![BUF_INIT_BYTE; bsize]; + + // The main read/write loop. + // + // Each iteration reads blocks from the input and writes + // blocks to this output. Read/write statistics are updated on + // each iteration and cumulative statistics are reported to + // the progress reporting thread. + while below_count_limit(&i.settings.count, &rstat, &wstat) { + // Read a block from the input then write the block to the output. // - // These objects are counters, initialized to zero. After each - // iteration of the main loop, each will be incremented by the - // number of blocks read and written, respectively. - let mut rstat = ReadStat::default(); - let mut wstat = WriteStat::default(); - - // The time at which the main loop starts executing. - // - // When `status=progress` is given on the command-line, the - // `dd` program reports its progress every second or so. Part - // of its report includes the throughput in bytes per second, - // which requires knowing how long the process has been - // running. - let start = time::Instant::now(); - - // A good buffer size for reading. - // - // This is an educated guess about a good buffer size based on - // the input and output block sizes. - let bsize = calc_bsize(i.settings.ibs, self.settings.obs); - - // Start a thread that reports transfer progress. - // - // The `dd` program reports its progress after every block is written, - // at most every 1 second, and only if `status=progress` is given on - // the command-line or a SIGUSR1 signal is received. We - // perform this reporting in a new thread so as not to take - // any CPU time away from the actual reading and writing of - // data. We send a `ProgUpdate` from the transmitter `prog_tx` - // to the receives `rx`, and the receiver prints the transfer - // information. - let (prog_tx, rx) = mpsc::channel(); - let output_thread = thread::spawn(gen_prog_updater(rx, i.settings.status)); - let mut progress_as_secs = 0; - - // Optimization: if no blocks are to be written, then don't - // bother allocating any buffers. - if let Some(Num::Blocks(0) | Num::Bytes(0)) = i.settings.count { - return finalize(&mut self, rstat, wstat, start, &prog_tx, output_thread); - }; - - // Create a common buffer with a capacity of the block size. - // This is the max size needed. - let mut buf = vec![BUF_INIT_BYTE; bsize]; - - // The main read/write loop. - // - // Each iteration reads blocks from the input and writes - // blocks to this output. Read/write statistics are updated on - // each iteration and cumulative statistics are reported to - // the progress reporting thread. - while below_count_limit(&i.settings.count, &rstat, &wstat) { - // Read a block from the input then write the block to the output. - // - // As an optimization, make an educated guess about the - // best buffer size for reading based on the number of - // blocks already read and the number of blocks remaining. - let loop_bsize = - calc_loop_bsize(&i.settings.count, &rstat, &wstat, i.settings.ibs, bsize); - let rstat_update = read_helper(&mut i, &mut buf, loop_bsize)?; - if rstat_update.is_empty() { - break; - } - let wstat_update = self.write_blocks(&buf)?; - - // Update the read/write stats and inform the progress thread once per second. - // - // If the receiver is disconnected, `send()` returns an - // error. Since it is just reporting progress and is not - // crucial to the operation of `dd`, let's just ignore the - // error. - rstat += rstat_update; - wstat += wstat_update; - let prog_update = ProgUpdate::new(rstat, wstat, start.elapsed(), false); - if prog_update.duration.as_secs() >= progress_as_secs { - progress_as_secs = prog_update.duration.as_secs() + 1; - prog_tx.send(prog_update).unwrap_or(()); - } + // As an optimization, make an educated guess about the + // best buffer size for reading based on the number of + // blocks already read and the number of blocks remaining. + let loop_bsize = calc_loop_bsize(&i.settings.count, &rstat, &wstat, i.settings.ibs, bsize); + let rstat_update = read_helper(&mut i, &mut buf, loop_bsize)?; + if rstat_update.is_empty() { + break; + } + let wstat_update = o.write_blocks(&buf)?; + + // Update the read/write stats and inform the progress thread once per second. + // + // If the receiver is disconnected, `send()` returns an + // error. Since it is just reporting progress and is not + // crucial to the operation of `dd`, let's just ignore the + // error. + rstat += rstat_update; + wstat += wstat_update; + let prog_update = ProgUpdate::new(rstat, wstat, start.elapsed(), false); + if prog_update.duration.as_secs() >= progress_as_secs { + progress_as_secs = prog_update.duration.as_secs() + 1; + prog_tx.send(prog_update).unwrap_or(()); } - finalize(&mut self, rstat, wstat, start, &prog_tx, output_thread) } + finalize(&mut o, rstat, wstat, start, &prog_tx, output_thread) } /// Flush output, print final stats, and join with the progress thread. @@ -925,7 +924,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { } None => Output::new_stdout(&settings)?, }; - o.dd_out(i).map_err_context(|| "IO error".to_string()) + dd_copy(i, o).map_err_context(|| "IO error".to_string()) } pub fn uu_app() -> Command { From 93e1760bd33c90a3560dff98bd953b8b8b2769e4 Mon Sep 17 00:00:00 2001 From: Chen Chi <114895836+ChenChiii@users.noreply.github.com> Date: Thu, 23 Feb 2023 22:49:04 -0800 Subject: [PATCH 48/94] realpath: move help strings to markdown file --- src/uu/realpath/realpath.md | 7 +++++++ src/uu/realpath/src/realpath.rs | 5 +++-- 2 files changed, 10 insertions(+), 2 deletions(-) create mode 100644 src/uu/realpath/realpath.md diff --git a/src/uu/realpath/realpath.md b/src/uu/realpath/realpath.md new file mode 100644 index 000000000..69fb39407 --- /dev/null +++ b/src/uu/realpath/realpath.md @@ -0,0 +1,7 @@ +# realpath + +``` +realpath [OPTION]... FILE... +``` + +Print the resolved path \ No newline at end of file diff --git a/src/uu/realpath/src/realpath.rs b/src/uu/realpath/src/realpath.rs index 1a6d42790..cb7a09a41 100644 --- a/src/uu/realpath/src/realpath.rs +++ b/src/uu/realpath/src/realpath.rs @@ -20,11 +20,12 @@ use uucore::{ error::{FromIo, UResult}, format_usage, fs::{canonicalize, MissingHandling, ResolveMode}, + help_about, help_usage, }; use uucore::{error::UClapError, show, show_if_err}; -static ABOUT: &str = "Print the resolved path"; -const USAGE: &str = "{} [OPTION]... FILE..."; +static ABOUT: &str = help_about!("realpath.md"); +const USAGE: &str = help_usage!("realpath.md"); static OPT_QUIET: &str = "quiet"; static OPT_STRIP: &str = "strip"; From 3640a90d9fe50fac9e68a03eedc06ccfbe398fd6 Mon Sep 17 00:00:00 2001 From: Cheng-Hao Date: Thu, 23 Feb 2023 23:17:08 -0800 Subject: [PATCH 49/94] fmt: move help strings to markdown file --- src/uu/fmt/fmt.md | 7 +++++++ src/uu/fmt/src/fmt.rs | 6 +++--- 2 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 src/uu/fmt/fmt.md diff --git a/src/uu/fmt/fmt.md b/src/uu/fmt/fmt.md new file mode 100644 index 000000000..5cf4fa6e6 --- /dev/null +++ b/src/uu/fmt/fmt.md @@ -0,0 +1,7 @@ +# fmt + +``` +fmt [OPTION]... [FILE]... +``` + +Reformat paragraphs from input files (or stdin) to stdout. diff --git a/src/uu/fmt/src/fmt.rs b/src/uu/fmt/src/fmt.rs index a97f64945..bd03b5497 100644 --- a/src/uu/fmt/src/fmt.rs +++ b/src/uu/fmt/src/fmt.rs @@ -14,7 +14,7 @@ use std::io::{stdin, stdout, Write}; use std::io::{BufReader, BufWriter, Read}; use uucore::display::Quotable; use uucore::error::{FromIo, UResult, USimpleError}; -use uucore::{format_usage, show_warning}; +use uucore::{format_usage, help_about, help_usage, show_warning}; use self::linebreak::break_lines; use self::parasplit::ParagraphStream; @@ -22,8 +22,8 @@ use self::parasplit::ParagraphStream; mod linebreak; mod parasplit; -static ABOUT: &str = "Reformat paragraphs from input files (or stdin) to stdout."; -const USAGE: &str = "{} [OPTION]... [FILE]..."; +static ABOUT: &str = help_about!("fmt.md"); +const USAGE: &str = help_usage!("fmt.md"); static MAX_WIDTH: usize = 2500; static OPT_CROWN_MARGIN: &str = "crown-margin"; From f3ccb977639e250d176d1c59e919f0e9303e19c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Pawe=C5=82=20Gajc?= Date: Fri, 24 Feb 2023 15:55:44 +0100 Subject: [PATCH 50/94] Add OpenMandriva Lx to the documentation --- docs/src/installation.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/src/installation.md b/docs/src/installation.md index 063935ed6..6baea3441 100644 --- a/docs/src/installation.md +++ b/docs/src/installation.md @@ -79,6 +79,13 @@ pamac install uutils-coreutils nix-env -iA nixos.uutils-coreutils ``` +### OpenMandriva Lx +[![openmandriva cooker package](https://repology.org/badge/version-for-repo/openmandriva_cooker/uutils-coreutils.svg)](https://repology.org/project/uutils-coreutils/versions) + +```bash +dnf install uutils-coreutils +``` + ### Ubuntu [![Ubuntu package](https://repology.org/badge/version-for-repo/ubuntu_23_04/uutils-coreutils.svg)](https://packages.ubuntu.com/source/lunar/rust-coreutils) From 9a8b96d9118722392dd83e233e4b1a8cfa18fd1d Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Fri, 24 Feb 2023 16:02:59 +0100 Subject: [PATCH 51/94] Add openmandriva to the ignore list --- docs/src/installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/installation.md b/docs/src/installation.md index 6baea3441..d08376578 100644 --- a/docs/src/installation.md +++ b/docs/src/installation.md @@ -1,4 +1,4 @@ - + # Installation From ac39a169a12588cc4b0ea7e24b02f8826a970e7a Mon Sep 17 00:00:00 2001 From: Koki Ueha Date: Sat, 25 Feb 2023 10:27:03 +0000 Subject: [PATCH 52/94] df: move help strings to markdown file --- src/uu/df/df.md | 18 ++++++++++++++++++ src/uu/df/src/df.rs | 17 +++++------------ 2 files changed, 23 insertions(+), 12 deletions(-) create mode 100644 src/uu/df/df.md diff --git a/src/uu/df/df.md b/src/uu/df/df.md new file mode 100644 index 000000000..1a192f8fd --- /dev/null +++ b/src/uu/df/df.md @@ -0,0 +1,18 @@ +# df + +``` +df [OPTION]... [FILE]... +``` + +Show information about the file system on which each FILE resides, +or all file systems by default. + +## After Help + +Display values are in units of the first available SIZE from --block-size, +and the DF_BLOCK_SIZE, BLOCK_SIZE and BLOCKSIZE environment variables. +Otherwise, units default to 1024 bytes (or 512 if POSIXLY_CORRECT is set). + +SIZE is an integer and optional unit (example: 10M is 10*1024*1024). +Units are K, M, G, T, P, E, Z, Y (powers of 1024) or KB, MB,... (powers +of 1000). diff --git a/src/uu/df/src/df.rs b/src/uu/df/src/df.rs index c501b0bab..685ebe58a 100644 --- a/src/uu/df/src/df.rs +++ b/src/uu/df/src/df.rs @@ -19,7 +19,7 @@ use uucore::error::FromIo; use uucore::error::{UError, UResult, USimpleError}; use uucore::fsext::{read_fs_list, MountInfo}; use uucore::parse_size::ParseSizeError; -use uucore::{format_usage, show}; +use uucore::{format_usage, help_about, help_section, help_usage, show}; use clap::{crate_version, parser::ValueSource, Arg, ArgAction, ArgMatches, Command}; @@ -33,16 +33,9 @@ use crate::columns::{Column, ColumnError}; use crate::filesystem::Filesystem; use crate::table::Table; -static ABOUT: &str = "Show information about the file system on which each FILE resides,\n\ - or all file systems by default."; -const USAGE: &str = "{} [OPTION]... [FILE]..."; -const LONG_HELP: &str = "Display values are in units of the first available SIZE from --block-size, -and the DF_BLOCK_SIZE, BLOCK_SIZE and BLOCKSIZE environment variables. -Otherwise, units default to 1024 bytes (or 512 if POSIXLY_CORRECT is set). - -SIZE is an integer and optional unit (example: 10M is 10*1024*1024). -Units are K, M, G, T, P, E, Z, Y (powers of 1024) or KB, MB,... (powers -of 1000)."; +const ABOUT: &str = help_about!("df.md"); +const USAGE: &str = help_usage!("df.md"); +const AFTER_HELP: &str = help_section!("after help", "df.md"); static OPT_HELP: &str = "help"; static OPT_ALL: &str = "all"; @@ -487,7 +480,7 @@ pub fn uu_app() -> Command { .version(crate_version!()) .about(ABOUT) .override_usage(format_usage(USAGE)) - .after_help(LONG_HELP) + .after_help(AFTER_HELP) .infer_long_args(true) .disable_help_flag(true) .arg( From ba4a9a05a146b96513d3416d2a5ff4446719397f Mon Sep 17 00:00:00 2001 From: Koki Ueha Date: Sat, 25 Feb 2023 10:46:19 +0000 Subject: [PATCH 53/94] dircolors: move help strings to markdown file --- src/uu/dircolors/dircolors.md | 13 +++++++++++++ src/uu/dircolors/src/dircolors.rs | 13 +++++-------- 2 files changed, 18 insertions(+), 8 deletions(-) create mode 100644 src/uu/dircolors/dircolors.md diff --git a/src/uu/dircolors/dircolors.md b/src/uu/dircolors/dircolors.md new file mode 100644 index 000000000..959b8fd19 --- /dev/null +++ b/src/uu/dircolors/dircolors.md @@ -0,0 +1,13 @@ +# dircolors + +``` +dircolors [OPTION]... [FILE] +``` + +Output commands to set the LS_COLORS environment variable. + +## After Help + +If FILE is specified, read it to determine which colors to use for which +file types and extensions. Otherwise, a precompiled database is used. +For details on the format of these files, run 'dircolors --print-database' diff --git a/src/uu/dircolors/src/dircolors.rs b/src/uu/dircolors/src/dircolors.rs index 852d422a0..abab966af 100644 --- a/src/uu/dircolors/src/dircolors.rs +++ b/src/uu/dircolors/src/dircolors.rs @@ -16,6 +16,7 @@ use std::io::{BufRead, BufReader}; use clap::{crate_version, Arg, ArgAction, Command}; use uucore::display::Quotable; use uucore::error::{UResult, USimpleError, UUsageError}; +use uucore::{help_about, help_section, help_usage}; mod options { pub const BOURNE_SHELL: &str = "bourne-shell"; @@ -25,13 +26,9 @@ mod options { pub const FILE: &str = "FILE"; } -static USAGE: &str = "{} [OPTION]... [FILE]"; -static ABOUT: &str = "Output commands to set the LS_COLORS environment variable."; -static LONG_HELP: &str = " - If FILE is specified, read it to determine which colors to use for which - file types and extensions. Otherwise, a precompiled database is used. - For details on the format of these files, run 'dircolors --print-database' -"; +const USAGE: &str = help_usage!("dircolors.md"); +const ABOUT: &str = help_about!("dircolors.md"); +const AFTER_HELP: &str = help_section!("after help", "dircolors.md"); mod colors; use self::colors::INTERNAL_DB; @@ -170,7 +167,7 @@ pub fn uu_app() -> Command { Command::new(uucore::util_name()) .version(crate_version!()) .about(ABOUT) - .after_help(LONG_HELP) + .after_help(AFTER_HELP) .override_usage(format_usage(USAGE)) .infer_long_args(true) .arg( From 351be7f47888d727e63cfc1afccf66e44f7fefac Mon Sep 17 00:00:00 2001 From: Zan Baldwin Date: Sun, 26 Feb 2023 13:13:51 +0100 Subject: [PATCH 54/94] uptime: correctly calculate boot-time --- src/uu/uptime/src/uptime.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/uu/uptime/src/uptime.rs b/src/uu/uptime/src/uptime.rs index d13445f60..3a561f419 100644 --- a/src/uu/uptime/src/uptime.rs +++ b/src/uu/uptime/src/uptime.rs @@ -115,8 +115,8 @@ fn process_utmpx() -> (Option, usize) { USER_PROCESS => nusers += 1, BOOT_TIME => { let dt = line.login_time(); - if dt.second() > 0 { - boot_time = Some(dt.second() as time_t); + if dt.unix_timestamp() > 0 { + boot_time = Some(dt.unix_timestamp() as time_t); } } _ => continue, From 9e217ab30df70de834842f02a9705f087e4c3931 Mon Sep 17 00:00:00 2001 From: Terts Diepraam Date: Sun, 26 Feb 2023 16:42:14 +0100 Subject: [PATCH 55/94] utils: create a script to explore binary sizes --- util/size-experiment.py | 90 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 util/size-experiment.py diff --git a/util/size-experiment.py b/util/size-experiment.py new file mode 100644 index 000000000..8acacf9d1 --- /dev/null +++ b/util/size-experiment.py @@ -0,0 +1,90 @@ +import subprocess +from itertools import product +import shutil +import os +from collections import defaultdict +from pprint import pprint + +# Set to false if you've only changed the analysis and do not want to recompile +# the binaries. +RECOMPILE = True + +STRIP_VALS = ["none", "debuginfo", "symbols"] +PANIC_VALS = ["unwind", "abort"] +OPT_LEVEL_VALS = [3, "s", "z"] +LTO_VALS = ["off", "thin", "fat"] + + +def config(name, val): + return ["--config", f"profile.release.{name}={val!r}"] + + +sizes = {} + +for (strip, panic, opt, lto) in product( + STRIP_VALS, PANIC_VALS, OPT_LEVEL_VALS, LTO_VALS +): + if RECOMPILE: + cmd = [ + "cargo", + "build", + "--release", + "--features=unix", + *config("strip", strip), + *config("panic", panic), + *config("opt-level", opt), + *config("lto", lto), + ] + print("RUN:", " ".join(cmd)) + res = subprocess.call(cmd) + + shutil.copyfile( + "target/release/coreutils", + "-".join(["coreutils", strip, panic, str(opt), lto]), + ) + print(res) + + sizes[(strip, panic, opt, lto)] = os.path.getsize( + "-".join(["coreutils", strip, panic, str(opt), lto]) + ) + +changes_absolute = defaultdict(list) +changes_percent = defaultdict(list) + + +def with_val_at_idx(val, idx, other): + other = list(other) + other.insert(idx, val) + return tuple(other) + + +def collect_diff(idx, name): + all_params = [STRIP_VALS, PANIC_VALS, OPT_LEVEL_VALS, LTO_VALS] + vals = all_params.pop(idx) + for other in product(*all_params): + baseline = sizes[with_val_at_idx(vals[0], idx, other)] + for val in vals[1:]: + changes = sizes[with_val_at_idx(val, idx, other)] - baseline + changes_absolute[f"{name}={val}"].append(changes) + changes_percent[f"{name}={val}"].append(changes / baseline * 100) + + +collect_diff(0, "strip") +collect_diff(1, "panic") +collect_diff(2, "opt-level") +collect_diff(3, "lto") + + +def analyze(l): + return f"MIN: {float(min(l)):.3}, AVG: {float(sum(l)/len(l)):.3}, MAX: {float(max(l)):.3}" + + +print("Absolute changes") +pprint({k: analyze(v) for k, v in changes_absolute.items()}) +print() +print("Percent changes") +pprint({k: analyze(v) for k, v in changes_percent.items()}) + +print() +print(changes_percent["opt-level=s"]) +print(changes_percent["opt-level=z"]) From f06d46ef293aa60dbe5a279aa720e8379fffb347 Mon Sep 17 00:00:00 2001 From: Koki Ueha Date: Mon, 27 Feb 2023 14:13:06 +0000 Subject: [PATCH 56/94] dirname: move help strings to markdown file --- src/uu/dirname/dirname.md | 12 ++++++++++++ src/uu/dirname/src/dirname.rs | 12 +++++------- 2 files changed, 17 insertions(+), 7 deletions(-) create mode 100644 src/uu/dirname/dirname.md diff --git a/src/uu/dirname/dirname.md b/src/uu/dirname/dirname.md new file mode 100644 index 000000000..f08c1c4c9 --- /dev/null +++ b/src/uu/dirname/dirname.md @@ -0,0 +1,12 @@ +# dirname + +``` +dirname [OPTION] NAME... +``` + +Strip last component from file name + +## After Help + +Output each NAME with its last non-slash component and trailing slashes +removed; if NAME contains no /'s, output '.' (meaning the current directory). diff --git a/src/uu/dirname/src/dirname.rs b/src/uu/dirname/src/dirname.rs index bebdd4d36..687aaa550 100644 --- a/src/uu/dirname/src/dirname.rs +++ b/src/uu/dirname/src/dirname.rs @@ -9,13 +9,11 @@ use clap::{crate_version, Arg, ArgAction, Command}; use std::path::Path; use uucore::display::print_verbatim; use uucore::error::{UResult, UUsageError}; -use uucore::format_usage; +use uucore::{format_usage, help_about, help_section, help_usage}; -const ABOUT: &str = "Strip last component from file name"; -const USAGE: &str = "{} [OPTION] NAME..."; -const LONG_USAGE: &str = "\ - Output each NAME with its last non-slash component and trailing slashes \n\ - removed; if NAME contains no /'s, output '.' (meaning the current directory)."; +const ABOUT: &str = help_about!("dirname.md"); +const USAGE: &str = help_usage!("dirname.md"); +const AFTER_HELP: &str = help_section!("after help", "dirname.md"); mod options { pub const ZERO: &str = "zero"; @@ -26,7 +24,7 @@ mod options { pub fn uumain(args: impl uucore::Args) -> UResult<()> { let args = args.collect_lossy(); - let matches = uu_app().after_help(LONG_USAGE).try_get_matches_from(args)?; + let matches = uu_app().after_help(AFTER_HELP).try_get_matches_from(args)?; let separator = if matches.get_flag(options::ZERO) { "\0" From 028b97098dbe759c4613a7649c1ae94b6a718064 Mon Sep 17 00:00:00 2001 From: Daniel Hofstetter Date: Mon, 27 Feb 2023 10:11:27 +0100 Subject: [PATCH 57/94] comm: move help strings to markdown file --- src/uu/comm/comm.md | 13 +++++++++++++ src/uu/comm/src/comm.rs | 8 +++----- 2 files changed, 16 insertions(+), 5 deletions(-) create mode 100644 src/uu/comm/comm.md diff --git a/src/uu/comm/comm.md b/src/uu/comm/comm.md new file mode 100644 index 000000000..91f4467d4 --- /dev/null +++ b/src/uu/comm/comm.md @@ -0,0 +1,13 @@ +# comm + +``` +comm [OPTION]... FILE1 FILE2 +``` + +Compare two sorted files line by line. + +When FILE1 or FILE2 (not both) is -, read standard input. + +With no options, produce three-column output. Column one contains +lines unique to FILE1, column two contains lines unique to FILE2, +and column three contains lines common to both files. diff --git a/src/uu/comm/src/comm.rs b/src/uu/comm/src/comm.rs index c14717df2..3bc45b6a5 100644 --- a/src/uu/comm/src/comm.rs +++ b/src/uu/comm/src/comm.rs @@ -13,13 +13,12 @@ use std::io::{self, stdin, BufRead, BufReader, Stdin}; use std::path::Path; use uucore::error::FromIo; use uucore::error::UResult; -use uucore::format_usage; +use uucore::{format_usage, help_about, help_usage}; use clap::{crate_version, Arg, ArgAction, ArgMatches, Command}; -static ABOUT: &str = "Compare two sorted files line by line"; -static LONG_HELP: &str = ""; -const USAGE: &str = "{} [OPTION]... FILE1 FILE2"; +const ABOUT: &str = help_about!("comm.md"); +const USAGE: &str = help_usage!("comm.md"); mod options { pub const COLUMN_1: &str = "1"; @@ -160,7 +159,6 @@ pub fn uu_app() -> Command { Command::new(uucore::util_name()) .version(crate_version!()) .about(ABOUT) - .after_help(LONG_HELP) .override_usage(format_usage(USAGE)) .infer_long_args(true) .arg( From b83c30b12e88a8549e2f9367d80664eb73090187 Mon Sep 17 00:00:00 2001 From: Benjamin Bara Date: Sat, 11 Feb 2023 21:14:12 +0100 Subject: [PATCH 58/94] tail: improve GNU compatibility --- src/uu/tail/src/args.rs | 23 +++- src/uu/tail/src/parse.rs | 171 +++++++++++---------------- tests/by-util/test_tail.rs | 236 +++++++++++++++++++++++++++++++++++++ 3 files changed, 325 insertions(+), 105 deletions(-) diff --git a/src/uu/tail/src/args.rs b/src/uu/tail/src/args.rs index 6cb77757c..aa4b1c867 100644 --- a/src/uu/tail/src/args.rs +++ b/src/uu/tail/src/args.rs @@ -64,7 +64,12 @@ impl FilterMode { let mode = if let Some(arg) = matches.get_one::(options::BYTES) { match parse_num(arg) { Ok(signum) => Self::Bytes(signum), - Err(e) => return Err(UUsageError::new(1, format!("invalid number of bytes: {e}"))), + Err(e) => { + return Err(USimpleError::new( + 1, + format!("invalid number of bytes: {e}"), + )) + } } } else if let Some(arg) = matches.get_one::(options::LINES) { match parse_num(arg) { @@ -72,7 +77,12 @@ impl FilterMode { let delimiter = if zero_term { 0 } else { b'\n' }; Self::Lines(signum, delimiter) } - Err(e) => return Err(UUsageError::new(1, format!("invalid number of lines: {e}"))), + Err(e) => { + return Err(USimpleError::new( + 1, + format!("invalid number of lines: {e}"), + )) + } } } else if zero_term { Self::default_zero() @@ -307,14 +317,19 @@ pub fn arg_iterate<'a>( if let Some(s) = second.to_str() { match parse::parse_obsolete(s) { Some(Ok(iter)) => Ok(Box::new(vec![first].into_iter().chain(iter).chain(args))), - Some(Err(e)) => Err(UUsageError::new( + Some(Err(e)) => Err(USimpleError::new( 1, match e { - parse::ParseError::Syntax => format!("bad argument format: {}", s.quote()), parse::ParseError::Overflow => format!( "invalid argument: {} Value too large for defined datatype", s.quote() ), + parse::ParseError::Context => { + format!( + "option used in invalid context -- {}", + s.chars().nth(1).unwrap_or_default() + ) + } }, )), None => Ok(Box::new(vec![first, second].into_iter().chain(args))), diff --git a/src/uu/tail/src/parse.rs b/src/uu/tail/src/parse.rs index 2129d8e29..15c770bc5 100644 --- a/src/uu/tail/src/parse.rs +++ b/src/uu/tail/src/parse.rs @@ -7,92 +7,67 @@ use std::ffi::OsString; #[derive(PartialEq, Eq, Debug)] pub enum ParseError { - Syntax, Overflow, + Context, } /// Parses obsolete syntax -/// tail -NUM\[kmzv\] // spell-checker:disable-line +/// tail -\[NUM\]\[bl\]\[f\] and tail +\[NUM\]\[bcl\]\[f\] // spell-checker:disable-line pub fn parse_obsolete(src: &str) -> Option, ParseError>> { - let mut chars = src.char_indices(); - if let Some((_, '-')) = chars.next() { - let mut num_end = 0usize; - let mut has_num = false; - let mut last_char = 0 as char; - for (n, c) in &mut chars { - if c.is_ascii_digit() { - has_num = true; - num_end = n; - } else { - last_char = c; - break; - } - } - if has_num { - match src[1..=num_end].parse::() { - Ok(num) => { - let mut quiet = false; - let mut verbose = false; - let mut zero_terminated = false; - let mut multiplier = None; - let mut c = last_char; - loop { - // not that here, we only match lower case 'k', 'c', and 'm' - match c { - // we want to preserve order - // this also saves us 1 heap allocation - 'q' => { - quiet = true; - verbose = false; - } - 'v' => { - verbose = true; - quiet = false; - } - 'z' => zero_terminated = true, - 'c' => multiplier = Some(1), - 'b' => multiplier = Some(512), - 'k' => multiplier = Some(1024), - 'm' => multiplier = Some(1024 * 1024), - '\0' => {} - _ => return Some(Err(ParseError::Syntax)), - } - if let Some((_, next)) = chars.next() { - c = next; - } else { - break; - } - } - let mut options = Vec::new(); - if quiet { - options.push(OsString::from("-q")); - } - if verbose { - options.push(OsString::from("-v")); - } - if zero_terminated { - options.push(OsString::from("-z")); - } - if let Some(n) = multiplier { - options.push(OsString::from("-c")); - let num = match num.checked_mul(n) { - Some(n) => n, - None => return Some(Err(ParseError::Overflow)), - }; - options.push(OsString::from(format!("{num}"))); - } else { - options.push(OsString::from("-n")); - options.push(OsString::from(format!("{num}"))); - } - Some(Ok(options.into_iter())) - } - Err(_) => Some(Err(ParseError::Overflow)), - } + let mut chars = src.chars(); + let sign = chars.next()?; + if sign != '+' && sign != '-' { + return None; + } + + let numbers: String = chars.clone().take_while(|&c| c.is_ascii_digit()).collect(); + let has_num = !numbers.is_empty(); + let num: usize = if has_num { + if let Ok(num) = numbers.parse() { + num } else { - None + return Some(Err(ParseError::Overflow)); } } else { - None + 10 + }; + + let mut follow = false; + let mut mode = None; + let mut first_char = true; + for char in chars.skip_while(|&c| c.is_ascii_digit()) { + if sign == '-' && char == 'c' && !has_num { + // special case: -c should be handled by clap (is ambiguous) + return None; + } else if char == 'f' { + follow = true; + } else if first_char && (char == 'b' || char == 'c' || char == 'l') { + mode = Some(char); + } else if has_num && sign == '-' { + return Some(Err(ParseError::Context)); + } else { + return None; + } + first_char = false; } + + let mut options = Vec::new(); + if follow { + options.push(OsString::from("-f")); + } + let mode = mode.unwrap_or('l'); + if mode == 'b' || mode == 'c' { + options.push(OsString::from("-c")); + let n = if mode == 'b' { 512 } else { 1 }; + let num = match num.checked_mul(n) { + Some(n) => n, + None => return Some(Err(ParseError::Overflow)), + }; + options.push(OsString::from(format!("{sign}{num}"))); + } else { + options.push(OsString::from("-n")); + options.push(OsString::from(format!("{sign}{num}"))); + } + Some(Ok(options.into_iter())) } #[cfg(test)] @@ -113,40 +88,35 @@ mod tests { } #[test] fn test_parse_numbers_obsolete() { - assert_eq!(obsolete("-5"), obsolete_result(&["-n", "5"])); - assert_eq!(obsolete("-100"), obsolete_result(&["-n", "100"])); - assert_eq!(obsolete("-5m"), obsolete_result(&["-c", "5242880"])); - assert_eq!(obsolete("-1k"), obsolete_result(&["-c", "1024"])); - assert_eq!(obsolete("-2b"), obsolete_result(&["-c", "1024"])); - assert_eq!(obsolete("-1mmk"), obsolete_result(&["-c", "1024"])); - assert_eq!(obsolete("-1vz"), obsolete_result(&["-v", "-z", "-n", "1"])); - assert_eq!( - obsolete("-1vzqvq"), // spell-checker:disable-line - obsolete_result(&["-q", "-z", "-n", "1"]) - ); - assert_eq!(obsolete("-1vzc"), obsolete_result(&["-v", "-z", "-c", "1"])); - assert_eq!( - obsolete("-105kzm"), - obsolete_result(&["-z", "-c", "110100480"]) - ); + assert_eq!(obsolete("+2c"), obsolete_result(&["-c", "+2"])); + assert_eq!(obsolete("-5"), obsolete_result(&["-n", "-5"])); + assert_eq!(obsolete("-100"), obsolete_result(&["-n", "-100"])); + assert_eq!(obsolete("-2b"), obsolete_result(&["-c", "-1024"])); } #[test] fn test_parse_errors_obsolete() { - assert_eq!(obsolete("-5n"), Some(Err(ParseError::Syntax))); - assert_eq!(obsolete("-5c5"), Some(Err(ParseError::Syntax))); + assert_eq!(obsolete("-5n"), Some(Err(ParseError::Context))); + assert_eq!(obsolete("-5c5"), Some(Err(ParseError::Context))); + assert_eq!(obsolete("-1vzc"), Some(Err(ParseError::Context))); + assert_eq!(obsolete("-5m"), Some(Err(ParseError::Context))); + assert_eq!(obsolete("-1k"), Some(Err(ParseError::Context))); + assert_eq!(obsolete("-1mmk"), Some(Err(ParseError::Context))); + assert_eq!(obsolete("-105kzm"), Some(Err(ParseError::Context))); + assert_eq!(obsolete("-1vz"), Some(Err(ParseError::Context))); + assert_eq!( + obsolete("-1vzqvq"), // spell-checker:disable-line + Some(Err(ParseError::Context)) + ); } #[test] fn test_parse_obsolete_no_match() { assert_eq!(obsolete("-k"), None); assert_eq!(obsolete("asd"), None); + assert_eq!(obsolete("-cc"), None); } #[test] #[cfg(target_pointer_width = "64")] fn test_parse_obsolete_overflow_x64() { - assert_eq!( - obsolete("-1000000000000000m"), - Some(Err(ParseError::Overflow)) - ); assert_eq!( obsolete("-10000000000000000000000"), Some(Err(ParseError::Overflow)) @@ -156,6 +126,5 @@ mod tests { #[cfg(target_pointer_width = "32")] fn test_parse_obsolete_overflow_x32() { assert_eq!(obsolete("-42949672960"), Some(Err(ParseError::Overflow))); - assert_eq!(obsolete("-42949672k"), Some(Err(ParseError::Overflow))); } } diff --git a/tests/by-util/test_tail.rs b/tests/by-util/test_tail.rs index b2ec0e7bd..4644cbc02 100644 --- a/tests/by-util/test_tail.rs +++ b/tests/by-util/test_tail.rs @@ -4475,3 +4475,239 @@ fn test_args_sleep_interval_when_illegal_argument_then_usage_error(#[case] sleep .usage_error(format!("invalid number of seconds: '{sleep_interval}'")) .code_is(1); } + +#[test] +fn test_gnu_args_plus_c() { + let scene = TestScenario::new(util_name!()); + + // obs-plus-c1 + scene + .ucmd() + .arg("+2c") + .pipe_in("abcd") + .succeeds() + .stdout_only("bcd"); + // obs-plus-c2 + scene + .ucmd() + .arg("+8c") + .pipe_in("abcd") + .succeeds() + .stdout_only(""); + // obs-plus-x1: same as +10c + scene + .ucmd() + .arg("+c") + .pipe_in(format!("x{}z", "y".repeat(10))) + .succeeds() + .stdout_only("yyz"); +} + +#[test] +fn test_gnu_args_c() { + let scene = TestScenario::new(util_name!()); + + // obs-c3 + scene + .ucmd() + .arg("-1c") + .pipe_in("abcd") + .succeeds() + .stdout_only("d"); + // obs-c4 + scene + .ucmd() + .arg("-9c") + .pipe_in("abcd") + .succeeds() + .stdout_only("abcd"); + // obs-c5 + scene + .ucmd() + .arg("-12c") + .pipe_in(format!("x{}z", "y".repeat(12))) + .succeeds() + .stdout_only(&format!("{}z", "y".repeat(11))); +} + +#[test] +fn test_gnu_args_l() { + let scene = TestScenario::new(util_name!()); + + // obs-l1 + scene + .ucmd() + .arg("-1l") + .pipe_in("x") + .succeeds() + .stdout_only("x"); + // obs-l2 + scene + .ucmd() + .arg("-1l") + .pipe_in("x\ny\n") + .succeeds() + .stdout_only("y\n"); + // obs-l3 + scene + .ucmd() + .arg("-1l") + .pipe_in("x\ny") + .succeeds() + .stdout_only("y"); + // obs-l: same as -10l + scene + .ucmd() + .arg("-l") + .pipe_in(format!("x{}z", "y\n".repeat(10))) + .succeeds() + .stdout_only(&format!("{}z", "y\n".repeat(9))); +} + +#[test] +fn test_gnu_args_plus_l() { + let scene = TestScenario::new(util_name!()); + + // obs-plus-l4 + scene + .ucmd() + .arg("+1l") + .pipe_in("x\ny\n") + .succeeds() + .stdout_only("x\ny\n"); + // ops-plus-l5 + scene + .ucmd() + .arg("+2l") + .pipe_in("x\ny\n") + .succeeds() + .stdout_only("y\n"); + // obs-plus-x2: same as +10l + scene + .ucmd() + .arg("+l") + .pipe_in(format!("x\n{}z", "y\n".repeat(10))) + .succeeds() + .stdout_only("y\ny\nz"); +} + +#[test] +fn test_gnu_args_number() { + let scene = TestScenario::new(util_name!()); + + // obs-1 + scene + .ucmd() + .arg("-1") + .pipe_in("x") + .succeeds() + .stdout_only("x"); + // obs-2 + scene + .ucmd() + .arg("-1") + .pipe_in("x\ny\n") + .succeeds() + .stdout_only("y\n"); + // obs-3 + scene + .ucmd() + .arg("-1") + .pipe_in("x\ny") + .succeeds() + .stdout_only("y"); +} + +#[test] +fn test_gnu_args_plus_number() { + let scene = TestScenario::new(util_name!()); + + // obs-plus-4 + scene + .ucmd() + .arg("+1") + .pipe_in("x\ny\n") + .succeeds() + .stdout_only("x\ny\n"); + // ops-plus-5 + scene + .ucmd() + .arg("+2") + .pipe_in("x\ny\n") + .succeeds() + .stdout_only("y\n"); +} + +#[test] +fn test_gnu_args_b() { + let scene = TestScenario::new(util_name!()); + + // obs-b + scene + .ucmd() + .arg("-b") + .pipe_in("x\n".repeat(512 * 10 / 2 + 1)) + .succeeds() + .stdout_only(&"x\n".repeat(512 * 10 / 2)); +} + +#[test] +fn test_gnu_args_err() { + let scene = TestScenario::new(util_name!()); + + // err-1 + scene + .ucmd() + .arg("+cl") + .fails() + .no_stdout() + .stderr_is("tail: cannot open '+cl' for reading: No such file or directory\n") + .code_is(1); + // err-2 + scene + .ucmd() + .arg("-cl") + .fails() + .no_stdout() + .stderr_is("tail: invalid number of bytes: 'l'\n") + .code_is(1); + // err-3 + scene + .ucmd() + .arg("+2cz") + .fails() + .no_stdout() + .stderr_is("tail: cannot open '+2cz' for reading: No such file or directory\n") + .code_is(1); + // err-4 + scene + .ucmd() + .arg("-2cX") + .fails() + .no_stdout() + .stderr_is("tail: option used in invalid context -- 2\n") + .code_is(1); + // err-5 + scene + .ucmd() + .arg("-c99999999999999999999") + .fails() + .no_stdout() + .stderr_is("tail: invalid number of bytes: '99999999999999999999'\n") + .code_is(1); + // err-6 + scene + .ucmd() + .arg("-c --") + .fails() + .no_stdout() + .stderr_is("tail: invalid number of bytes: '-'\n") + .code_is(1); + scene + .ucmd() + .arg("-5cz") + .fails() + .no_stdout() + .stderr_is("tail: option used in invalid context -- 5\n") + .code_is(1); +} From dc34e89d5044f545740b14fd8d3a092b1338ab11 Mon Sep 17 00:00:00 2001 From: Benjamin Bara Date: Thu, 16 Feb 2023 21:16:21 +0100 Subject: [PATCH 59/94] tail: skip clap for obsolete args --- src/uu/tail/src/args.rs | 124 +++++++++++++++++++++--------- src/uu/tail/src/parse.rs | 153 ++++++++++++++++++++----------------- tests/by-util/test_tail.rs | 108 +++++++++++++++++++++++++- 3 files changed, 278 insertions(+), 107 deletions(-) diff --git a/src/uu/tail/src/args.rs b/src/uu/tail/src/args.rs index aa4b1c867..f18f99d1b 100644 --- a/src/uu/tail/src/args.rs +++ b/src/uu/tail/src/args.rs @@ -13,7 +13,6 @@ use fundu::DurationParser; use is_terminal::IsTerminal; use same_file::Handle; use std::collections::VecDeque; -use std::ffi::OsString; use std::time::Duration; use uucore::error::{UResult, USimpleError, UUsageError}; use uucore::parse_size::{parse_size, ParseSizeError}; @@ -59,6 +58,19 @@ pub enum FilterMode { } impl FilterMode { + fn from_obsolete_args(args: &parse::ObsoleteArgs) -> Self { + let signum = if args.plus { + Signum::Positive(args.num) + } else { + Signum::Negative(args.num) + }; + if args.lines { + Self::Lines(signum, b'\n') + } else { + Self::Bytes(signum) + } + } + fn from(matches: &ArgMatches) -> UResult { let zero_term = matches.get_flag(options::ZERO_TERM); let mode = if let Some(arg) = matches.get_one::(options::BYTES) { @@ -132,6 +144,29 @@ pub struct Settings { } impl Settings { + pub fn from_obsolete_args(args: &parse::ObsoleteArgs, name: Option<&str>) -> Self { + let mut settings: Self = Self { + sleep_sec: Duration::from_secs_f32(1.0), + max_unchanged_stats: 5, + ..Default::default() + }; + if args.follow { + settings.follow = if name.is_some() { + Some(FollowMode::Name) + } else { + Some(FollowMode::Descriptor) + }; + } + settings.mode = FilterMode::from_obsolete_args(args); + let input = if let Some(name) = name { + Input::from(name.to_string()) + } else { + Input::default() + }; + settings.inputs.push_back(input); + settings + } + pub fn from(matches: &clap::ArgMatches) -> UResult { let mut settings: Self = Self { sleep_sec: Duration::from_secs_f32(1.0), @@ -308,37 +343,24 @@ impl Settings { } } -pub fn arg_iterate<'a>( - mut args: impl uucore::Args + 'a, -) -> UResult + 'a>> { - // argv[0] is always present - let first = args.next().unwrap(); - if let Some(second) = args.next() { - if let Some(s) = second.to_str() { - match parse::parse_obsolete(s) { - Some(Ok(iter)) => Ok(Box::new(vec![first].into_iter().chain(iter).chain(args))), - Some(Err(e)) => Err(USimpleError::new( - 1, - match e { - parse::ParseError::Overflow => format!( - "invalid argument: {} Value too large for defined datatype", - s.quote() - ), - parse::ParseError::Context => { - format!( - "option used in invalid context -- {}", - s.chars().nth(1).unwrap_or_default() - ) - } - }, - )), - None => Ok(Box::new(vec![first, second].into_iter().chain(args))), - } - } else { - Err(UUsageError::new(1, "bad argument encoding".to_owned())) - } - } else { - Ok(Box::new(vec![first].into_iter())) +pub fn parse_obsolete(args: &str) -> UResult> { + match parse::parse_obsolete(args) { + Some(Ok(args)) => Ok(Some(args)), + None => Ok(None), + Some(Err(e)) => Err(USimpleError::new( + 1, + match e { + parse::ParseError::OutOfRange => format!( + "invalid number: {}: Numerical result out of range", + args.quote() + ), + parse::ParseError::Overflow => format!("invalid number: {}", args.quote()), + parse::ParseError::Context => format!( + "option used in invalid context -- {}", + args.chars().nth(1).unwrap_or_default() + ), + }, + )), } } @@ -366,9 +388,29 @@ fn parse_num(src: &str) -> Result { }) } -pub fn parse_args(args: impl uucore::Args) -> UResult { - let matches = uu_app().try_get_matches_from(arg_iterate(args)?)?; - Settings::from(&matches) +pub fn parse_args(mut args: impl uucore::Args) -> UResult { + let first = args.next().unwrap(); + let second = match args.next() { + Some(second) => second, + None => return Settings::from(&uu_app().try_get_matches_from(vec![first])?), + }; + let second_str = match second.to_str() { + Some(second_str) => second_str, + None => { + let second_string = second.to_string_lossy(); + return Err(USimpleError::new( + 1, + format!("bad argument encoding: '{second_string}'"), + )); + } + }; + match parse_obsolete(second_str)? { + Some(obsolete_args) => Ok(Settings::from_obsolete_args(&obsolete_args, args.next())), + None => { + let args = vec![first, second].into_iter().chain(args); + Settings::from(&uu_app().try_get_matches_from(args)?) + } + } } pub fn uu_app() -> Command { @@ -497,6 +539,8 @@ pub fn uu_app() -> Command { #[cfg(test)] mod tests { + use crate::parse::ObsoleteArgs; + use super::*; #[test] @@ -528,4 +572,14 @@ mod tests { assert!(result.is_ok()); assert_eq!(result.unwrap(), Signum::Negative(1)); } + + #[test] + fn test_parse_obsolete_settings_f() { + let args = ObsoleteArgs { follow: true, ..Default::default() }; + let result = Settings::from_obsolete_args(&args, None); + assert_eq!(result.follow, Some(FollowMode::Descriptor)); + + let result = Settings::from_obsolete_args(&args, Some("test".into())); + assert_eq!(result.follow, Some(FollowMode::Name)); + } } diff --git a/src/uu/tail/src/parse.rs b/src/uu/tail/src/parse.rs index 15c770bc5..2f4ebb62e 100644 --- a/src/uu/tail/src/parse.rs +++ b/src/uu/tail/src/parse.rs @@ -3,16 +3,34 @@ // * For the full copyright and license information, please view the LICENSE // * file that was distributed with this source code. -use std::ffi::OsString; +#[derive(PartialEq, Eq, Debug, Copy, Clone)] +pub struct ObsoleteArgs { + pub num: u64, + pub plus: bool, + pub lines: bool, + pub follow: bool, +} + +impl Default for ObsoleteArgs { + fn default() -> Self { + Self { + num: 10, + plus: false, + lines: true, + follow: false, + } + } +} #[derive(PartialEq, Eq, Debug)] pub enum ParseError { + OutOfRange, Overflow, Context, } /// Parses obsolete syntax /// tail -\[NUM\]\[bl\]\[f\] and tail +\[NUM\]\[bcl\]\[f\] // spell-checker:disable-line -pub fn parse_obsolete(src: &str) -> Option, ParseError>> { +pub fn parse_obsolete(src: &str) -> Option> { let mut chars = src.chars(); let sign = chars.next()?; if sign != '+' && sign != '-' { @@ -21,27 +39,27 @@ pub fn parse_obsolete(src: &str) -> Option let numbers: String = chars.clone().take_while(|&c| c.is_ascii_digit()).collect(); let has_num = !numbers.is_empty(); - let num: usize = if has_num { + let num: u64 = if has_num { if let Ok(num) = numbers.parse() { num } else { - return Some(Err(ParseError::Overflow)); + return Some(Err(ParseError::OutOfRange)); } } else { 10 }; let mut follow = false; - let mut mode = None; + let mut mode = 'l'; let mut first_char = true; for char in chars.skip_while(|&c| c.is_ascii_digit()) { - if sign == '-' && char == 'c' && !has_num { - // special case: -c should be handled by clap (is ambiguous) + if !has_num && first_char && sign == '-' && (char == 'c' || char == 'f') { + // special cases: -c, -f should be handled by clap (are ambiguous) return None; } else if char == 'f' { follow = true; } else if first_char && (char == 'b' || char == 'c' || char == 'l') { - mode = Some(char); + mode = char; } else if has_num && sign == '-' { return Some(Err(ParseError::Context)); } else { @@ -49,82 +67,81 @@ pub fn parse_obsolete(src: &str) -> Option } first_char = false; } + let multiplier = if mode == 'b' { 512 } else { 1 }; + let num = match num.checked_mul(multiplier) { + Some(n) => n, + None => return Some(Err(ParseError::Overflow)), + }; - let mut options = Vec::new(); - if follow { - options.push(OsString::from("-f")); - } - let mode = mode.unwrap_or('l'); - if mode == 'b' || mode == 'c' { - options.push(OsString::from("-c")); - let n = if mode == 'b' { 512 } else { 1 }; - let num = match num.checked_mul(n) { - Some(n) => n, - None => return Some(Err(ParseError::Overflow)), - }; - options.push(OsString::from(format!("{sign}{num}"))); - } else { - options.push(OsString::from("-n")); - options.push(OsString::from(format!("{sign}{num}"))); - } - Some(Ok(options.into_iter())) + Some(Ok(ObsoleteArgs { + num, + plus: sign == '+', + lines: mode == 'l', + follow, + })) } #[cfg(test)] mod tests { use super::*; - fn obsolete(src: &str) -> Option, ParseError>> { - let r = parse_obsolete(src); - match r { - Some(s) => match s { - Ok(v) => Some(Ok(v.map(|s| s.to_str().unwrap().to_owned()).collect())), - Err(e) => Some(Err(e)), - }, - None => None, - } - } - fn obsolete_result(src: &[&str]) -> Option, ParseError>> { - Some(Ok(src.iter().map(|s| s.to_string()).collect())) - } #[test] fn test_parse_numbers_obsolete() { - assert_eq!(obsolete("+2c"), obsolete_result(&["-c", "+2"])); - assert_eq!(obsolete("-5"), obsolete_result(&["-n", "-5"])); - assert_eq!(obsolete("-100"), obsolete_result(&["-n", "-100"])); - assert_eq!(obsolete("-2b"), obsolete_result(&["-c", "-1024"])); + assert_eq!( + parse_obsolete("+2c"), + Some(Ok(ObsoleteArgs { + num: 2, + plus: true, + lines: false, + follow: false, + })) + ); + assert_eq!( + parse_obsolete("-5"), + Some(Ok(ObsoleteArgs { + num: 5, + plus: false, + lines: true, + follow: false, + })) + ); + assert_eq!( + parse_obsolete("+100f"), + Some(Ok(ObsoleteArgs { + num: 100, + plus: true, + lines: true, + follow: true, + })) + ); + assert_eq!( + parse_obsolete("-2b"), + Some(Ok(ObsoleteArgs { + num: 1024, + plus: false, + lines: false, + follow: false, + })) + ); } #[test] fn test_parse_errors_obsolete() { - assert_eq!(obsolete("-5n"), Some(Err(ParseError::Context))); - assert_eq!(obsolete("-5c5"), Some(Err(ParseError::Context))); - assert_eq!(obsolete("-1vzc"), Some(Err(ParseError::Context))); - assert_eq!(obsolete("-5m"), Some(Err(ParseError::Context))); - assert_eq!(obsolete("-1k"), Some(Err(ParseError::Context))); - assert_eq!(obsolete("-1mmk"), Some(Err(ParseError::Context))); - assert_eq!(obsolete("-105kzm"), Some(Err(ParseError::Context))); - assert_eq!(obsolete("-1vz"), Some(Err(ParseError::Context))); + assert_eq!(parse_obsolete("-5n"), Some(Err(ParseError::Context))); + assert_eq!(parse_obsolete("-5c5"), Some(Err(ParseError::Context))); + assert_eq!(parse_obsolete("-1vzc"), Some(Err(ParseError::Context))); + assert_eq!(parse_obsolete("-5m"), Some(Err(ParseError::Context))); + assert_eq!(parse_obsolete("-1k"), Some(Err(ParseError::Context))); + assert_eq!(parse_obsolete("-1mmk"), Some(Err(ParseError::Context))); + assert_eq!(parse_obsolete("-105kzm"), Some(Err(ParseError::Context))); + assert_eq!(parse_obsolete("-1vz"), Some(Err(ParseError::Context))); assert_eq!( - obsolete("-1vzqvq"), // spell-checker:disable-line + parse_obsolete("-1vzqvq"), // spell-checker:disable-line Some(Err(ParseError::Context)) ); } #[test] fn test_parse_obsolete_no_match() { - assert_eq!(obsolete("-k"), None); - assert_eq!(obsolete("asd"), None); - assert_eq!(obsolete("-cc"), None); - } - #[test] - #[cfg(target_pointer_width = "64")] - fn test_parse_obsolete_overflow_x64() { - assert_eq!( - obsolete("-10000000000000000000000"), - Some(Err(ParseError::Overflow)) - ); - } - #[test] - #[cfg(target_pointer_width = "32")] - fn test_parse_obsolete_overflow_x32() { - assert_eq!(obsolete("-42949672960"), Some(Err(ParseError::Overflow))); + assert_eq!(parse_obsolete("-k"), None); + assert_eq!(parse_obsolete("asd"), None); + assert_eq!(parse_obsolete("-cc"), None); } } diff --git a/tests/by-util/test_tail.rs b/tests/by-util/test_tail.rs index 4644cbc02..d1e5ba632 100644 --- a/tests/by-util/test_tail.rs +++ b/tests/by-util/test_tail.rs @@ -47,6 +47,13 @@ static FOLLOW_NAME_EXP: &str = "follow_name.expected"; #[cfg(not(windows))] const DEFAULT_SLEEP_INTERVAL_MILLIS: u64 = 1000; +// The binary integer "10000000" is *not* a valid UTF-8 encoding +// of a character: https://en.wikipedia.org/wiki/UTF-8#Encoding +#[cfg(unix)] +const INVALID_UTF8: u8 = 0x80; +#[cfg(windows)] +const INVALID_UTF16: u16 = 0xD800; + #[test] fn test_invalid_arg() { new_ucmd!().arg("--definitely-invalid").fails().code_is(1); @@ -469,16 +476,13 @@ fn test_follow_non_utf8_bytes() { // Now append some bytes that are not valid UTF-8. // - // The binary integer "10000000" is *not* a valid UTF-8 encoding - // of a character: https://en.wikipedia.org/wiki/UTF-8#Encoding - // // We also write the newline character because our implementation // of `tail` is attempting to read a line of input, so the // presence of a newline character will force the `follow()` // function to conclude reading input bytes and start writing them // to output. The newline character is not fundamental to this // test, it is just a requirement of the current implementation. - let expected = [0b10000000, b'\n']; + let expected = [INVALID_UTF8, b'\n']; at.append_bytes(FOOBAR_TXT, &expected); child @@ -4710,4 +4714,100 @@ fn test_gnu_args_err() { .no_stdout() .stderr_is("tail: option used in invalid context -- 5\n") .code_is(1); + scene + .ucmd() + .arg("-9999999999999999999b") + .fails() + .no_stdout() + .stderr_is("tail: invalid number: '-9999999999999999999b'\n") + .code_is(1); + scene + .ucmd() + .arg("-999999999999999999999b") + .fails() + .no_stdout() + .stderr_is( + "tail: invalid number: '-999999999999999999999b': Numerical result out of range\n", + ) + .code_is(1); +} + +#[test] +fn test_gnu_args_f() { + let scene = TestScenario::new(util_name!()); + let at = &scene.fixtures; + + let mut p = scene + .ucmd() + .set_stdin(Stdio::piped()) + .arg("+f") + .run_no_wait(); + p.make_assertion_with_delay(500).is_alive(); + p.kill() + .make_assertion() + .with_all_output() + .no_stderr() + .no_stdout(); + + let source = "file"; + at.touch(source); + let mut p = scene.ucmd().args(&["+f", source]).run_no_wait(); + p.make_assertion_with_delay(500).is_alive(); + p.kill() + .make_assertion() + .with_all_output() + .no_stderr() + .no_stdout(); +} + +#[test] +#[cfg(unix)] +fn test_obsolete_encoding_unix() { + use std::ffi::OsStr; + use std::os::unix::ffi::OsStrExt; + + let scene = TestScenario::new(util_name!()); + let invalid_utf8_arg = OsStr::from_bytes(&[b'-', INVALID_UTF8, b'b']); + let valid_utf8_arg = OsStr::from_bytes(&[b'-', b'b']); + + scene + .ucmd() + .arg(invalid_utf8_arg) + .fails() + .no_stdout() + .stderr_is("tail: bad argument encoding: '-�b'\n") + .code_is(1); + scene + .ucmd() + .args(&[valid_utf8_arg, invalid_utf8_arg]) + .fails() + .no_stdout() + .stderr_is("tail: bad argument encoding\n") + .code_is(1); +} + +#[test] +#[cfg(windows)] +fn test_obsolete_encoding_windows() { + use std::ffi::OsString; + use std::os::windows::ffi::OsStringExt; + + let scene = TestScenario::new(util_name!()); + let invalid_utf16_arg = OsString::from_wide(&['-' as u16, INVALID_UTF16, 'b' as u16]); + let valid_utf16_arg = OsString::from("-b"); + + scene + .ucmd() + .arg(&invalid_utf16_arg) + .fails() + .no_stdout() + .stderr_is("tail: bad argument encoding: '-�b'\n") + .code_is(1); + scene + .ucmd() + .args(&[&valid_utf16_arg, &invalid_utf16_arg]) + .fails() + .no_stdout() + .stderr_is("tail: bad argument encoding\n") + .code_is(1); } From f6edea2d0565d476d6725f57f20bb4b4abf16f20 Mon Sep 17 00:00:00 2001 From: Benjamin Bara Date: Sat, 18 Feb 2023 15:44:49 +0100 Subject: [PATCH 60/94] tail: enable non-utf8 paths --- src/uu/tail/src/args.rs | 12 ++++++++---- src/uu/tail/src/paths.rs | 13 +++++++------ tests/by-util/test_tail.rs | 16 ---------------- 3 files changed, 15 insertions(+), 26 deletions(-) diff --git a/src/uu/tail/src/args.rs b/src/uu/tail/src/args.rs index f18f99d1b..7bf06f853 100644 --- a/src/uu/tail/src/args.rs +++ b/src/uu/tail/src/args.rs @@ -13,6 +13,7 @@ use fundu::DurationParser; use is_terminal::IsTerminal; use same_file::Handle; use std::collections::VecDeque; +use std::ffi::OsString; use std::time::Duration; use uucore::error::{UResult, USimpleError, UUsageError}; use uucore::parse_size::{parse_size, ParseSizeError}; @@ -144,7 +145,7 @@ pub struct Settings { } impl Settings { - pub fn from_obsolete_args(args: &parse::ObsoleteArgs, name: Option<&str>) -> Self { + pub fn from_obsolete_args(args: &parse::ObsoleteArgs, name: Option) -> Self { let mut settings: Self = Self { sleep_sec: Duration::from_secs_f32(1.0), max_unchanged_stats: 5, @@ -159,7 +160,7 @@ impl Settings { } settings.mode = FilterMode::from_obsolete_args(args); let input = if let Some(name) = name { - Input::from(name.to_string()) + Input::from(&name) } else { Input::default() }; @@ -249,7 +250,7 @@ impl Settings { let mut inputs: VecDeque = matches .get_many::(options::ARG_FILES) - .map(|v| v.map(|string| Input::from(string.clone())).collect()) + .map(|v| v.map(|string| Input::from(&string)).collect()) .unwrap_or_default(); // apply default and add '-' to inputs if none is present @@ -575,7 +576,10 @@ mod tests { #[test] fn test_parse_obsolete_settings_f() { - let args = ObsoleteArgs { follow: true, ..Default::default() }; + let args = ObsoleteArgs { + follow: true, + ..Default::default() + }; let result = Settings::from_obsolete_args(&args, None); assert_eq!(result.follow, Some(FollowMode::Descriptor)); diff --git a/src/uu/tail/src/paths.rs b/src/uu/tail/src/paths.rs index d8e6ece9a..b1cbc9bb8 100644 --- a/src/uu/tail/src/paths.rs +++ b/src/uu/tail/src/paths.rs @@ -6,6 +6,7 @@ // spell-checker:ignore tailable seekable stdlib (stdlib) use crate::text; +use std::ffi::OsStr; use std::fs::{File, Metadata}; use std::io::{Seek, SeekFrom}; #[cfg(unix)] @@ -26,21 +27,21 @@ pub struct Input { } impl Input { - // TODO: from &str may be the better choice - pub fn from(string: String) -> Self { - let kind = if string == text::DASH { + pub fn from>(string: &T) -> Self { + let valid_string = string.as_ref().to_str(); + let kind = if valid_string.is_some() && valid_string.unwrap() == text::DASH { InputKind::Stdin } else { - InputKind::File(PathBuf::from(&string)) + InputKind::File(PathBuf::from(string.as_ref())) }; let display_name = match kind { - InputKind::File(_) => string, + InputKind::File(_) => string.as_ref().to_string_lossy().to_string(), InputKind::Stdin => { if cfg!(unix) { text::STDIN_HEADER.to_string() } else { - string + string.as_ref().to_string_lossy().to_string() } } }; diff --git a/tests/by-util/test_tail.rs b/tests/by-util/test_tail.rs index d1e5ba632..b450d303b 100644 --- a/tests/by-util/test_tail.rs +++ b/tests/by-util/test_tail.rs @@ -4768,7 +4768,6 @@ fn test_obsolete_encoding_unix() { let scene = TestScenario::new(util_name!()); let invalid_utf8_arg = OsStr::from_bytes(&[b'-', INVALID_UTF8, b'b']); - let valid_utf8_arg = OsStr::from_bytes(&[b'-', b'b']); scene .ucmd() @@ -4777,13 +4776,6 @@ fn test_obsolete_encoding_unix() { .no_stdout() .stderr_is("tail: bad argument encoding: '-�b'\n") .code_is(1); - scene - .ucmd() - .args(&[valid_utf8_arg, invalid_utf8_arg]) - .fails() - .no_stdout() - .stderr_is("tail: bad argument encoding\n") - .code_is(1); } #[test] @@ -4794,7 +4786,6 @@ fn test_obsolete_encoding_windows() { let scene = TestScenario::new(util_name!()); let invalid_utf16_arg = OsString::from_wide(&['-' as u16, INVALID_UTF16, 'b' as u16]); - let valid_utf16_arg = OsString::from("-b"); scene .ucmd() @@ -4803,11 +4794,4 @@ fn test_obsolete_encoding_windows() { .no_stdout() .stderr_is("tail: bad argument encoding: '-�b'\n") .code_is(1); - scene - .ucmd() - .args(&[&valid_utf16_arg, &invalid_utf16_arg]) - .fails() - .no_stdout() - .stderr_is("tail: bad argument encoding\n") - .code_is(1); } From 9b49f368c7cf7fb91054f2b609432715efc3fbdc Mon Sep 17 00:00:00 2001 From: Benjamin Bara Date: Sun, 19 Feb 2023 14:24:07 +0100 Subject: [PATCH 61/94] tail: parse default before obsolete --- src/uu/tail/src/args.rs | 119 +++++++++++++++++++++++-------------- tests/by-util/test_tail.rs | 16 ++--- 2 files changed, 84 insertions(+), 51 deletions(-) diff --git a/src/uu/tail/src/args.rs b/src/uu/tail/src/args.rs index 7bf06f853..9d0d6db7c 100644 --- a/src/uu/tail/src/args.rs +++ b/src/uu/tail/src/args.rs @@ -130,7 +130,7 @@ pub enum VerificationResult { NoOutput, } -#[derive(Debug, Default)] +#[derive(Debug)] pub struct Settings { pub follow: Option, pub max_unchanged_stats: u32, @@ -144,13 +144,26 @@ pub struct Settings { pub inputs: VecDeque, } -impl Settings { - pub fn from_obsolete_args(args: &parse::ObsoleteArgs, name: Option) -> Self { - let mut settings: Self = Self { - sleep_sec: Duration::from_secs_f32(1.0), +impl Default for Settings { + fn default() -> Self { + Self { max_unchanged_stats: 5, - ..Default::default() - }; + sleep_sec: Duration::from_secs_f32(1.0), + follow: Default::default(), + mode: Default::default(), + pid: Default::default(), + retry: Default::default(), + use_polling: Default::default(), + verbose: Default::default(), + presume_input_pipe: Default::default(), + inputs: Default::default(), + } + } +} + +impl Settings { + pub fn from_obsolete_args(args: &parse::ObsoleteArgs, name: Option<&OsString>) -> Self { + let mut settings: Self = Default::default(); if args.follow { settings.follow = if name.is_some() { Some(FollowMode::Name) @@ -170,25 +183,25 @@ impl Settings { pub fn from(matches: &clap::ArgMatches) -> UResult { let mut settings: Self = Self { - sleep_sec: Duration::from_secs_f32(1.0), - max_unchanged_stats: 5, + follow: if matches.get_flag(options::FOLLOW_RETRY) { + Some(FollowMode::Name) + } else if matches.value_source(options::FOLLOW) != Some(ValueSource::CommandLine) { + None + } else if matches.get_one::(options::FOLLOW) + == Some(String::from("name")).as_ref() + { + Some(FollowMode::Name) + } else { + Some(FollowMode::Descriptor) + }, + retry: matches.get_flag(options::RETRY) || matches.get_flag(options::FOLLOW_RETRY), + use_polling: matches.get_flag(options::USE_POLLING), + mode: FilterMode::from(matches)?, + verbose: matches.get_flag(options::verbosity::VERBOSE), + presume_input_pipe: matches.get_flag(options::PRESUME_INPUT_PIPE), ..Default::default() }; - settings.follow = if matches.get_flag(options::FOLLOW_RETRY) { - Some(FollowMode::Name) - } else if matches.value_source(options::FOLLOW) != Some(ValueSource::CommandLine) { - None - } else if matches.get_one::(options::FOLLOW) == Some(String::from("name")).as_ref() - { - Some(FollowMode::Name) - } else { - Some(FollowMode::Descriptor) - }; - - settings.retry = - matches.get_flag(options::RETRY) || matches.get_flag(options::FOLLOW_RETRY); - if let Some(source) = matches.get_one::(options::SLEEP_INT) { // Advantage of `fundu` over `Duration::(try_)from_secs_f64(source.parse().unwrap())`: // * doesn't panic on errors like `Duration::from_secs_f64` would. @@ -205,8 +218,6 @@ impl Settings { })?; } - settings.use_polling = matches.get_flag(options::USE_POLLING); - if let Some(s) = matches.get_one::(options::MAX_UNCHANGED_STATS) { settings.max_unchanged_stats = match s.parse::() { Ok(s) => s, @@ -246,8 +257,6 @@ impl Settings { } } - settings.mode = FilterMode::from(matches)?; - let mut inputs: VecDeque = matches .get_many::(options::ARG_FILES) .map(|v| v.map(|string| Input::from(&string)).collect()) @@ -258,13 +267,10 @@ impl Settings { inputs.push_front(Input::default()); } - settings.verbose = (matches.get_flag(options::verbosity::VERBOSE) || inputs.len() > 1) - && !matches.get_flag(options::verbosity::QUIET); + settings.verbose = inputs.len() > 1 && !matches.get_flag(options::verbosity::QUIET); settings.inputs = inputs; - settings.presume_input_pipe = matches.get_flag(options::PRESUME_INPUT_PIPE); - Ok(settings) } @@ -342,6 +348,19 @@ impl Settings { VerificationResult::Ok } + + pub fn is_default(&self) -> bool { + let default = Self::default(); + self.max_unchanged_stats == default.max_unchanged_stats + && self.sleep_sec == default.sleep_sec + && self.follow == default.follow + && self.mode == default.mode + && self.pid == default.pid + && self.retry == default.retry + && self.use_polling == default.use_polling + && (self.verbose == default.verbose || self.inputs.len() > 1) + && self.presume_input_pipe == default.presume_input_pipe + } } pub fn parse_obsolete(args: &str) -> UResult> { @@ -389,28 +408,42 @@ fn parse_num(src: &str) -> Result { }) } -pub fn parse_args(mut args: impl uucore::Args) -> UResult { - let first = args.next().unwrap(); - let second = match args.next() { +pub fn parse_args(args: impl uucore::Args) -> UResult { + let args_vec: Vec = args.collect(); + let clap_result = match uu_app().try_get_matches_from(args_vec.clone()) { + Ok(matches) => { + let settings = Settings::from(&matches)?; + if !settings.is_default() { + // non-default settings can't have obsolete arguments + return Ok(settings); + } + Ok(settings) + } + Err(err) => Err(err.into()), + }; + + // clap parsing failed or resulted to default -> check for obsolete/deprecated args + // argv[0] is always present + let second = match args_vec.get(1) { Some(second) => second, - None => return Settings::from(&uu_app().try_get_matches_from(vec![first])?), + None => return clap_result, }; let second_str = match second.to_str() { Some(second_str) => second_str, None => { - let second_string = second.to_string_lossy(); + let invalid_string = second.to_string_lossy(); return Err(USimpleError::new( 1, - format!("bad argument encoding: '{second_string}'"), + format!("bad argument encoding: '{invalid_string}'"), )); } }; match parse_obsolete(second_str)? { - Some(obsolete_args) => Ok(Settings::from_obsolete_args(&obsolete_args, args.next())), - None => { - let args = vec![first, second].into_iter().chain(args); - Settings::from(&uu_app().try_get_matches_from(args)?) - } + Some(obsolete_args) => Ok(Settings::from_obsolete_args( + &obsolete_args, + args_vec.get(2), + )), + None => clap_result, } } @@ -583,7 +616,7 @@ mod tests { let result = Settings::from_obsolete_args(&args, None); assert_eq!(result.follow, Some(FollowMode::Descriptor)); - let result = Settings::from_obsolete_args(&args, Some("test".into())); + let result = Settings::from_obsolete_args(&args, Some(&"file".into())); assert_eq!(result.follow, Some(FollowMode::Name)); } } diff --git a/tests/by-util/test_tail.rs b/tests/by-util/test_tail.rs index b450d303b..eb18e858f 100644 --- a/tests/by-util/test_tail.rs +++ b/tests/by-util/test_tail.rs @@ -4737,11 +4737,9 @@ fn test_gnu_args_f() { let scene = TestScenario::new(util_name!()); let at = &scene.fixtures; - let mut p = scene - .ucmd() - .set_stdin(Stdio::piped()) - .arg("+f") - .run_no_wait(); + let source = "file"; + at.touch(source); + let mut p = scene.ucmd().args(&["+f", source]).run_no_wait(); p.make_assertion_with_delay(500).is_alive(); p.kill() .make_assertion() @@ -4749,9 +4747,11 @@ fn test_gnu_args_f() { .no_stderr() .no_stdout(); - let source = "file"; - at.touch(source); - let mut p = scene.ucmd().args(&["+f", source]).run_no_wait(); + let mut p = scene + .ucmd() + .set_stdin(Stdio::piped()) + .arg("+f") + .run_no_wait(); p.make_assertion_with_delay(500).is_alive(); p.kill() .make_assertion() From 26a945f3b54d3e3d86dccda3dafd08f71092d198 Mon Sep 17 00:00:00 2001 From: Benjamin Bara Date: Sun, 19 Feb 2023 15:16:58 +0100 Subject: [PATCH 62/94] tail: simplify obsolete parsing --- src/uu/tail/src/args.rs | 117 ++++++++++++++++++------------------- src/uu/tail/src/parse.rs | 122 ++++++++++++++++++++++++++------------- src/uu/tail/src/paths.rs | 3 +- 3 files changed, 140 insertions(+), 102 deletions(-) diff --git a/src/uu/tail/src/args.rs b/src/uu/tail/src/args.rs index 9d0d6db7c..8e039b5f4 100644 --- a/src/uu/tail/src/args.rs +++ b/src/uu/tail/src/args.rs @@ -348,39 +348,33 @@ impl Settings { VerificationResult::Ok } - - pub fn is_default(&self) -> bool { - let default = Self::default(); - self.max_unchanged_stats == default.max_unchanged_stats - && self.sleep_sec == default.sleep_sec - && self.follow == default.follow - && self.mode == default.mode - && self.pid == default.pid - && self.retry == default.retry - && self.use_polling == default.use_polling - && (self.verbose == default.verbose || self.inputs.len() > 1) - && self.presume_input_pipe == default.presume_input_pipe - } } -pub fn parse_obsolete(args: &str) -> UResult> { - match parse::parse_obsolete(args) { - Some(Ok(args)) => Ok(Some(args)), +pub fn parse_obsolete(arg: &OsString, input: Option<&OsString>) -> UResult> { + match parse::parse_obsolete(arg) { + Some(Ok(args)) => Ok(Some(Settings::from_obsolete_args(&args, input))), None => Ok(None), - Some(Err(e)) => Err(USimpleError::new( - 1, - match e { - parse::ParseError::OutOfRange => format!( - "invalid number: {}: Numerical result out of range", - args.quote() - ), - parse::ParseError::Overflow => format!("invalid number: {}", args.quote()), - parse::ParseError::Context => format!( - "option used in invalid context -- {}", - args.chars().nth(1).unwrap_or_default() - ), - }, - )), + Some(Err(e)) => { + let arg_str = arg.to_string_lossy(); + Err(USimpleError::new( + 1, + match e { + parse::ParseError::OutOfRange => format!( + "invalid number: {}: Numerical result out of range", + arg_str.quote() + ), + parse::ParseError::Overflow => format!("invalid number: {}", arg_str.quote()), + // this ensures compatibility to GNU's error message (as tested in misc/tail) + parse::ParseError::Context => format!( + "option used in invalid context -- {}", + arg_str.chars().nth(1).unwrap_or_default() + ), + parse::ParseError::InvalidEncoding => { + format!("bad argument encoding: '{arg_str}'") + } + }, + )) + } } } @@ -410,39 +404,41 @@ fn parse_num(src: &str) -> Result { pub fn parse_args(args: impl uucore::Args) -> UResult { let args_vec: Vec = args.collect(); - let clap_result = match uu_app().try_get_matches_from(args_vec.clone()) { - Ok(matches) => { - let settings = Settings::from(&matches)?; - if !settings.is_default() { - // non-default settings can't have obsolete arguments - return Ok(settings); - } - Ok(settings) - } + let clap_args = uu_app().try_get_matches_from(args_vec.clone()); + let clap_result = match clap_args { + Ok(matches) => Ok(Settings::from(&matches)?), Err(err) => Err(err.into()), }; - // clap parsing failed or resulted to default -> check for obsolete/deprecated args - // argv[0] is always present - let second = match args_vec.get(1) { - Some(second) => second, - None => return clap_result, - }; - let second_str = match second.to_str() { - Some(second_str) => second_str, - None => { - let invalid_string = second.to_string_lossy(); - return Err(USimpleError::new( - 1, - format!("bad argument encoding: '{invalid_string}'"), - )); - } - }; - match parse_obsolete(second_str)? { - Some(obsolete_args) => Ok(Settings::from_obsolete_args( - &obsolete_args, - args_vec.get(2), - )), + // clap isn't able to handle obsolete syntax. + // therefore, we want to check further for obsolete arguments. + // argv[0] is always present, argv[1] might be obsolete arguments + // argv[2] might contain an input file, argv[3] isn't allowed in obsolete mode + if args_vec.len() != 2 && args_vec.len() != 3 { + return clap_result; + } + + // At this point, there are a few possible cases: + // + // 1. clap has succeeded and the arguments would be invalid for the obsolete syntax. + // 2. The case of `tail -c 5` is ambiguous. clap parses this as `tail -c5`, + // but it could also be interpreted as valid obsolete syntax (tail -c on file '5'). + // GNU chooses to interpret this as `tail -c5`, like clap. + // 3. `tail -f foo` is also ambiguous, but has the same effect in both cases. We can safely + // use the clap result here. + // 4. clap succeeded for obsolete arguments starting with '+', but misinterprets them as + // input files (e.g. 'tail +f'). + // 5. clap failed because of unknown flags, but possibly valid obsolete arguments + // (e.g. tail -l; tail -10c). + // + // In cases 4 & 5, we want to try parsing the obsolete arguments, which corresponds to + // checking whether clap succeeded or the first argument starts with '+'. + let possible_obsolete_args = &args_vec[1]; + if clap_result.is_ok() && !possible_obsolete_args.to_string_lossy().starts_with('+') { + return clap_result; + } + match parse_obsolete(possible_obsolete_args, args_vec.get(2))? { + Some(settings) => Ok(settings), None => clap_result, } } @@ -477,6 +473,7 @@ pub fn uu_app() -> Command { .num_args(0..=1) .require_equals(true) .value_parser(["descriptor", "name"]) + .overrides_with(options::FOLLOW) .help("Print the file as it grows"), ) .arg( diff --git a/src/uu/tail/src/parse.rs b/src/uu/tail/src/parse.rs index 2f4ebb62e..96cf1e918 100644 --- a/src/uu/tail/src/parse.rs +++ b/src/uu/tail/src/parse.rs @@ -3,6 +3,8 @@ // * For the full copyright and license information, please view the LICENSE // * file that was distributed with this source code. +use std::ffi::OsString; + #[derive(PartialEq, Eq, Debug, Copy, Clone)] pub struct ObsoleteArgs { pub num: u64, @@ -27,20 +29,31 @@ pub enum ParseError { OutOfRange, Overflow, Context, + InvalidEncoding, } /// Parses obsolete syntax -/// tail -\[NUM\]\[bl\]\[f\] and tail +\[NUM\]\[bcl\]\[f\] // spell-checker:disable-line -pub fn parse_obsolete(src: &str) -> Option> { - let mut chars = src.chars(); - let sign = chars.next()?; - if sign != '+' && sign != '-' { +/// tail -\[NUM\]\[bcl\]\[f\] and tail +\[NUM\]\[bcl\]\[f\] // spell-checker:disable-line +pub fn parse_obsolete(src: &OsString) -> Option> { + let mut rest = match src.to_str() { + Some(src) => src, + None => return Some(Err(ParseError::InvalidEncoding)), + }; + let sign = if let Some(r) = rest.strip_prefix('-') { + rest = r; + '-' + } else if let Some(r) = rest.strip_prefix('+') { + rest = r; + '+' + } else { return None; - } + }; - let numbers: String = chars.clone().take_while(|&c| c.is_ascii_digit()).collect(); - let has_num = !numbers.is_empty(); + let end_num = rest + .find(|c: char| !c.is_ascii_digit()) + .unwrap_or(rest.len()); + let has_num = !rest[..end_num].is_empty(); let num: u64 = if has_num { - if let Ok(num) = numbers.parse() { + if let Ok(num) = rest[..end_num].parse() { num } else { return Some(Err(ParseError::OutOfRange)); @@ -48,25 +61,30 @@ pub fn parse_obsolete(src: &str) -> Option> { } else { 10 }; + rest = &rest[end_num..]; - let mut follow = false; - let mut mode = 'l'; - let mut first_char = true; - for char in chars.skip_while(|&c| c.is_ascii_digit()) { - if !has_num && first_char && sign == '-' && (char == 'c' || char == 'f') { - // special cases: -c, -f should be handled by clap (are ambiguous) - return None; - } else if char == 'f' { - follow = true; - } else if first_char && (char == 'b' || char == 'c' || char == 'l') { - mode = char; - } else if has_num && sign == '-' { + let mode = if let Some(r) = rest.strip_prefix('l') { + rest = r; + 'l' + } else if let Some(r) = rest.strip_prefix('c') { + rest = r; + 'c' + } else if let Some(r) = rest.strip_prefix('b') { + rest = r; + 'b' + } else { + 'l' + }; + + let follow = rest.contains('f'); + if !rest.chars().all(|f| f == 'f') { + // GNU allows an arbitrary amount of following fs, but nothing else + if sign == '-' && has_num { return Some(Err(ParseError::Context)); - } else { - return None; } - first_char = false; + return None; } + let multiplier = if mode == 'b' { 512 } else { 1 }; let num = match num.checked_mul(multiplier) { Some(n) => n, @@ -87,7 +105,7 @@ mod tests { #[test] fn test_parse_numbers_obsolete() { assert_eq!( - parse_obsolete("+2c"), + parse_obsolete(&OsString::from("+2c")), Some(Ok(ObsoleteArgs { num: 2, plus: true, @@ -96,7 +114,7 @@ mod tests { })) ); assert_eq!( - parse_obsolete("-5"), + parse_obsolete(&OsString::from("-5")), Some(Ok(ObsoleteArgs { num: 5, plus: false, @@ -105,7 +123,7 @@ mod tests { })) ); assert_eq!( - parse_obsolete("+100f"), + parse_obsolete(&OsString::from("+100f")), Some(Ok(ObsoleteArgs { num: 100, plus: true, @@ -114,7 +132,7 @@ mod tests { })) ); assert_eq!( - parse_obsolete("-2b"), + parse_obsolete(&OsString::from("-2b")), Some(Ok(ObsoleteArgs { num: 1024, plus: false, @@ -125,23 +143,47 @@ mod tests { } #[test] fn test_parse_errors_obsolete() { - assert_eq!(parse_obsolete("-5n"), Some(Err(ParseError::Context))); - assert_eq!(parse_obsolete("-5c5"), Some(Err(ParseError::Context))); - assert_eq!(parse_obsolete("-1vzc"), Some(Err(ParseError::Context))); - assert_eq!(parse_obsolete("-5m"), Some(Err(ParseError::Context))); - assert_eq!(parse_obsolete("-1k"), Some(Err(ParseError::Context))); - assert_eq!(parse_obsolete("-1mmk"), Some(Err(ParseError::Context))); - assert_eq!(parse_obsolete("-105kzm"), Some(Err(ParseError::Context))); - assert_eq!(parse_obsolete("-1vz"), Some(Err(ParseError::Context))); assert_eq!( - parse_obsolete("-1vzqvq"), // spell-checker:disable-line + parse_obsolete(&OsString::from("-5n")), + Some(Err(ParseError::Context)) + ); + assert_eq!( + parse_obsolete(&OsString::from("-5c5")), + Some(Err(ParseError::Context)) + ); + assert_eq!( + parse_obsolete(&OsString::from("-1vzc")), + Some(Err(ParseError::Context)) + ); + assert_eq!( + parse_obsolete(&OsString::from("-5m")), + Some(Err(ParseError::Context)) + ); + assert_eq!( + parse_obsolete(&OsString::from("-1k")), + Some(Err(ParseError::Context)) + ); + assert_eq!( + parse_obsolete(&OsString::from("-1mmk")), + Some(Err(ParseError::Context)) + ); + assert_eq!( + parse_obsolete(&OsString::from("-105kzm")), + Some(Err(ParseError::Context)) + ); + assert_eq!( + parse_obsolete(&OsString::from("-1vz")), + Some(Err(ParseError::Context)) + ); + assert_eq!( + parse_obsolete(&OsString::from("-1vzqvq")), // spell-checker:disable-line Some(Err(ParseError::Context)) ); } #[test] fn test_parse_obsolete_no_match() { - assert_eq!(parse_obsolete("-k"), None); - assert_eq!(parse_obsolete("asd"), None); - assert_eq!(parse_obsolete("-cc"), None); + assert_eq!(parse_obsolete(&OsString::from("-k")), None); + assert_eq!(parse_obsolete(&OsString::from("asd")), None); + assert_eq!(parse_obsolete(&OsString::from("-cc")), None); } } diff --git a/src/uu/tail/src/paths.rs b/src/uu/tail/src/paths.rs index b1cbc9bb8..4badd6866 100644 --- a/src/uu/tail/src/paths.rs +++ b/src/uu/tail/src/paths.rs @@ -28,8 +28,7 @@ pub struct Input { impl Input { pub fn from>(string: &T) -> Self { - let valid_string = string.as_ref().to_str(); - let kind = if valid_string.is_some() && valid_string.unwrap() == text::DASH { + let kind = if string.as_ref() == Path::new(text::DASH) { InputKind::Stdin } else { InputKind::File(PathBuf::from(string.as_ref())) From f08f4d60165ef092f5031d2f46e4007f60287b5e Mon Sep 17 00:00:00 2001 From: Koki Ueha Date: Wed, 1 Mar 2023 12:44:20 +0000 Subject: [PATCH 63/94] chown: move help strings to markdown file --- src/uu/chown/chown.md | 8 ++++++++ src/uu/chown/src/chown.rs | 8 +++----- 2 files changed, 11 insertions(+), 5 deletions(-) create mode 100644 src/uu/chown/chown.md diff --git a/src/uu/chown/chown.md b/src/uu/chown/chown.md new file mode 100644 index 000000000..11b653ea9 --- /dev/null +++ b/src/uu/chown/chown.md @@ -0,0 +1,8 @@ +# chown + +``` +chown [OPTION]... [OWNER][:[GROUP]] FILE... +chown [OPTION]... --reference=RFILE FILE... +``` + +Change file owner and group diff --git a/src/uu/chown/src/chown.rs b/src/uu/chown/src/chown.rs index 401703cdf..3faecd571 100644 --- a/src/uu/chown/src/chown.rs +++ b/src/uu/chown/src/chown.rs @@ -9,8 +9,8 @@ use uucore::display::Quotable; pub use uucore::entries::{self, Group, Locate, Passwd}; -use uucore::format_usage; use uucore::perms::{chown_base, options, IfFrom}; +use uucore::{format_usage, help_about, help_usage}; use uucore::error::{FromIo, UResult, USimpleError}; @@ -19,11 +19,9 @@ use clap::{crate_version, Arg, ArgAction, ArgMatches, Command}; use std::fs; use std::os::unix::fs::MetadataExt; -static ABOUT: &str = "Change file owner and group"; +static ABOUT: &str = help_about!("chown.md"); -const USAGE: &str = "\ - {} [OPTION]... [OWNER][:[GROUP]] FILE... - {} [OPTION]... --reference=RFILE FILE..."; +const USAGE: &str = help_usage!("chown.md"); fn parse_gid_uid_and_filter(matches: &ArgMatches) -> UResult<(Option, Option, IfFrom)> { let filter = if let Some(spec) = matches.get_one::(options::FROM) { From 0d782e09c146c32066d08cc23b04cb51eb43cc45 Mon Sep 17 00:00:00 2001 From: Daniel Hofstetter Date: Tue, 28 Feb 2023 10:57:14 +0100 Subject: [PATCH 64/94] comm: implement --zero-terminated --- src/uu/comm/src/comm.rs | 108 ++++++++++++++++------ tests/by-util/test_comm.rs | 22 +++-- tests/fixtures/comm/a_nul | Bin 0 -> 3 bytes tests/fixtures/comm/ab_nul.expected | Bin 0 -> 9 bytes tests/fixtures/comm/ab_nul_total.expected | Bin 0 -> 21 bytes tests/fixtures/comm/b_nul | Bin 0 -> 3 bytes 6 files changed, 93 insertions(+), 37 deletions(-) create mode 100644 tests/fixtures/comm/a_nul create mode 100644 tests/fixtures/comm/ab_nul.expected create mode 100644 tests/fixtures/comm/ab_nul_total.expected create mode 100644 tests/fixtures/comm/b_nul diff --git a/src/uu/comm/src/comm.rs b/src/uu/comm/src/comm.rs index 3bc45b6a5..02a9221d4 100644 --- a/src/uu/comm/src/comm.rs +++ b/src/uu/comm/src/comm.rs @@ -8,11 +8,11 @@ // spell-checker:ignore (ToDO) delim mkdelim use std::cmp::Ordering; +use std::fmt::Display; use std::fs::File; use std::io::{self, stdin, BufRead, BufReader, Stdin}; use std::path::Path; -use uucore::error::FromIo; -use uucore::error::UResult; +use uucore::error::{FromIo, UResult}; use uucore::{format_usage, help_about, help_usage}; use clap::{crate_version, Arg, ArgAction, ArgMatches, Command}; @@ -29,6 +29,7 @@ mod options { pub const FILE_1: &str = "FILE1"; pub const FILE_2: &str = "FILE2"; pub const TOTAL: &str = "total"; + pub const ZERO_TERMINATED: &str = "zero-terminated"; } fn column_width(col: &str, opts: &ArgMatches) -> usize { @@ -39,23 +40,66 @@ fn column_width(col: &str, opts: &ArgMatches) -> usize { } } -fn ensure_nl(line: &mut String) { - if !line.ends_with('\n') { - line.push('\n'); +#[repr(u8)] +#[derive(Clone, Copy)] +enum LineEnding { + Newline = b'\n', + Nul = 0, +} + +impl From for u8 { + fn from(line_ending: LineEnding) -> Self { + line_ending as Self } } -enum LineReader { +impl From for LineEnding { + fn from(is_zero_terminated: bool) -> Self { + if is_zero_terminated { + Self::Nul + } else { + Self::Newline + } + } +} + +impl Display for LineEnding { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Newline => writeln!(f), + Self::Nul => write!(f, "\0"), + } + } +} + +enum Input { Stdin(Stdin), FileIn(BufReader), } +struct LineReader { + line_ending: LineEnding, + input: Input, +} + impl LineReader { - fn read_line(&mut self, buf: &mut String) -> io::Result { - match *self { - Self::Stdin(ref mut r) => r.read_line(buf), - Self::FileIn(ref mut r) => r.read_line(buf), + fn new(input: Input, line_ending: LineEnding) -> Self { + Self { input, line_ending } + } + + fn read_line(&mut self, buf: &mut Vec) -> io::Result { + let line_ending = self.line_ending.into(); + + let result = match &mut self.input { + Input::Stdin(r) => r.lock().read_until(line_ending, buf), + Input::FileIn(r) => r.read_until(line_ending, buf), + }; + + if !buf.ends_with(&[line_ending]) { + buf.push(line_ending); } + + result } } @@ -71,9 +115,9 @@ fn comm(a: &mut LineReader, b: &mut LineReader, opts: &ArgMatches) { let delim_col_2 = delim.repeat(width_col_1); let delim_col_3 = delim.repeat(width_col_1 + width_col_2); - let ra = &mut String::new(); + let ra = &mut Vec::new(); let mut na = a.read_line(ra); - let rb = &mut String::new(); + let rb = &mut Vec::new(); let mut nb = b.read_line(rb); let mut total_col_1 = 0; @@ -96,8 +140,7 @@ fn comm(a: &mut LineReader, b: &mut LineReader, opts: &ArgMatches) { match ord { Ordering::Less => { if !opts.get_flag(options::COLUMN_1) { - ensure_nl(ra); - print!("{ra}"); + print!("{}", String::from_utf8_lossy(ra)); } ra.clear(); na = a.read_line(ra); @@ -105,8 +148,7 @@ fn comm(a: &mut LineReader, b: &mut LineReader, opts: &ArgMatches) { } Ordering::Greater => { if !opts.get_flag(options::COLUMN_2) { - ensure_nl(rb); - print!("{delim_col_2}{rb}"); + print!("{delim_col_2}{}", String::from_utf8_lossy(rb)); } rb.clear(); nb = b.read_line(rb); @@ -114,8 +156,7 @@ fn comm(a: &mut LineReader, b: &mut LineReader, opts: &ArgMatches) { } Ordering::Equal => { if !opts.get_flag(options::COLUMN_3) { - ensure_nl(ra); - print!("{delim_col_3}{ra}"); + print!("{delim_col_3}{}", String::from_utf8_lossy(ra)); } ra.clear(); rb.clear(); @@ -127,17 +168,20 @@ fn comm(a: &mut LineReader, b: &mut LineReader, opts: &ArgMatches) { } if opts.get_flag(options::TOTAL) { - println!("{total_col_1}{delim}{total_col_2}{delim}{total_col_3}{delim}total"); + let line_ending = LineEnding::from(opts.get_flag(options::ZERO_TERMINATED)); + print!("{total_col_1}{delim}{total_col_2}{delim}{total_col_3}{delim}total{line_ending}"); } } -fn open_file(name: &str) -> io::Result { - match name { - "-" => Ok(LineReader::Stdin(stdin())), - _ => { - let f = File::open(Path::new(name))?; - Ok(LineReader::FileIn(BufReader::new(f))) - } +fn open_file(name: &str, line_ending: LineEnding) -> io::Result { + if name == "-" { + Ok(LineReader::new(Input::Stdin(stdin()), line_ending)) + } else { + let f = File::open(Path::new(name))?; + Ok(LineReader::new( + Input::FileIn(BufReader::new(f)), + line_ending, + )) } } @@ -146,10 +190,11 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { let args = args.collect_lossy(); let matches = uu_app().try_get_matches_from(args)?; + let line_ending = LineEnding::from(matches.get_flag(options::ZERO_TERMINATED)); let filename1 = matches.get_one::(options::FILE_1).unwrap(); let filename2 = matches.get_one::(options::FILE_2).unwrap(); - let mut f1 = open_file(filename1).map_err_context(|| filename1.to_string())?; - let mut f2 = open_file(filename2).map_err_context(|| filename2.to_string())?; + let mut f1 = open_file(filename1, line_ending).map_err_context(|| filename1.to_string())?; + let mut f2 = open_file(filename2, line_ending).map_err_context(|| filename2.to_string())?; comm(&mut f1, &mut f2, &matches); Ok(()) @@ -187,6 +232,13 @@ pub fn uu_app() -> Command { .default_value(options::DELIMITER_DEFAULT) .hide_default_value(true), ) + .arg( + Arg::new(options::ZERO_TERMINATED) + .long(options::ZERO_TERMINATED) + .short('z') + .help("line delimiter is NUL, not newline") + .action(ArgAction::SetTrue), + ) .arg( Arg::new(options::FILE_1) .required(true) diff --git a/tests/by-util/test_comm.rs b/tests/by-util/test_comm.rs index 053746908..aa0791f97 100644 --- a/tests/by-util/test_comm.rs +++ b/tests/by-util/test_comm.rs @@ -95,19 +95,23 @@ fn output_delimiter_nul() { .stdout_only_fixture("ab_delimiter_nul.expected"); } -// even though (info) documentation suggests this is an option -// in latest GNU Coreutils comm, it actually is not. -// this test is essentially an alarm in case some well-intending -// developer implements it. -//marked as unimplemented as error message not set yet. -#[cfg_attr(not(feature = "test_unimplemented"), ignore)] #[test] fn zero_terminated() { for param in ["-z", "--zero-terminated"] { new_ucmd!() - .args(&[param, "a", "b"]) - .fails() - .stderr_only("error to be defined"); + .args(&[param, "a_nul", "b_nul"]) + .succeeds() + .stdout_only_fixture("ab_nul.expected"); + } +} + +#[test] +fn zero_terminated_with_total() { + for param in ["-z", "--zero-terminated"] { + new_ucmd!() + .args(&[param, "--total", "a_nul", "b_nul"]) + .succeeds() + .stdout_only_fixture("ab_nul_total.expected"); } } diff --git a/tests/fixtures/comm/a_nul b/tests/fixtures/comm/a_nul new file mode 100644 index 0000000000000000000000000000000000000000..3142c801ccd54e1a9bc9d587e5ab16369ead768d GIT binary patch literal 3 KcmYdfr~&{1pa9$e literal 0 HcmV?d00001 diff --git a/tests/fixtures/comm/ab_nul.expected b/tests/fixtures/comm/ab_nul.expected new file mode 100644 index 0000000000000000000000000000000000000000..f826106bb1926f3ce2c6553b72bb0a007c432f4f GIT binary patch literal 9 QcmYdf;7nrR Date: Thu, 2 Mar 2023 14:23:46 +0100 Subject: [PATCH 65/94] GH action: be consistent in the install --- .github/workflows/GnuTests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml index fd2d66835..b74364f14 100644 --- a/.github/workflows/GnuTests.yml +++ b/.github/workflows/GnuTests.yml @@ -86,7 +86,7 @@ jobs: run: | ## Install dependencies sudo apt-get update - sudo apt-get install autoconf autopoint bison texinfo gperf gcc g++ gdb python3-pyinotify jq valgrind libexpect-perl + sudo apt-get install -y autoconf autopoint bison texinfo gperf gcc g++ gdb python3-pyinotify jq valgrind libexpect-perl - name: Add various locales shell: bash run: | @@ -316,8 +316,8 @@ jobs: - name: Install dependencies run: | ## Install dependencies - sudo apt update - sudo apt install autoconf autopoint bison texinfo gperf gcc g++ gdb python3-pyinotify jq valgrind libexpect-perl -y + sudo apt-get update + sudo apt-get install -y autoconf autopoint bison texinfo gperf gcc g++ gdb python3-pyinotify jq valgrind libexpect-perl - name: Add various locales run: | ## Add various locales From 794df2ae7a53dfec25e658d53515dc43cc9d93cd Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Thu, 2 Mar 2023 14:24:43 +0100 Subject: [PATCH 66/94] GH action: install missing dependencies Fails with: ``` 2023-03-01T22:02:40.2587471Z configure: WARNING: libacl development library was not found or not usable. 2023-03-01T22:02:40.2588131Z configure: WARNING: GNU coreutils will be built without ACL support. 2023-03-01T22:03:03.7621309Z configure: WARNING: libattr development library was not found or not usable. 2023-03-01T22:03:03.7621976Z configure: WARNING: GNU coreutils will be built without xattr support. 2023-03-01T22:03:04.6538269Z configure: WARNING: libcap library was not found or not usable. 2023-03-01T22:03:04.6539022Z configure: WARNING: GNU coreutils will be built without capability support. ``` --- .github/workflows/GnuTests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml index b74364f14..7c8c307ec 100644 --- a/.github/workflows/GnuTests.yml +++ b/.github/workflows/GnuTests.yml @@ -86,7 +86,7 @@ jobs: run: | ## Install dependencies sudo apt-get update - sudo apt-get install -y autoconf autopoint bison texinfo gperf gcc g++ gdb python3-pyinotify jq valgrind libexpect-perl + sudo apt-get install -y autoconf autopoint bison texinfo gperf gcc g++ gdb python3-pyinotify jq valgrind libexpect-perl libacl1-dev libattr1-dev libcap-dev - name: Add various locales shell: bash run: | @@ -317,7 +317,7 @@ jobs: run: | ## Install dependencies sudo apt-get update - sudo apt-get install -y autoconf autopoint bison texinfo gperf gcc g++ gdb python3-pyinotify jq valgrind libexpect-perl + sudo apt-get install -y autoconf autopoint bison texinfo gperf gcc g++ gdb python3-pyinotify jq valgrind libexpect-perl libacl1-dev libattr1-dev libcap-dev - name: Add various locales run: | ## Add various locales From a7a55c743349b842dc9403c04fd42fe567b56c2d Mon Sep 17 00:00:00 2001 From: Daniel Hofstetter Date: Thu, 2 Mar 2023 16:18:42 +0100 Subject: [PATCH 67/94] comm: allow multiple occurrence of -z --- src/uu/comm/src/comm.rs | 1 + tests/by-util/test_comm.rs | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/src/uu/comm/src/comm.rs b/src/uu/comm/src/comm.rs index 02a9221d4..26e704037 100644 --- a/src/uu/comm/src/comm.rs +++ b/src/uu/comm/src/comm.rs @@ -236,6 +236,7 @@ pub fn uu_app() -> Command { Arg::new(options::ZERO_TERMINATED) .long(options::ZERO_TERMINATED) .short('z') + .overrides_with(options::ZERO_TERMINATED) .help("line delimiter is NUL, not newline") .action(ArgAction::SetTrue), ) diff --git a/tests/by-util/test_comm.rs b/tests/by-util/test_comm.rs index aa0791f97..0ee5c44c2 100644 --- a/tests/by-util/test_comm.rs +++ b/tests/by-util/test_comm.rs @@ -105,6 +105,16 @@ fn zero_terminated() { } } +#[test] +fn zero_terminated_provided_multiple_times() { + for param in ["-z", "--zero-terminated"] { + new_ucmd!() + .args(&[param, param, param, "a_nul", "b_nul"]) + .succeeds() + .stdout_only_fixture("ab_nul.expected"); + } +} + #[test] fn zero_terminated_with_total() { for param in ["-z", "--zero-terminated"] { From 2bae3dd4f2c49ad44972df83bd2740ef36415dc6 Mon Sep 17 00:00:00 2001 From: Alexander Kunde Date: Thu, 2 Mar 2023 16:29:43 +0100 Subject: [PATCH 68/94] wc: move help strings to markdown file --- src/uu/wc/src/wc.rs | 7 +++---- src/uu/wc/wc.md | 9 +++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) create mode 100644 src/uu/wc/wc.md diff --git a/src/uu/wc/src/wc.rs b/src/uu/wc/src/wc.rs index 4a4838350..21c4643c6 100644 --- a/src/uu/wc/src/wc.rs +++ b/src/uu/wc/src/wc.rs @@ -15,7 +15,7 @@ use count_fast::{count_bytes_chars_and_lines_fast, count_bytes_fast}; use countable::WordCountable; use unicode_width::UnicodeWidthChar; use utf8::{BufReadDecoder, BufReadDecoderError}; -use uucore::{format_usage, show}; +use uucore::{format_usage, show, help_about, help_usage}; use word_count::{TitledWordCount, WordCount}; use clap::{crate_version, Arg, ArgAction, ArgMatches, Command}; @@ -98,9 +98,8 @@ impl Settings { } } -static ABOUT: &str = "Display newline, word, and byte counts for each FILE, and a total line if -more than one FILE is specified. With no FILE, or when FILE is -, read standard input."; -const USAGE: &str = "{} [OPTION]... [FILE]..."; +static ABOUT: &str = help_about!("wc.md"); +const USAGE: &str = help_usage!("wc.md"); pub mod options { pub static BYTES: &str = "bytes"; diff --git a/src/uu/wc/wc.md b/src/uu/wc/wc.md new file mode 100644 index 000000000..f1b243606 --- /dev/null +++ b/src/uu/wc/wc.md @@ -0,0 +1,9 @@ +# wc + +## Usage +``` +wc [OPTION]... [FILE]... +``` + +Display newline, word, and byte counts for each FILE, and a total line if +more than one FILE is specified. With no FILE, or when FILE is -, read standard input. From df5ceaaa97e4e291a31b1ab7645eaef17566777d Mon Sep 17 00:00:00 2001 From: Alexander Kunde Date: Thu, 2 Mar 2023 18:00:31 +0100 Subject: [PATCH 69/94] truncate: move help strings to markdown file --- src/uu/truncate/src/truncate.rs | 26 +++++--------------------- src/uu/truncate/truncate.md | 28 ++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 21 deletions(-) create mode 100644 src/uu/truncate/truncate.md diff --git a/src/uu/truncate/src/truncate.rs b/src/uu/truncate/src/truncate.rs index 090865313..c71f5079b 100644 --- a/src/uu/truncate/src/truncate.rs +++ b/src/uu/truncate/src/truncate.rs @@ -14,7 +14,7 @@ use std::os::unix::fs::FileTypeExt; use std::path::Path; use uucore::display::Quotable; use uucore::error::{FromIo, UResult, USimpleError, UUsageError}; -use uucore::format_usage; +use uucore::{format_usage, help_about, help_section, help_usage}; use uucore::parse_size::{parse_size, ParseSizeError}; #[derive(Debug, Eq, PartialEq)] @@ -73,25 +73,9 @@ impl TruncateMode { } } -const ABOUT: &str = "Shrink or extend the size of each file to the specified size."; -const USAGE: &str = "{} [OPTION]... [FILE]..."; -const LONG_USAGE: &str = "\ -SIZE is an integer with an optional prefix and optional unit. -The available units (K, M, G, T, P, E, Z, and Y) use the following format: - 'KB' => 1000 (kilobytes) - 'K' => 1024 (kibibytes) - 'MB' => 1000*1000 (megabytes) - 'M' => 1024*1024 (mebibytes) - 'GB' => 1000*1000*1000 (gigabytes) - 'G' => 1024*1024*1024 (gibibytes) -SIZE may also be prefixed by one of the following to adjust the size of each -file based on its current size: - '+' => extend by - '-' => reduce by - '<' => at most - '>' => at least - '/' => round down to multiple of - '%' => round up to multiple of"; +const ABOUT: &str = help_about!("truncate.md"); +const AFTER_HELP: &str = help_section!("after help", "truncate.md"); +const USAGE: &str = help_usage!("truncate.md"); pub mod options { pub static IO_BLOCKS: &str = "io-blocks"; @@ -104,7 +88,7 @@ pub mod options { #[uucore::main] pub fn uumain(args: impl uucore::Args) -> UResult<()> { let matches = uu_app() - .after_help(LONG_USAGE) + .after_help(AFTER_HELP) .try_get_matches_from(args) .map_err(|e| { e.print().expect("Error writing clap::Error"); diff --git a/src/uu/truncate/truncate.md b/src/uu/truncate/truncate.md new file mode 100644 index 000000000..64ab4e019 --- /dev/null +++ b/src/uu/truncate/truncate.md @@ -0,0 +1,28 @@ +# truncate + +## Usage + +```sh +truncate [OPTION]... [FILE]... +``` + +Shrink or extend the size of each file to the specified size. + +## After help + +SIZE is an integer with an optional prefix and optional unit. +The available units (K, M, G, T, P, E, Z, and Y) use the following format: + 'KB' => 1000 (kilobytes) + 'K' => 1024 (kibibytes) + 'MB' => 1000*1000 (megabytes) + 'M' => 1024*1024 (mebibytes) + 'GB' => 1000*1000*1000 (gigabytes) + 'G' => 1024*1024*1024 (gibibytes) +SIZE may also be prefixed by one of the following to adjust the size of each +file based on its current size: + '+' => extend by + '-' => reduce by + '<' => at most + '>' => at least + '/' => round down to multiple of + '%' => round up to multiple of \ No newline at end of file From 427611122229b08cb3f8f74ac318a36dceb82f0f Mon Sep 17 00:00:00 2001 From: Alexander Kunde Date: Thu, 2 Mar 2023 18:10:53 +0100 Subject: [PATCH 70/94] change ABOUT from static to const --- src/uu/wc/src/wc.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/uu/wc/src/wc.rs b/src/uu/wc/src/wc.rs index 21c4643c6..c1f4a2db1 100644 --- a/src/uu/wc/src/wc.rs +++ b/src/uu/wc/src/wc.rs @@ -98,7 +98,7 @@ impl Settings { } } -static ABOUT: &str = help_about!("wc.md"); +const ABOUT: &str = help_about!("wc.md"); const USAGE: &str = help_usage!("wc.md"); pub mod options { From 4bae3e0cd977f60a2cc17cc5ca483db4ed7d17c6 Mon Sep 17 00:00:00 2001 From: Alexander Kunde Date: Thu, 2 Mar 2023 18:14:10 +0100 Subject: [PATCH 71/94] remove redundant line --- src/uu/wc/wc.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/uu/wc/wc.md b/src/uu/wc/wc.md index f1b243606..db470a423 100644 --- a/src/uu/wc/wc.md +++ b/src/uu/wc/wc.md @@ -1,7 +1,6 @@ # wc -## Usage -``` +```sh wc [OPTION]... [FILE]... ``` From b73d5206f502a46d13dc1d51e1907547d2a2a862 Mon Sep 17 00:00:00 2001 From: Alexander Kunde Date: Thu, 2 Mar 2023 18:18:40 +0100 Subject: [PATCH 72/94] remove redundant line in markdown --- src/uu/truncate/truncate.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/uu/truncate/truncate.md b/src/uu/truncate/truncate.md index 64ab4e019..36e71e999 100644 --- a/src/uu/truncate/truncate.md +++ b/src/uu/truncate/truncate.md @@ -1,7 +1,5 @@ # truncate -## Usage - ```sh truncate [OPTION]... [FILE]... ``` From aead80efdbb5e87f2ba72817599a142212099ffb Mon Sep 17 00:00:00 2001 From: Alexander Kunde Date: Thu, 2 Mar 2023 18:55:11 +0100 Subject: [PATCH 73/94] cargo fmt --- src/uu/wc/src/wc.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/uu/wc/src/wc.rs b/src/uu/wc/src/wc.rs index c1f4a2db1..0b7b164a8 100644 --- a/src/uu/wc/src/wc.rs +++ b/src/uu/wc/src/wc.rs @@ -15,7 +15,7 @@ use count_fast::{count_bytes_chars_and_lines_fast, count_bytes_fast}; use countable::WordCountable; use unicode_width::UnicodeWidthChar; use utf8::{BufReadDecoder, BufReadDecoderError}; -use uucore::{format_usage, show, help_about, help_usage}; +use uucore::{format_usage, help_about, help_usage, show}; use word_count::{TitledWordCount, WordCount}; use clap::{crate_version, Arg, ArgAction, ArgMatches, Command}; From 73bb60d72490cefbf8926ca170a83720dd13bacd Mon Sep 17 00:00:00 2001 From: Alexander Kunde Date: Thu, 2 Mar 2023 18:56:08 +0100 Subject: [PATCH 74/94] cargo fmt --- src/uu/truncate/src/truncate.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/uu/truncate/src/truncate.rs b/src/uu/truncate/src/truncate.rs index c71f5079b..f050b52b4 100644 --- a/src/uu/truncate/src/truncate.rs +++ b/src/uu/truncate/src/truncate.rs @@ -14,8 +14,8 @@ use std::os::unix::fs::FileTypeExt; use std::path::Path; use uucore::display::Quotable; use uucore::error::{FromIo, UResult, USimpleError, UUsageError}; -use uucore::{format_usage, help_about, help_section, help_usage}; use uucore::parse_size::{parse_size, ParseSizeError}; +use uucore::{format_usage, help_about, help_section, help_usage}; #[derive(Debug, Eq, PartialEq)] enum TruncateMode { From d29119728af09da6cc9955f7832a3d55dd2bb758 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guilherme=20Gon=C3=A7alves?= Date: Tue, 28 Feb 2023 10:06:42 +0000 Subject: [PATCH 75/94] tests/du: run test_du_time with TZ=UTC. du --time formats a file's timestamp according to the local timezone, but the test implicitly assumed UTC. This caused it to fail when running locally in my UTC+1 machine. $ cargo test --features "du touch" --no-default-features Failure: https://gist.github.com/eggpi/651e01559b7c59e9457c1b22fe4c0c19 --- tests/by-util/test_du.rs | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/tests/by-util/test_du.rs b/tests/by-util/test_du.rs index 1232beda9..4a3cc3530 100644 --- a/tests/by-util/test_du.rs +++ b/tests/by-util/test_du.rs @@ -391,7 +391,12 @@ fn test_du_h_flag_empty_file() { fn test_du_time() { let ts = TestScenario::new(util_name!()); + // du --time formats the timestamp according to the local timezone. We set the TZ + // environment variable to UTC in the commands below to ensure consistent outputs + // and test results regardless of the timezone of the machine this test runs in. + ts.ccmd("touch") + .env("TZ", "UTC") .arg("-a") .arg("-t") .arg("201505150000") @@ -399,19 +404,35 @@ fn test_du_time() { .succeeds(); ts.ccmd("touch") + .env("TZ", "UTC") .arg("-m") .arg("-t") .arg("201606160000") .arg("date_test") .succeeds(); - let result = ts.ucmd().arg("--time").arg("date_test").succeeds(); + let result = ts + .ucmd() + .env("TZ", "UTC") + .arg("--time") + .arg("date_test") + .succeeds(); result.stdout_only("0\t2016-06-16 00:00\tdate_test\n"); - let result = ts.ucmd().arg("--time=atime").arg("date_test").succeeds(); + let result = ts + .ucmd() + .env("TZ", "UTC") + .arg("--time=atime") + .arg("date_test") + .succeeds(); result.stdout_only("0\t2015-05-15 00:00\tdate_test\n"); - let result = ts.ucmd().arg("--time=ctime").arg("date_test").succeeds(); + let result = ts + .ucmd() + .env("TZ", "UTC") + .arg("--time=ctime") + .arg("date_test") + .succeeds(); result.stdout_only("0\t2016-06-16 00:00\tdate_test\n"); if birth_supported() { From 9fb6477c895fc9bb590c8f088f46a89cd76b984e Mon Sep 17 00:00:00 2001 From: zleyyij <75810274+zleyyij@users.noreply.github.com> Date: Fri, 3 Mar 2023 04:37:09 -0700 Subject: [PATCH 76/94] chcon: move help strings to a markdown file (#4437) * arch: move help strings to a markdown file #4368 --------- Co-authored-by: zleyyij Co-authored-by: Sylvestre Ledru Co-authored-by: Terts Diepraam --- src/uu/chcon/chcon.md | 11 +++++++++++ src/uu/chcon/src/chcon.rs | 14 ++++---------- 2 files changed, 15 insertions(+), 10 deletions(-) create mode 100644 src/uu/chcon/chcon.md diff --git a/src/uu/chcon/chcon.md b/src/uu/chcon/chcon.md new file mode 100644 index 000000000..c4266be6e --- /dev/null +++ b/src/uu/chcon/chcon.md @@ -0,0 +1,11 @@ + +# chcon + +``` +chcon [OPTION]... CONTEXT FILE... +chcon [OPTION]... [-u USER] [-r ROLE] [-l RANGE] [-t TYPE] FILE... +chcon [OPTION]... --reference=RFILE FILE... +``` + +Change the SELinux security context of each FILE to CONTEXT. +With --reference, change the security context of each FILE to that of RFILE. \ No newline at end of file diff --git a/src/uu/chcon/src/chcon.rs b/src/uu/chcon/src/chcon.rs index 3acd2ea43..2f316393a 100644 --- a/src/uu/chcon/src/chcon.rs +++ b/src/uu/chcon/src/chcon.rs @@ -1,11 +1,9 @@ // spell-checker:ignore (vars) RFILE - #![allow(clippy::upper_case_acronyms)] use clap::builder::ValueParser; use uucore::error::{UResult, USimpleError, UUsageError}; -use uucore::format_usage; -use uucore::{display::Quotable, show_error, show_warning}; +use uucore::{display::Quotable, format_usage, help_about, help_usage, show_error, show_warning}; use clap::{Arg, ArgAction, Command}; use selinux::{OpaqueSecurityContext, SecurityContext}; @@ -21,13 +19,9 @@ mod fts; use errors::*; -static VERSION: &str = env!("CARGO_PKG_VERSION"); -static ABOUT: &str = "Change the SELinux security context of each FILE to CONTEXT. \n\ - With --reference, change the security context of each FILE to that of RFILE."; -const USAGE: &str = "\ - {} [OPTION]... CONTEXT FILE... \n \ - {} [OPTION]... [-u USER] [-r ROLE] [-l RANGE] [-t TYPE] FILE... \n \ - {} [OPTION]... --reference=RFILE FILE..."; +const VERSION: &str = env!("CARGO_PKG_VERSION"); +const ABOUT: &str = help_about!("chcon.md"); +const USAGE: &str = help_usage!("chcon.md"); pub mod options { pub static HELP: &str = "help"; From c35d1762aaf758452fbfaa68337446c490c55766 Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Fri, 3 Mar 2023 12:53:45 +0100 Subject: [PATCH 77/94] Add RFILE to the exclude list --- src/uu/chown/chown.md | 1 + 1 file changed, 1 insertion(+) diff --git a/src/uu/chown/chown.md b/src/uu/chown/chown.md index 11b653ea9..83101c74c 100644 --- a/src/uu/chown/chown.md +++ b/src/uu/chown/chown.md @@ -1,3 +1,4 @@ + # chown ``` From 53d3d4d616d81140923d9ce74cd99f1862cd16c7 Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Fri, 3 Mar 2023 13:12:49 +0100 Subject: [PATCH 78/94] remove the sh in the syntax --- src/uu/truncate/truncate.md | 1 - 1 file changed, 1 deletion(-) diff --git a/src/uu/truncate/truncate.md b/src/uu/truncate/truncate.md index 36e71e999..841c4b16c 100644 --- a/src/uu/truncate/truncate.md +++ b/src/uu/truncate/truncate.md @@ -1,6 +1,5 @@ # truncate -```sh truncate [OPTION]... [FILE]... ``` From 2085d4d4ab1ec1ca977129466c2872ff8fc8728f Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Fri, 3 Mar 2023 13:52:47 +0100 Subject: [PATCH 79/94] remove the sh in the syntax --- src/uu/wc/wc.md | 1 - 1 file changed, 1 deletion(-) diff --git a/src/uu/wc/wc.md b/src/uu/wc/wc.md index db470a423..3d0014b37 100644 --- a/src/uu/wc/wc.md +++ b/src/uu/wc/wc.md @@ -1,6 +1,5 @@ # wc -```sh wc [OPTION]... [FILE]... ``` From 422a27d3758f367615ea48d4b635f984b3242303 Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Fri, 3 Mar 2023 18:42:38 +0100 Subject: [PATCH 80/94] parent 9d5dc500e6c13e3923f13d3bc4231288d81846b8 author Sylvestre Ledru 1677865358 +0100 committer Sylvestre Ledru 1677951797 +0100 md: Fix a bunch of warnings in the docs --- CODE_OF_CONDUCT.md | 6 +- CONTRIBUTING.md | 9 +-- DEVELOPER_INSTRUCTIONS.md | 40 ++++------ README.md | 118 ++++++++++++++------------- docs/src/build.md | 2 +- docs/src/contributing.md | 4 +- docs/src/index.md | 4 + docs/src/installation.md | 13 ++- docs/src/multicall.md | 8 +- docs/src/test_coverage.md | 2 + src/uu/arch/arch.md | 3 - src/uu/base32/base32.md | 4 +- src/uu/base64/base64.md | 4 +- src/uu/chcon/chcon.md | 4 +- src/uu/cp/README.md | 6 +- src/uu/cut/BENCHMARKING.md | 33 ++++---- src/uu/dd/BENCHMARKING.md | 4 +- src/uu/dd/dd.md | 67 ++++++++-------- src/uu/dircolors/README.md | 16 ++-- src/uu/du/du.md | 6 +- src/uu/expr/expr.md | 16 ++-- src/uu/factor/BENCHMARKING.md | 8 +- src/uu/hashsum/BENCHMARKING.md | 10 ++- src/uu/head/BENCHMARKING.md | 26 ++++-- src/uu/join/BENCHMARKING.md | 35 ++++---- src/uu/ls/BENCHMARKING.md | 11 ++- src/uu/mkdir/mkdir.md | 3 +- src/uu/numfmt/numfmt.md | 22 ++--- src/uu/realpath/realpath.md | 2 +- src/uu/seq/BENCHMARKING.md | 12 ++- src/uu/shuf/BENCHMARKING.md | 6 +- src/uu/shuf/shuf.md | 2 +- src/uu/sleep/sleep.md | 2 +- src/uu/sort/BENCHMARKING.md | 142 ++++++++++++++++----------------- src/uu/split/BENCHMARKING.md | 22 +++-- src/uu/split/README.md | 3 +- src/uu/sum/BENCHMARKING.md | 8 +- src/uu/tac/BENCHMARKING.md | 29 ++++--- src/uu/tail/README.md | 59 +++++++++----- src/uu/truncate/truncate.md | 3 +- src/uu/wc/BENCHMARKING.md | 59 +++++++++----- src/uu/wc/wc.md | 1 + 42 files changed, 470 insertions(+), 364 deletions(-) diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index d196c6e95..39474f7ab 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -116,7 +116,7 @@ the community. This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.0, available at -https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. +. Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder](https://github.com/mozilla/diversity). @@ -124,5 +124,5 @@ enforcement ladder](https://github.com/mozilla/diversity). [homepage]: https://www.contributor-covenant.org For answers to common questions about this code of conduct, see the FAQ at -https://www.contributor-covenant.org/faq. Translations are available at -https://www.contributor-covenant.org/translations. +. Translations are available at +. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7a749ebac..f2e5763a1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,20 +38,19 @@ search the issues to make sure no one else is working on it. ## Platforms -We take pride in supporting many operating systems and architectures. +We take pride in supporting many operating systems and architectures. **Tip:** -For Windows, Microsoft provides some images (VMWare, Hyper-V, VirtualBox and Parallels) +For Windows, Microsoft provides some images (VMWare, Hyper-V, VirtualBox and Parallels) for development: -https://developer.microsoft.com/windows/downloads/virtual-machines/ - + ## Commit messages To help the project maintainers review pull requests from contributors across numerous utilities, the team has settled on conventions for commit messages. -From http://git-scm.com/book/ch5-2.html: +From : ``` Short (50 chars or less) summary of changes diff --git a/DEVELOPER_INSTRUCTIONS.md b/DEVELOPER_INSTRUCTIONS.md index 28deb2677..f7dfc689e 100644 --- a/DEVELOPER_INSTRUCTIONS.md +++ b/DEVELOPER_INSTRUCTIONS.md @@ -1,21 +1,19 @@ -Documentation -------------- +# Documentation The source of the documentation is available on: -https://uutils.github.io/dev/coreutils/ + The documentation is updated everyday on this repository: -https://github.com/uutils/uutils.github.io/ + -Running GNU tests ------------------ +## Running GNU tests -- Check out https://github.com/coreutils/coreutils next to your fork as gnu -- Check out https://github.com/coreutils/gnulib next to your fork as gnulib +- Check out next to your fork as gnu +- Check out next to your fork as gnulib - Rename the checkout of your fork to uutils At the end you should have uutils, gnu and gnulib checked out next to each other. @@ -23,9 +21,7 @@ At the end you should have uutils, gnu and gnulib checked out next to each other - Run `cd uutils && ./util/build-gnu.sh && cd ..` to get everything ready (this may take a while) - Finally, you can run tests with `bash uutils/util/run-gnu-test.sh `. Instead of `` insert the tests you want to run, e.g. `tests/misc/wc-proc.sh`. - -Code Coverage Report Generation ---------------------------------- +## Code Coverage Report Generation @@ -36,13 +32,13 @@ Code coverage report can be generated using [grcov](https://github.com/mozilla/g To generate [gcov-based](https://github.com/mozilla/grcov#example-how-to-generate-gcda-files-for-a-rust-project) coverage report ```bash -$ export CARGO_INCREMENTAL=0 -$ export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort" -$ export RUSTDOCFLAGS="-Cpanic=abort" -$ cargo build # e.g., --features feat_os_unix -$ cargo test # e.g., --features feat_os_unix test_pathchk -$ grcov . -s . --binary-path ./target/debug/ -t html --branch --ignore-not-existing --ignore build.rs --excl-br-line "^\s*((debug_)?assert(_eq|_ne)?\#\[derive\()" -o ./target/debug/coverage/ -$ # open target/debug/coverage/index.html in browser +export CARGO_INCREMENTAL=0 +export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort" +export RUSTDOCFLAGS="-Cpanic=abort" +cargo build # e.g., --features feat_os_unix +cargo test # e.g., --features feat_os_unix test_pathchk +grcov . -s . --binary-path ./target/debug/ -t html --branch --ignore-not-existing --ignore build.rs --excl-br-line "^\s*((debug_)?assert(_eq|_ne)?\#\[derive\()" -o ./target/debug/coverage/ +# open target/debug/coverage/index.html in browser ``` if changes are not reflected in the report then run `cargo clean` and run the above commands. @@ -52,19 +48,17 @@ if changes are not reflected in the report then run `cargo clean` and run the ab If you are using stable version of Rust that doesn't enable code coverage instrumentation by default then add `-Z-Zinstrument-coverage` flag to `RUSTFLAGS` env variable specified above. - -pre-commit hooks ----------------- +## pre-commit hooks A configuration for `pre-commit` is provided in the repository. It allows automatically checking every git commit you make to ensure it compiles, and passes `clippy` and `rustfmt` without warnings. To use the provided hook: 1. [Install `pre-commit`](https://pre-commit.com/#install) -2. Run `pre-commit install` while in the repository directory +1. Run `pre-commit install` while in the repository directory Your git commits will then automatically be checked. If a check fails, an error message will explain why, and your commit will be canceled. You can then make the suggested changes, and run `git commit ...` again. -### Using Clippy +## Using Clippy The `msrv` key in the clippy configuration file `clippy.toml` is used to disable lints pertaining to newer features by specifying the minimum supported Rust version (MSRV). However, this key is only supported on `nightly`. To invoke clippy without errors, use `cargo +nightly clippy`. In order to also check tests and non-default crate features, use `cargo +nightly clippy --all-targets --all-features`. diff --git a/README.md b/README.md index 66a0395e9..1bd87401c 100644 --- a/README.md +++ b/README.md @@ -21,11 +21,12 @@ or different behavior might be experienced. To install it: -``` -$ cargo install coreutils -$ ~/.cargo/bin/coreutils +```bash +cargo install coreutils +~/.cargo/bin/coreutils ``` + ## Why? uutils aims to work on as many platforms as possible, to be able to use the @@ -35,6 +36,7 @@ chosen not only because it is fast and safe, but is also excellent for writing cross-platform code. ## Documentation + uutils has both user and developer documentation available: - [User Manual](https://uutils.github.io/user/) @@ -46,8 +48,8 @@ Both can also be generated locally, the instructions for that can be found in th ## Requirements -* Rust (`cargo`, `rustc`) -* GNU Make (optional) +- Rust (`cargo`, `rustc`) +- GNU Make (optional) ### Rust Version @@ -65,8 +67,8 @@ or GNU Make. For either method, we first need to fetch the repository: ```bash -$ git clone https://github.com/uutils/coreutils -$ cd coreutils +git clone https://github.com/uutils/coreutils +cd coreutils ``` ### Cargo @@ -75,7 +77,7 @@ Building uutils using Cargo is easy because the process is the same as for every other Rust program: ```bash -$ cargo build --release +cargo build --release ``` This command builds the most portable common core set of uutils into a multicall @@ -86,11 +88,11 @@ expanded sets of uutils for a platform (on that platform) is as simple as specifying it as a feature: ```bash -$ cargo build --release --features macos +cargo build --release --features macos # or ... -$ cargo build --release --features windows +cargo build --release --features windows # or ... -$ cargo build --release --features unix +cargo build --release --features unix ``` If you don't want to build every utility available on your platform into the @@ -98,7 +100,7 @@ final binary, you can also specify which ones you want to build manually. For example: ```bash -$ cargo build --features "base32 cat echo rm" --no-default-features +cargo build --features "base32 cat echo rm" --no-default-features ``` If you don't want to build the multicall binary and would prefer to build @@ -108,7 +110,7 @@ is contained in its own package within the main repository, named specific packages (using the `--package` [aka `-p`] option). For example: ```bash -$ cargo build -p uu_base32 -p uu_cat -p uu_echo -p uu_rm +cargo build -p uu_base32 -p uu_cat -p uu_echo -p uu_rm ``` ### GNU Make @@ -118,29 +120,29 @@ Building using `make` is a simple process as well. To simply build all available utilities: ```bash -$ make +make ``` To build all but a few of the available utilities: ```bash -$ make SKIP_UTILS='UTILITY_1 UTILITY_2' +make SKIP_UTILS='UTILITY_1 UTILITY_2' ``` To build only a few of the available utilities: ```bash -$ make UTILS='UTILITY_1 UTILITY_2' +make UTILS='UTILITY_1 UTILITY_2' ``` ## Installation -### Cargo +### Install with Cargo Likewise, installing can simply be done using: ```bash -$ cargo install --path . +cargo install --path . ``` This command will install uutils into Cargo's *bin* folder (*e.g.* `$HOME/.cargo/bin`). @@ -148,49 +150,49 @@ This command will install uutils into Cargo's *bin* folder (*e.g.* `$HOME/.cargo This does not install files necessary for shell completion. For shell completion to work, use `GNU Make` or see `Manually install shell completions`. -### GNU Make +### Install with GNU Make To install all available utilities: ```bash -$ make install +make install ``` To install using `sudo` switch `-E` must be used: ```bash -$ sudo -E make install +sudo -E make install ``` To install all but a few of the available utilities: ```bash -$ make SKIP_UTILS='UTILITY_1 UTILITY_2' install +make SKIP_UTILS='UTILITY_1 UTILITY_2' install ``` To install only a few of the available utilities: ```bash -$ make UTILS='UTILITY_1 UTILITY_2' install +make UTILS='UTILITY_1 UTILITY_2' install ``` To install every program with a prefix (e.g. uu-echo uu-cat): ```bash -$ make PROG_PREFIX=PREFIX_GOES_HERE install +make PROG_PREFIX=PREFIX_GOES_HERE install ``` To install the multicall binary: ```bash -$ make MULTICALL=y install +make MULTICALL=y install ``` Set install parent directory (default value is /usr/local): ```bash # DESTDIR is also supported -$ make PREFIX=/my/path install +make PREFIX=/my/path install ``` Installing with `make` installs shell completions for all installed utilities @@ -203,6 +205,7 @@ The `coreutils` binary can generate completions for the `bash`, `elvish`, `fish` and `zsh` shells. It prints the result to stdout. The syntax is: + ```bash cargo run completion ``` @@ -220,106 +223,107 @@ Un-installation differs depending on how you have installed uutils. If you used Cargo to install, use Cargo to uninstall. If you used GNU Make to install, use Make to uninstall. -### Cargo +### Uninstall with Cargo To uninstall uutils: ```bash -$ cargo uninstall uutils +cargo uninstall uutils ``` -### GNU Make +### Uninstall with GNU Make To uninstall all utilities: ```bash -$ make uninstall +make uninstall ``` To uninstall every program with a set prefix: ```bash -$ make PROG_PREFIX=PREFIX_GOES_HERE uninstall +make PROG_PREFIX=PREFIX_GOES_HERE uninstall ``` To uninstall the multicall binary: ```bash -$ make MULTICALL=y uninstall +make MULTICALL=y uninstall ``` To uninstall from a custom parent directory: ```bash # DESTDIR is also supported -$ make PREFIX=/my/path uninstall +make PREFIX=/my/path uninstall ``` + ## Testing Testing can be done using either Cargo or `make`. -### Cargo +### Testing with Cargo Just like with building, we follow the standard procedure for testing using Cargo: ```bash -$ cargo test +cargo test ``` By default, `cargo test` only runs the common programs. To run also platform specific tests, run: ```bash -$ cargo test --features unix +cargo test --features unix ``` If you would prefer to test a select few utilities: ```bash -$ cargo test --features "chmod mv tail" --no-default-features +cargo test --features "chmod mv tail" --no-default-features ``` If you also want to test the core utilities: ```bash -$ cargo test -p uucore -p coreutils +cargo test -p uucore -p coreutils ``` To debug: ```bash -$ gdb --args target/debug/coreutils ls +gdb --args target/debug/coreutils ls (gdb) b ls.rs:79 (gdb) run ``` -### GNU Make +### Testing with GNU Make To simply test all available utilities: ```bash -$ make test +make test ``` To test all but a few of the available utilities: ```bash -$ make SKIP_UTILS='UTILITY_1 UTILITY_2' test +make SKIP_UTILS='UTILITY_1 UTILITY_2' test ``` To test only a few of the available utilities: ```bash -$ make UTILS='UTILITY_1 UTILITY_2' test +make UTILS='UTILITY_1 UTILITY_2' test ``` To include tests for unimplemented behavior: ```bash -$ make UTILS='UTILITY_1 UTILITY_2' SPEC=y test +make UTILS='UTILITY_1 UTILITY_2' SPEC=y test ``` ### Run Busybox Tests @@ -330,19 +334,19 @@ requires `make`. To run busybox tests for all utilities for which busybox has tests ```bash -$ make busytest +make busytest ``` To run busybox tests for a few of the available utilities ```bash -$ make UTILS='UTILITY_1 UTILITY_2' busytest +make UTILS='UTILITY_1 UTILITY_2' busytest ``` To pass an argument like "-v" to the busybox test runtime ```bash -$ make UTILS='UTILITY_1 UTILITY_2' RUNTEST_ARGS='-v' busytest +make UTILS='UTILITY_1 UTILITY_2' RUNTEST_ARGS='-v' busytest ``` ### Comparing with GNU @@ -356,14 +360,14 @@ breakdown of the GNU test results of the main branch can be found To run locally: ```bash -$ bash util/build-gnu.sh -$ bash util/run-gnu-test.sh +bash util/build-gnu.sh +bash util/run-gnu-test.sh # To run a single test: -$ bash util/run-gnu-test.sh tests/touch/not-owner.sh # for example +bash util/run-gnu-test.sh tests/touch/not-owner.sh # for example # To run several tests: -$ bash util/run-gnu-test.sh tests/touch/not-owner.sh tests/rm/no-give-up.sh # for example +bash util/run-gnu-test.sh tests/touch/not-owner.sh tests/rm/no-give-up.sh # for example # If this is a perl (.pl) test, to run in debug: -$ DEBUG=1 bash util/run-gnu-test.sh tests/misc/sm3sum.pl +DEBUG=1 bash util/run-gnu-test.sh tests/misc/sm3sum.pl ``` Note that it relies on individual utilities (not the multicall binary). @@ -387,7 +391,6 @@ To improve the GNU compatibility, the following process is recommended: 1. Start to modify the Rust implementation to match the expected behavior 1. Add a test to make sure that we don't regress (our test suite is super quick) - ## Contributing To contribute to uutils, please see [CONTRIBUTING](CONTRIBUTING.md). @@ -395,11 +398,12 @@ To contribute to uutils, please see [CONTRIBUTING](CONTRIBUTING.md). ## Utilities Please note that this is not fully accurate: -* Some new options can be added / removed in the GNU implementation; -* Some error management might be missing; -* Some behaviors might be different. -See https://github.com/uutils/coreutils/issues/3336 for the main meta bugs +- Some new options can be added / removed in the GNU implementation; +- Some error management might be missing; +- Some behaviors might be different. + +See for the main meta bugs (many are missing). | Done | WIP | diff --git a/docs/src/build.md b/docs/src/build.md index 6505b5b6e..e35b0ebe8 100644 --- a/docs/src/build.md +++ b/docs/src/build.md @@ -1,3 +1,3 @@ # Build from source -{{#include ../../README.md:build }} \ No newline at end of file +{{#include ../../README.md:build }} diff --git a/docs/src/contributing.md b/docs/src/contributing.md index 79ef4d13b..f69e1b377 100644 --- a/docs/src/contributing.md +++ b/docs/src/contributing.md @@ -1 +1,3 @@ -{{ #include ../../CONTRIBUTING.md }} \ No newline at end of file + + +{{ #include ../../CONTRIBUTING.md }} diff --git a/docs/src/index.md b/docs/src/index.md index 7bac8b816..0212b7526 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -1,5 +1,9 @@ + + {{#include logo.svg}} + +