From f406b56f4a76488346da9ff10265f8e7ea0a208e Mon Sep 17 00:00:00 2001 From: Miles Liu Date: Wed, 1 Feb 2023 15:39:07 +0800 Subject: [PATCH 01/10] timeout: fix subprocess is never terminated --- src/uu/timeout/src/timeout.rs | 33 +++++++++++++++++++------- src/uucore/src/lib/features/process.rs | 15 ++++++++++++ tests/by-util/test_timeout.rs | 16 +++++++++++++ 3 files changed, 55 insertions(+), 9 deletions(-) diff --git a/src/uu/timeout/src/timeout.rs b/src/uu/timeout/src/timeout.rs index 26235e0df..fe342727b 100644 --- a/src/uu/timeout/src/timeout.rs +++ b/src/uu/timeout/src/timeout.rs @@ -196,6 +196,22 @@ fn report_if_verbose(signal: usize, cmd: &str, verbose: bool) { } } +fn send_signal(process: &mut Child, signal: usize, foreground: bool) { + // NOTE: GNU timeout doesn't check for errors of signal. + // The subprocess might have exited just after the timeout. + // Sending a signal now would return "No such process", but we should still try to kill the children. + _ = process.send_signal(signal); + if !foreground { + _ = process.send_signal_group(signal); + let kill_signal = signal_by_name_or_value("KILL").unwrap(); + let continued_signal = signal_by_name_or_value("CONT").unwrap(); + if signal != kill_signal && signal != continued_signal { + _ = process.send_signal(continued_signal); + _ = process.send_signal_group(continued_signal); + } + } +} + /// Wait for a child process and send a kill signal if it does not terminate. /// /// This function waits for the child `process` for the time period @@ -217,10 +233,11 @@ fn report_if_verbose(signal: usize, cmd: &str, verbose: bool) { /// If there is a problem sending the `SIGKILL` signal or waiting for /// the process after that signal is sent. fn wait_or_kill_process( - mut process: Child, + process: &mut Child, cmd: &str, duration: Duration, preserve_status: bool, + foreground: bool, verbose: bool, ) -> std::io::Result { match process.wait_or_timeout(duration) { @@ -234,7 +251,7 @@ fn wait_or_kill_process( Ok(None) => { let signal = signal_by_name_or_value("KILL").unwrap(); report_if_verbose(signal, cmd, verbose); - process.send_signal(signal)?; + send_signal(process, signal, foreground); process.wait()?; Ok(ExitStatus::SignalSent(signal).into()) } @@ -300,7 +317,7 @@ fn timeout( enable_pipe_errors()?; - let mut process = process::Command::new(&cmd[0]) + let process = &mut process::Command::new(&cmd[0]) .args(&cmd[1..]) .stdin(Stdio::inherit()) .stdout(Stdio::inherit()) @@ -335,7 +352,7 @@ fn timeout( .into()), Ok(None) => { report_if_verbose(signal, &cmd[0], verbose); - process.send_signal(signal)?; + send_signal(process, signal, foreground); match kill_after { None => { if preserve_status { @@ -350,6 +367,7 @@ fn timeout( &cmd[0], kill_after, preserve_status, + foreground, verbose, ) { Ok(status) => Err(status.into()), @@ -363,11 +381,8 @@ fn timeout( } Err(_) => { // We're going to return ERR_EXIT_STATUS regardless of - // whether `send_signal()` succeeds or fails, so just - // ignore the return value. - process - .send_signal(signal) - .map_err(|e| USimpleError::new(ExitStatus::TimeoutFailed.into(), format!("{e}")))?; + // whether `send_signal()` succeeds or fails + send_signal(process, signal, foreground); Err(ExitStatus::TimeoutFailed.into()) } } diff --git a/src/uucore/src/lib/features/process.rs b/src/uucore/src/lib/features/process.rs index 933b1f21b..b8b2f178f 100644 --- a/src/uucore/src/lib/features/process.rs +++ b/src/uucore/src/lib/features/process.rs @@ -48,6 +48,9 @@ pub trait ChildExt { /// send the signal to an unrelated process that recycled the PID. fn send_signal(&mut self, signal: usize) -> io::Result<()>; + /// Send a signal to a process group. + fn send_signal_group(&mut self, signal: usize) -> io::Result<()>; + /// Wait for a process to finish or return after the specified duration. /// A `timeout` of zero disables the timeout. fn wait_or_timeout(&mut self, timeout: Duration) -> io::Result>; @@ -62,6 +65,18 @@ impl ChildExt for Child { } } + fn send_signal_group(&mut self, signal: usize) -> io::Result<()> { + // Ignore the signal, so we don't go into a signal loop. + if unsafe { libc::signal(signal as i32, libc::SIG_IGN) } != 0 { + return Err(io::Error::last_os_error()); + } + if unsafe { libc::kill(0, signal as i32) } != 0 { + Err(io::Error::last_os_error()) + } else { + Ok(()) + } + } + fn wait_or_timeout(&mut self, timeout: Duration) -> io::Result> { if timeout == Duration::from_micros(0) { return self.wait().map(Some); diff --git a/tests/by-util/test_timeout.rs b/tests/by-util/test_timeout.rs index deedea6c7..eb6c99211 100644 --- a/tests/by-util/test_timeout.rs +++ b/tests/by-util/test_timeout.rs @@ -138,3 +138,19 @@ fn test_kill_after_long() { .no_stdout() .no_stderr(); } + +#[test] +fn test_kill_subprocess() { + new_ucmd!() + .args(&[ + // Make sure the CI can spawn the subprocess. + "10", + "sh", + "-c", + "sh -c \"trap 'echo xyz' TERM; sleep 30\"", + ]) + .fails() + .code_is(124) + .stdout_contains("xyz") + .stderr_contains("Terminated"); +} From 08d2492cac10e71932c4be13ab093da5c5010ac3 Mon Sep 17 00:00:00 2001 From: Terts Diepraam Date: Tue, 14 Mar 2023 17:09:49 +0100 Subject: [PATCH 02/10] README: add logo and center title and logo --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 97e115a6a..296413bec 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,7 @@ +
+ +![uutils logo](docs/src/logo.svg) + # uutils coreutils [![Crates.io](https://img.shields.io/crates/v/coreutils.svg)](https://crates.io/crates/coreutils) @@ -9,7 +13,9 @@ [![CodeCov](https://codecov.io/gh/uutils/coreutils/branch/master/graph/badge.svg)](https://codecov.io/gh/uutils/coreutils) ![MSRV](https://img.shields.io/badge/MSRV-1.64.0-brightgreen) ------------------------------------------------ +
+ +--- From c83f6eeadd07c7c54a1ffad394e9a53a5ef4bc2d Mon Sep 17 00:00:00 2001 From: Terts Diepraam Date: Tue, 14 Mar 2023 17:16:10 +0100 Subject: [PATCH 03/10] Move testing instructions from README to CONTRIBUTING --- CONTRIBUTING.md | 130 ++++++++++++++++++++++++++++++++++++++++++++++++ README.md | 125 +--------------------------------------------- 2 files changed, 131 insertions(+), 124 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f658dbf81..9af1948fe 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -47,6 +47,136 @@ For Windows, Microsoft provides some images (VMWare, Hyper-V, VirtualBox and Par for development: +## Testing + +Testing can be done using either Cargo or `make`. + +### Testing with Cargo + +Just like with building, we follow the standard procedure for testing using +Cargo: + +```shell +cargo test +``` + +By default, `cargo test` only runs the common programs. To run also platform +specific tests, run: + +```shell +cargo test --features unix +``` + +If you would prefer to test a select few utilities: + +```shell +cargo test --features "chmod mv tail" --no-default-features +``` + +If you also want to test the core utilities: + +```shell +cargo test -p uucore -p coreutils +``` + +To debug: + +```shell +gdb --args target/debug/coreutils ls +(gdb) b ls.rs:79 +(gdb) run +``` + +### Testing with GNU Make + +To simply test all available utilities: + +```shell +make test +``` + +To test all but a few of the available utilities: + +```shell +make SKIP_UTILS='UTILITY_1 UTILITY_2' test +``` + +To test only a few of the available utilities: + +```shell +make UTILS='UTILITY_1 UTILITY_2' test +``` + +To include tests for unimplemented behavior: + +```shell +make UTILS='UTILITY_1 UTILITY_2' SPEC=y test +``` + +### Run Busybox Tests + +This testing functionality is only available on *nix operating systems and +requires `make`. + +To run busybox tests for all utilities for which busybox has tests + +```shell +make busytest +``` + +To run busybox tests for a few of the available utilities + +```shell +make UTILS='UTILITY_1 UTILITY_2' busytest +``` + +To pass an argument like "-v" to the busybox test runtime + +```shell +make UTILS='UTILITY_1 UTILITY_2' RUNTEST_ARGS='-v' busytest +``` + +### Comparing with GNU + +To run uutils against the GNU test suite locally, run the following commands: + +```shell +bash util/build-gnu.sh +bash util/run-gnu-test.sh +# To run a single test: +bash util/run-gnu-test.sh tests/touch/not-owner.sh # for example +# To run several tests: +bash util/run-gnu-test.sh tests/touch/not-owner.sh tests/rm/no-give-up.sh # for example +# If this is a perl (.pl) test, to run in debug: +DEBUG=1 bash util/run-gnu-test.sh tests/misc/sm3sum.pl +``` + +Note that it relies on individual utilities (not the multicall binary). + +### Improving the GNU compatibility + +The Python script `./util/remaining-gnu-error.py` shows the list of failing +tests in the CI. + +To improve the GNU compatibility, the following process is recommended: + +1. Identify a test (the smaller, the better) on a program that you understand or + is easy to understand. You can use the `./util/remaining-gnu-error.py` script + to help with this decision. +1. Build both the GNU and Rust coreutils using: `bash util/build-gnu.sh` +1. Run the test with `bash util/run-gnu-test.sh ` +1. Start to modify `` to understand what is wrong. Examples: + 1. Add `set -v` to have the bash verbose mode + 1. Add `echo $?` where needed + 1. When the variable `fail` is used in the test, `echo $fail` to see when the + test started to fail + 1. Bump the content of the output (ex: `cat err`) + 1. ... +1. Or, if the test is simple, extract the relevant information to create a new + test case running both GNU & Rust implementation +1. Start to modify the Rust implementation to match the expected behavior +1. Add a test to make sure that we don't regress (our test suite is super quick) + ## Commit messages To help the project maintainers review pull requests from contributors across diff --git a/README.md b/README.md index 296413bec..abf18255a 100644 --- a/README.md +++ b/README.md @@ -286,96 +286,7 @@ make PREFIX=/my/path uninstall -## Testing - -Testing can be done using either Cargo or `make`. - -### Testing with Cargo - -Just like with building, we follow the standard procedure for testing using -Cargo: - -```shell -cargo test -``` - -By default, `cargo test` only runs the common programs. To run also platform -specific tests, run: - -```shell -cargo test --features unix -``` - -If you would prefer to test a select few utilities: - -```shell -cargo test --features "chmod mv tail" --no-default-features -``` - -If you also want to test the core utilities: - -```shell -cargo test -p uucore -p coreutils -``` - -To debug: - -```shell -gdb --args target/debug/coreutils ls -(gdb) b ls.rs:79 -(gdb) run -``` - -### Testing with GNU Make - -To simply test all available utilities: - -```shell -make test -``` - -To test all but a few of the available utilities: - -```shell -make SKIP_UTILS='UTILITY_1 UTILITY_2' test -``` - -To test only a few of the available utilities: - -```shell -make UTILS='UTILITY_1 UTILITY_2' test -``` - -To include tests for unimplemented behavior: - -```shell -make UTILS='UTILITY_1 UTILITY_2' SPEC=y test -``` - -### Run Busybox Tests - -This testing functionality is only available on *nix operating systems and -requires `make`. - -To run busybox tests for all utilities for which busybox has tests - -```shell -make busytest -``` - -To run busybox tests for a few of the available utilities - -```shell -make UTILS='UTILITY_1 UTILITY_2' busytest -``` - -To pass an argument like "-v" to the busybox test runtime - -```shell -make UTILS='UTILITY_1 UTILITY_2' RUNTEST_ARGS='-v' busytest -``` - -### Comparing with GNU +## GNU test suite compatibility Below is the evolution of how many GNU tests uutils passes. A more detailed breakdown of the GNU test results of the main branch can be found @@ -383,40 +294,6 @@ breakdown of the GNU test results of the main branch can be found ![Evolution over time](https://github.com/uutils/coreutils-tracking/blob/main/gnu-results.png?raw=true) -To run locally: - -```shell -bash util/build-gnu.sh -bash util/run-gnu-test.sh -# To run a single test: -bash util/run-gnu-test.sh tests/touch/not-owner.sh # for example -# To run several tests: -bash util/run-gnu-test.sh tests/touch/not-owner.sh tests/rm/no-give-up.sh # for example -# If this is a perl (.pl) test, to run in debug: -DEBUG=1 bash util/run-gnu-test.sh tests/misc/sm3sum.pl -``` - -Note that it relies on individual utilities (not the multicall binary). - -### Improving the GNU compatibility - -The Python script `./util/remaining-gnu-error.py` shows the list of failing tests in the CI. - -To improve the GNU compatibility, the following process is recommended: - -1. Identify a test (the smaller, the better) on a program that you understand or is easy to understand. You can use the `./util/remaining-gnu-error.py` script to help with this decision. -1. Build both the GNU and Rust coreutils using: `bash util/build-gnu.sh` -1. Run the test with `bash util/run-gnu-test.sh ` -1. Start to modify `` to understand what is wrong. Examples: - 1. Add `set -v` to have the bash verbose mode - 1. Add `echo $?` where needed - 1. When the variable `fail` is used in the test, `echo $fail` to see when the test started to fail - 1. Bump the content of the output (ex: `cat err`) - 1. ... -1. Or, if the test is simple, extract the relevant information to create a new test case running both GNU & Rust implementation -1. Start to modify the Rust implementation to match the expected behavior -1. Add a test to make sure that we don't regress (our test suite is super quick) - ## Contributing To contribute to uutils, please see [CONTRIBUTING](CONTRIBUTING.md). From 04c7d2f60db7fa51bbd9f08067c9c3b66ea0c41d Mon Sep 17 00:00:00 2001 From: Terts Diepraam Date: Tue, 14 Mar 2023 17:19:12 +0100 Subject: [PATCH 04/10] README: rewrite description - Remove "attempt", I think we're good enough to call it more than an attempt - Make it more specifically about the GNU coreutils. --- README.md | 266 +++++++++++++++++++++++++++--------------------------- 1 file changed, 135 insertions(+), 131 deletions(-) diff --git a/README.md b/README.md index abf18255a..d2351417f 100644 --- a/README.md +++ b/README.md @@ -20,10 +20,9 @@ -uutils is an attempt at writing universal (as in cross-platform) CLI -utilities in [Rust](http://www.rust-lang.org). -While all programs have been implemented, some options might be missing -or different behavior might be experienced. +uutils coreutils is a cross-platform reimplementation of the GNU coreutils in +[Rust](http://www.rust-lang.org). While all programs have been implemented, some +options might be missing or different behavior might be experienced. To install it: @@ -33,13 +32,15 @@ cargo install coreutils ``` -## Why? -uutils aims to work on as many platforms as possible, to be able to use the -same utils on Linux, Mac, Windows and other platforms. This ensures, for -example, that scripts can be easily transferred between platforms. Rust was -chosen not only because it is fast and safe, but is also excellent for -writing cross-platform code. +## Goals + +uutils aims to be a drop-in replacement for the GNU utils. Differences with GNU +are treated as bugs. + +uutils aims to work on as many platforms as possible, to be able to use the same +utils on Linux, Mac, Windows and other platforms. This ensures, for example, +that scripts can be easily transferred between platforms. ## Documentation @@ -48,10 +49,11 @@ uutils has both user and developer documentation available: - [User Manual](https://uutils.github.io/user/) - [Developer Documentation](https://uutils.github.io/dev/coreutils/) -Both can also be generated locally, the instructions for that can be found in the -[coreutils docs](https://github.com/uutils/uutils.github.io) repository. +Both can also be generated locally, the instructions for that can be found in +the [coreutils docs](https://github.com/uutils/uutils.github.io) repository. + ## Requirements - Rust (`cargo`, `rustc`) @@ -59,13 +61,13 @@ Both can also be generated locally, the instructions for that can be found in th ### Rust Version -uutils follows Rust's release channels and is tested against stable, beta and nightly. -The current Minimum Supported Rust Version (MSRV) is `1.64.0`. +uutils follows Rust's release channels and is tested against stable, beta and +nightly. The current Minimum Supported Rust Version (MSRV) is `1.64.0`. ## Building -There are currently two methods to build the uutils binaries: either Cargo -or GNU Make. +There are currently two methods to build the uutils binaries: either Cargo or +GNU Make. > Building the full package, including all documentation, requires both Cargo > and Gnu Make on a Unix platform. @@ -79,8 +81,8 @@ cd coreutils ### Cargo -Building uutils using Cargo is easy because the process is the same as for -every other Rust program: +Building uutils using Cargo is easy because the process is the same as for every +other Rust program: ```shell cargo build --release @@ -89,9 +91,9 @@ cargo build --release This command builds the most portable common core set of uutils into a multicall (BusyBox-type) binary, named 'coreutils', on most Rust-supported platforms. -Additional platform-specific uutils are often available. Building these -expanded sets of uutils for a platform (on that platform) is as simple as -specifying it as a feature: +Additional platform-specific uutils are often available. Building these expanded +sets of uutils for a platform (on that platform) is as simple as specifying it +as a feature: ```shell cargo build --release --features macos @@ -102,18 +104,18 @@ cargo build --release --features unix ``` If you don't want to build every utility available on your platform into the -final binary, you can also specify which ones you want to build manually. -For example: +final binary, you can also specify which ones you want to build manually. For +example: ```shell cargo build --features "base32 cat echo rm" --no-default-features ``` -If you don't want to build the multicall binary and would prefer to build -the utilities as individual binaries, that is also possible. Each utility -is contained in its own package within the main repository, named -"uu_UTILNAME". To build individual utilities, use cargo to build just the -specific packages (using the `--package` [aka `-p`] option). For example: +If you don't want to build the multicall binary and would prefer to build the +utilities as individual binaries, that is also possible. Each utility is +contained in its own package within the main repository, named "uu_UTILNAME". To +build individual utilities, use cargo to build just the specific packages (using +the `--package` [aka `-p`] option). For example: ```shell cargo build -p uu_base32 -p uu_cat -p uu_echo -p uu_rm @@ -157,10 +159,12 @@ Likewise, installing can simply be done using: cargo install --path . --locked ``` -This command will install uutils into Cargo's *bin* folder (*e.g.* `$HOME/.cargo/bin`). +This command will install uutils into Cargo's _bin_ folder (_e.g._ +`$HOME/.cargo/bin`). -This does not install files necessary for shell completion or manpages. -For manpages or shell completion to work, use `GNU Make` or see `Manually install shell completions`/`Manually install manpages`. +This does not install files necessary for shell completion or manpages. For +manpages or shell completion to work, use `GNU Make` or see +`Manually install shell completions`/`Manually install manpages`. ### Install with GNU Make @@ -213,8 +217,8 @@ be generated; See `Manually install shell completions`. ### Manually install shell completions -The `coreutils` binary can generate completions for the `bash`, `elvish`, `fish`, `powershell` -and `zsh` shells. It prints the result to stdout. +The `coreutils` binary can generate completions for the `bash`, `elvish`, +`fish`, `powershell` and `zsh` shells. It prints the result to stdout. The syntax is: @@ -222,8 +226,8 @@ The syntax is: cargo run completion ``` -So, to install completions for `ls` on `bash` to `/usr/local/share/bash-completion/completions/ls`, -run: +So, to install completions for `ls` on `bash` to +`/usr/local/share/bash-completion/completions/ls`, run: ```shell cargo run completion ls bash > /usr/local/share/bash-completion/completions/ls @@ -232,12 +236,12 @@ cargo run completion ls bash > /usr/local/share/bash-completion/completions/ls ### Manually install manpages To generate manpages, the syntax is: + ```bash cargo run manpage ``` -So, to install the manpage for `ls` to `/usr/local/share/man/man1/ls.1` -run: +So, to install the manpage for `ls` to `/usr/local/share/man/man1/ls.1` run: ```bash cargo run manpage ls > /usr/local/share/man/man1/ls.1 @@ -245,8 +249,8 @@ cargo run manpage ls > /usr/local/share/man/man1/ls.1 ## Un-installation -Un-installation differs depending on how you have installed uutils. If you used -Cargo to install, use Cargo to uninstall. If you used GNU Make to install, use +Un-installation differs depending on how you have installed uutils. If you used +Cargo to install, use Cargo to uninstall. If you used GNU Make to install, use Make to uninstall. ### Uninstall with Cargo @@ -309,98 +313,98 @@ Please note that this is not fully accurate: See for the main meta bugs (many are missing). -| Done | WIP | -|-----------|-----------| -| arch | cp | -| base32 | date | -| base64 | dd | -| basename | df | -| basenc | expr | -| cat | install | -| chcon | ls | -| chgrp | more | -| chmod | numfmt | -| chown | od (`--strings` and 128-bit data types missing) | -| chroot | pr | -| cksum | printf | -| comm | sort | -| csplit | split | -| cut | tac | -| dircolors | test | -| dirname | dir | -| du | vdir | -| echo | stty | -| env | | -| expand | | -| factor | | -| false | | -| fmt | | -| fold | | -| groups | | -| hashsum | | -| head | | -| hostid | | -| hostname | | -| id | | -| join | | -| kill | | -| link | | -| ln | | -| logname | | -| ~~md5sum~~ (replaced by [hashsum](https://github.com/uutils/coreutils/blob/main/src/uu/hashsum/src/hashsum.rs)) | | -| ~~sha1sum~~ (replaced by [hashsum](https://github.com/uutils/coreutils/blob/main/src/uu/hashsum/src/hashsum.rs)) | | -| ~~sha224sum~~ (replaced by [hashsum](https://github.com/uutils/coreutils/blob/main/src/uu/hashsum/src/hashsum.rs)) | | -| ~~sha256sum~~ (replaced by [hashsum](https://github.com/uutils/coreutils/blob/main/src/uu/hashsum/src/hashsum.rs)) | | -| ~~sha384sum~~ (replaced by [hashsum](https://github.com/uutils/coreutils/blob/main/src/uu/hashsum/src/hashsum.rs)) | | -| ~~sha512sum~~ (replaced by [hashsum](https://github.com/uutils/coreutils/blob/main/src/uu/hashsum/src/hashsum.rs)) | | -| mkdir | | -| mkfifo | | -| mknod | | -| mktemp | | -| mv | | -| nice | | -| nl | | -| nohup | | -| nproc | | -| paste | | -| pathchk | | -| pinky | | -| printenv | | -| ptx | | -| pwd | | -| readlink | | -| realpath | | -| relpath | | -| rm | | -| rmdir | | -| runcon | | -| seq | | -| shred | | -| shuf | | -| sleep | | -| stat | | -| stdbuf | | -| sum | | -| sync | | -| tail | | -| tee | | -| timeout | | -| touch | | -| tr | | -| true | | -| truncate | | -| tsort | | -| tty | | -| uname | | -| unexpand | | -| uniq | | -| unlink | | -| uptime | | -| users | | -| wc | | -| who | | -| whoami | | -| yes | | +| Done | WIP | +| ------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------- | +| arch | cp | +| base32 | date | +| base64 | dd | +| basename | df | +| basenc | expr | +| cat | install | +| chcon | ls | +| chgrp | more | +| chmod | numfmt | +| chown | od (`--strings` and 128-bit data types missing) | +| chroot | pr | +| cksum | printf | +| comm | sort | +| csplit | split | +| cut | tac | +| dircolors | test | +| dirname | dir | +| du | vdir | +| echo | stty | +| env | | +| expand | | +| factor | | +| false | | +| fmt | | +| fold | | +| groups | | +| hashsum | | +| head | | +| hostid | | +| hostname | | +| id | | +| join | | +| kill | | +| link | | +| ln | | +| logname | | +| ~~md5sum~~ (replaced by [hashsum](https://github.com/uutils/coreutils/blob/main/src/uu/hashsum/src/hashsum.rs)) | | +| ~~sha1sum~~ (replaced by [hashsum](https://github.com/uutils/coreutils/blob/main/src/uu/hashsum/src/hashsum.rs)) | | +| ~~sha224sum~~ (replaced by [hashsum](https://github.com/uutils/coreutils/blob/main/src/uu/hashsum/src/hashsum.rs)) | | +| ~~sha256sum~~ (replaced by [hashsum](https://github.com/uutils/coreutils/blob/main/src/uu/hashsum/src/hashsum.rs)) | | +| ~~sha384sum~~ (replaced by [hashsum](https://github.com/uutils/coreutils/blob/main/src/uu/hashsum/src/hashsum.rs)) | | +| ~~sha512sum~~ (replaced by [hashsum](https://github.com/uutils/coreutils/blob/main/src/uu/hashsum/src/hashsum.rs)) | | +| mkdir | | +| mkfifo | | +| mknod | | +| mktemp | | +| mv | | +| nice | | +| nl | | +| nohup | | +| nproc | | +| paste | | +| pathchk | | +| pinky | | +| printenv | | +| ptx | | +| pwd | | +| readlink | | +| realpath | | +| relpath | | +| rm | | +| rmdir | | +| runcon | | +| seq | | +| shred | | +| shuf | | +| sleep | | +| stat | | +| stdbuf | | +| sum | | +| sync | | +| tail | | +| tee | | +| timeout | | +| touch | | +| tr | | +| true | | +| truncate | | +| tsort | | +| tty | | +| uname | | +| unexpand | | +| uniq | | +| unlink | | +| uptime | | +| users | | +| wc | | +| who | | +| whoami | | +| yes | | ## License From 20fcce1c21e647c493f5308fe1294541e6cfe7ea Mon Sep 17 00:00:00 2001 From: Terts Diepraam Date: Tue, 14 Mar 2023 17:26:18 +0100 Subject: [PATCH 05/10] README: remove list of utilities This was barely being updated and categorized subjectively instead of by a useful measure, so it did more harm than good. --- README.md | 107 ++---------------------------------------------------- 1 file changed, 3 insertions(+), 104 deletions(-) diff --git a/README.md b/README.md index d2351417f..fb4e860a1 100644 --- a/README.md +++ b/README.md @@ -296,116 +296,15 @@ Below is the evolution of how many GNU tests uutils passes. A more detailed breakdown of the GNU test results of the main branch can be found [in the user manual](https://uutils.github.io/user/test_coverage.html). +See for the main meta bugs +(many are missing). + ![Evolution over time](https://github.com/uutils/coreutils-tracking/blob/main/gnu-results.png?raw=true) ## Contributing To contribute to uutils, please see [CONTRIBUTING](CONTRIBUTING.md). -## Utilities - -Please note that this is not fully accurate: - -- Some new options can be added / removed in the GNU implementation; -- Some error management might be missing; -- Some behaviors might be different. - -See for the main meta bugs -(many are missing). - -| Done | WIP | -| ------------------------------------------------------------------------------------------------------------------ | ----------------------------------------------- | -| arch | cp | -| base32 | date | -| base64 | dd | -| basename | df | -| basenc | expr | -| cat | install | -| chcon | ls | -| chgrp | more | -| chmod | numfmt | -| chown | od (`--strings` and 128-bit data types missing) | -| chroot | pr | -| cksum | printf | -| comm | sort | -| csplit | split | -| cut | tac | -| dircolors | test | -| dirname | dir | -| du | vdir | -| echo | stty | -| env | | -| expand | | -| factor | | -| false | | -| fmt | | -| fold | | -| groups | | -| hashsum | | -| head | | -| hostid | | -| hostname | | -| id | | -| join | | -| kill | | -| link | | -| ln | | -| logname | | -| ~~md5sum~~ (replaced by [hashsum](https://github.com/uutils/coreutils/blob/main/src/uu/hashsum/src/hashsum.rs)) | | -| ~~sha1sum~~ (replaced by [hashsum](https://github.com/uutils/coreutils/blob/main/src/uu/hashsum/src/hashsum.rs)) | | -| ~~sha224sum~~ (replaced by [hashsum](https://github.com/uutils/coreutils/blob/main/src/uu/hashsum/src/hashsum.rs)) | | -| ~~sha256sum~~ (replaced by [hashsum](https://github.com/uutils/coreutils/blob/main/src/uu/hashsum/src/hashsum.rs)) | | -| ~~sha384sum~~ (replaced by [hashsum](https://github.com/uutils/coreutils/blob/main/src/uu/hashsum/src/hashsum.rs)) | | -| ~~sha512sum~~ (replaced by [hashsum](https://github.com/uutils/coreutils/blob/main/src/uu/hashsum/src/hashsum.rs)) | | -| mkdir | | -| mkfifo | | -| mknod | | -| mktemp | | -| mv | | -| nice | | -| nl | | -| nohup | | -| nproc | | -| paste | | -| pathchk | | -| pinky | | -| printenv | | -| ptx | | -| pwd | | -| readlink | | -| realpath | | -| relpath | | -| rm | | -| rmdir | | -| runcon | | -| seq | | -| shred | | -| shuf | | -| sleep | | -| stat | | -| stdbuf | | -| sum | | -| sync | | -| tail | | -| tee | | -| timeout | | -| touch | | -| tr | | -| true | | -| truncate | | -| tsort | | -| tty | | -| uname | | -| unexpand | | -| uniq | | -| unlink | | -| uptime | | -| users | | -| wc | | -| who | | -| whoami | | -| yes | | - ## License uutils is licensed under the MIT License - see the `LICENSE` file for details From 1d24c94341c8ff52825cd17233304ec971640942 Mon Sep 17 00:00:00 2001 From: Terts Diepraam Date: Tue, 14 Mar 2023 17:37:52 +0100 Subject: [PATCH 06/10] CONTRIBUTING: remove instructions for new utils --- CONTRIBUTING.md | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9af1948fe..6747bf46c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,21 +2,11 @@ # Contributing to coreutils -Contributions are very welcome, and should target Rust's main branch until the -standard libraries are stabilized. You may *claim* an item on the to-do list by -following these steps: - -1. Open an issue named "Implement [the utility of your choice]", e.g. "Implement - ls". -1. State that you are working on this utility. -1. Develop the utility. -1. Add integration tests. -1. Add the reference to your utility into Cargo.toml and Makefile. -1. Remove utility from the to-do list in the README. -1. Submit a pull request and close the issue. - -The steps above imply that, before starting to work on a utility, you should -search the issues to make sure no one else is working on it. +Contributions are very welcome via Pull Requests. If you don't know where to +start, take a look at the +[`good-first-issues`](https://github.com/uutils/coreutils/issues?q=is%3Aopen+is%3Aissue+label%3A%22good+first+issue%22). +If you have any questions, feel free to ask them in the issues or on +[Discord](https://discord.gg/wQVJbvJ). ## Best practices From de1f0ef315297ae60b3b566f26e9f70e190c9b79 Mon Sep 17 00:00:00 2001 From: Terts Diepraam Date: Tue, 14 Mar 2023 17:58:37 +0100 Subject: [PATCH 07/10] Merge DEVELOPER_INSTRUCTIONS into CONTRIBUTING --- CONTRIBUTING.md | 106 ++++++++++++++++++++++++++++++++++---- DEVELOPER_INSTRUCTIONS.md | 68 ------------------------ 2 files changed, 96 insertions(+), 78 deletions(-) delete mode 100644 DEVELOPER_INSTRUCTIONS.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6747bf46c..ca4478d25 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,13 +30,81 @@ If you have any questions, feel free to ask them in the issues or on ## Platforms -We take pride in supporting many operating systems and architectures. +We take pride in supporting many operating systems and architectures. Any code +you contribute must at least compile without warnings for all platforms in the +CI. However, you can use `#[cfg(...)]` attributes to create platform dependent features. -**Tip:** -For Windows, Microsoft provides some images (VMWare, Hyper-V, VirtualBox and Parallels) -for development: +**Tip:** For Windows, Microsoft provides some images (VMWare, Hyper-V, +VirtualBox and Parallels) for development: +## Tools + +We have an extensive CI that will check your code before it can be merged. This +section explains how to run those checks locally to avoid waiting for the CI. + +### pre-commit hooks + +A configuration for `pre-commit` is provided in the repository. It allows +automatically checking every git commit you make to ensure it compiles, and +passes `clippy` and `rustfmt` without warnings. + +To use the provided hook: + +1. [Install `pre-commit`](https://pre-commit.com/#install) +1. Run `pre-commit install` while in the repository directory + +Your git commits will then automatically be checked. If a check fails, an error +message will explain why, and your commit will be canceled. You can then make +the suggested changes, and run `git commit ...` again. + +### clippy + +```shell +cargo clippy --all-targets --all-features +``` + +The `msrv` key in the clippy configuration file `clippy.toml` is used to disable +lints pertaining to newer features by specifying the minimum supported Rust +version (MSRV). + +### rustfmt + +```shell +cargo fmt --all +``` + +### cargo-deny + +This project uses [cargo-deny](https://github.com/EmbarkStudios/cargo-deny/) to +detect duplicate dependencies, checks licenses, etc. To run it locally, first +install it and then run with: + +``` +cargo deny --all-features check all +``` + +### Markdown linter + +We use [markdownlint](https://github.com/DavidAnson/markdownlint) to lint the +Markdown files in the repository. + +### Spell checker + +We use `cspell` as spell checker for all files in the project. If you are using +VS Code, you can install the +[code spell checker](https://marketplace.visualstudio.com/items?itemName=streetsidesoftware.code-spell-checker) +extension to enable spell checking within your editor. Otherwise, you can +install [cspell](https://cspell.org/) separately. + +If you want to make the spell checker ignore a word, you can add + +```rust +// spell-checker:ignore word_to_ignore +``` + +at the top of the file. + ## Testing Testing can be done using either Cargo or `make`. @@ -225,15 +293,33 @@ uutils: add new utility gitignore: add temporary files ``` -## cargo-deny +## Code coverage -This project uses [cargo-deny](https://github.com/EmbarkStudios/cargo-deny/) to -detect duplicate dependencies, checks licenses, etc. To run it locally, first -install it and then run with: + +Code coverage report can be generated using [grcov](https://github.com/mozilla/grcov). + +### Using Nightly Rust + +To generate [gcov-based](https://github.com/mozilla/grcov#example-how-to-generate-gcda-files-for-a-rust-project) coverage report + +```shell +export CARGO_INCREMENTAL=0 +export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort" +export RUSTDOCFLAGS="-Cpanic=abort" +cargo build # e.g., --features feat_os_unix +cargo test # e.g., --features feat_os_unix test_pathchk +grcov . -s . --binary-path ./target/debug/ -t html --branch --ignore-not-existing --ignore build.rs --excl-br-line "^\s*((debug_)?assert(_eq|_ne)?\#\[derive\()" -o ./target/debug/coverage/ +# open target/debug/coverage/index.html in browser ``` -cargo deny --all-features check all -``` + +if changes are not reflected in the report then run `cargo clean` and run the above commands. + +### Using Stable Rust + +If you are using stable version of Rust that doesn't enable code coverage instrumentation by default +then add `-Z-Zinstrument-coverage` flag to `RUSTFLAGS` env variable specified above. + ## Other implementations diff --git a/DEVELOPER_INSTRUCTIONS.md b/DEVELOPER_INSTRUCTIONS.md deleted file mode 100644 index 3641f57b9..000000000 --- a/DEVELOPER_INSTRUCTIONS.md +++ /dev/null @@ -1,68 +0,0 @@ -# Documentation - -The source of the documentation is available on: - - - -The documentation is updated everyday on this repository: - - - -## Running GNU tests - - - -- Check out next to your fork as gnu -- Check out next to your fork as gnulib -- Rename the checkout of your fork to uutils - -At the end you should have uutils, gnu and gnulib checked out next to each other. - -- Run `cd uutils && ./util/build-gnu.sh && cd ..` to get everything ready (this may take a while) -- Finally, you can run tests with `bash uutils/util/run-gnu-test.sh `. Instead of `` insert the tests you want to run, e.g. `tests/misc/wc-proc.sh`. - -## Code Coverage Report Generation - - - -Code coverage report can be generated using [grcov](https://github.com/mozilla/grcov). - -### Using Nightly Rust - -To generate [gcov-based](https://github.com/mozilla/grcov#example-how-to-generate-gcda-files-for-a-rust-project) coverage report - -```shell -export CARGO_INCREMENTAL=0 -export RUSTFLAGS="-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort" -export RUSTDOCFLAGS="-Cpanic=abort" -cargo build # e.g., --features feat_os_unix -cargo test # e.g., --features feat_os_unix test_pathchk -grcov . -s . --binary-path ./target/debug/ -t html --branch --ignore-not-existing --ignore build.rs --excl-br-line "^\s*((debug_)?assert(_eq|_ne)?\#\[derive\()" -o ./target/debug/coverage/ -# open target/debug/coverage/index.html in browser -``` - -if changes are not reflected in the report then run `cargo clean` and run the above commands. - -### Using Stable Rust - -If you are using stable version of Rust that doesn't enable code coverage instrumentation by default -then add `-Z-Zinstrument-coverage` flag to `RUSTFLAGS` env variable specified above. - -## pre-commit hooks - -A configuration for `pre-commit` is provided in the repository. It allows automatically checking every git commit you make to ensure it compiles, and passes `clippy` and `rustfmt` without warnings. - -To use the provided hook: - -1. [Install `pre-commit`](https://pre-commit.com/#install) -1. Run `pre-commit install` while in the repository directory - -Your git commits will then automatically be checked. If a check fails, an error message will explain why, and your commit will be canceled. You can then make the suggested changes, and run `git commit ...` again. - -## Using Clippy - -The `msrv` key in the clippy configuration file `clippy.toml` is used to disable lints pertaining to newer features by specifying the minimum supported Rust version (MSRV). However, this key is only supported on `nightly`. To invoke clippy without errors, use `cargo +nightly clippy`. In order to also check tests and non-default crate features, use `cargo +nightly clippy --all-targets --all-features`. - -## Markdown linter - -We use to lint the Markdown files. From 4dd618e67becb520f6d41ddaa22e93014782206c Mon Sep 17 00:00:00 2001 From: Terts Diepraam Date: Tue, 14 Mar 2023 21:53:22 +0100 Subject: [PATCH 08/10] README and CONTRIBUTING: markdown and spelling fixes --- CONTRIBUTING.md | 2 +- README.md | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ca4478d25..6edfea44c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,4 +1,4 @@ - + # Contributing to coreutils diff --git a/README.md b/README.md index fb4e860a1..8b57855b5 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,6 @@ + + +
![uutils logo](docs/src/logo.svg) @@ -17,8 +20,6 @@ --- - - uutils coreutils is a cross-platform reimplementation of the GNU coreutils in [Rust](http://www.rust-lang.org). While all programs have been implemented, some From 1435d3573a59d64e156b99f2a46317b32256ddf6 Mon Sep 17 00:00:00 2001 From: Miles Liu Date: Wed, 15 Mar 2023 18:08:32 +0800 Subject: [PATCH 09/10] dd: fix unused imports on FreeBSD --- tests/by-util/test_dd.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/by-util/test_dd.rs b/tests/by-util/test_dd.rs index 5deeb12f0..71dd92da4 100644 --- a/tests/by-util/test_dd.rs +++ b/tests/by-util/test_dd.rs @@ -5,7 +5,7 @@ use crate::common::util::*; use std::fs::{File, OpenOptions}; use std::io::{BufReader, Read, Write}; use std::path::PathBuf; -#[cfg(all(not(windows), not(target_os = "macos")))] +#[cfg(all(unix, not(target_os = "macos"), not(target_os = "freebsd")))] use std::process::{Command, Stdio}; #[cfg(not(windows))] use std::thread::sleep; From 6c9d007d5bb90552e58162e3f3db782f98ef0a13 Mon Sep 17 00:00:00 2001 From: Daniel Hofstetter Date: Wed, 15 Mar 2023 16:56:11 +0100 Subject: [PATCH 10/10] Use https in link & fix quote in CONTRIBUTING.md --- CONTRIBUTING.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6edfea44c..8199b6d87 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -240,24 +240,30 @@ To improve the GNU compatibility, the following process is recommended: To help the project maintainers review pull requests from contributors across numerous utilities, the team has settled on conventions for commit messages. -From : +From : ``` -Short (50 chars or less) summary of changes +Capitalized, short (50 chars or less) summary More detailed explanatory text, if necessary. Wrap it to about 72 characters or so. In some contexts, the first line is treated as the subject of an email and the rest of the text as the body. The blank line separating the summary from the body is critical (unless you omit -the body entirely); tools like rebase can get confused if you run the +the body entirely); tools like rebase will confuse you if you run the two together. +Write your commit message in the imperative: "Fix bug" and not "Fixed bug" +or "Fixes bug." This convention matches up with commit messages generated +by commands like git merge and git revert. + Further paragraphs come after blank lines. - Bullet points are okay, too - - Typically a hyphen or asterisk is used for the bullet, preceded by a + - Typically a hyphen or asterisk is used for the bullet, followed by a single space, with blank lines in between, but conventions vary here + + - Use a hanging indent ``` Furthermore, here are a few examples for a summary line: