diff --git a/.cargo/config b/.cargo/config index 58e1381b1..0a8fd3d00 100644 --- a/.cargo/config +++ b/.cargo/config @@ -1,2 +1,11 @@ [target.x86_64-unknown-redox] linker = "x86_64-unknown-redox-gcc" + +[target.'cfg(feature = "cargo-clippy")'] +rustflags = [ + "-Wclippy::use_self", + "-Wclippy::needless_pass_by_value", + "-Wclippy::semicolon_if_nothing_returned", + "-Wclippy::single_char_pattern", + "-Wclippy::explicit_iter_loop", +] diff --git a/.editorconfig b/.editorconfig index d93fa7c0e..53ccc4f9a 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,4 +1,4 @@ -# EditorConfig (is awesome): http://EditorConfig.org +# EditorConfig (is awesome!; ref: http://EditorConfig.org; v2022.02.11 [rivy]) # * top-most EditorConfig file root = true @@ -13,27 +13,49 @@ insert_final_newline = true max_line_length = 100 trim_trailing_whitespace = true -[[Mm]akefile{,.*}, *.{mk,[Mm][Kk]}] +[{[Mm]akefile{,.*},*.{mak,mk,[Mm][Aa][Kk],[Mm][Kk]},[Gg][Nn][Uu]makefile}] # makefiles ~ TAB-style indentation indent_style = tab +[*.bash] +# `bash` shell scripts +indent_size = 4 +indent_style = space +# * ref: +# shell_variant = bash ## allow `shellcheck` to decide via script hash-bang/sha-bang line +switch_case_indent = true + [*.{bat,cmd,[Bb][Aa][Tt],[Cc][Mm][Dd]}] # BAT/CMD ~ DOS/Win requires BAT/CMD files to have CRLF EOLNs end_of_line = crlf +[*.{cjs,cjx,cts,ctx,js,jsx,mjs,mts,mtx,ts,tsx,json,jsonc}] +# js/ts/json ~ Prettier/XO-style == TAB indention + SPACE alignment +indent_size = 2 +indent_style = tab + [*.go] # go ~ TAB-style indentation (SPACE-style alignment); ref: @@ indent_style = tab -[*.{cjs,js,json,mjs,ts}] -# js/ts -indent_size = 2 - [*.{markdown,md,mkd,[Mm][Dd],[Mm][Kk][Dd],[Mm][Dd][Oo][Ww][Nn],[Mm][Kk][Dd][Oo][Ww][Nn],[Mm][Aa][Rr][Kk][Dd][Oo][Ww][Nn]}] # markdown indent_size = 2 indent_style = space +[*.sh] +# POSIX shell scripts +indent_size = 4 +indent_style = space +# * ref: +# shell_variant = posix ## allow `shellcheck` to decide via script hash-bang/sha-bang line +switch_case_indent = true + +[*.{sln,vc{,x}proj{,.*},[Ss][Ln][Nn],[Vv][Cc]{,[Xx]}[Pp][Rr][Oo][Jj]{,.*}}] +# MSVC sln/vcproj/vcxproj files, when used, will persistantly revert to CRLF EOLNs and eat final EOLs +end_of_line = crlf +insert_final_newline = false + [*.{yaml,yml,[Yy][Mm][Ll],[Yy][Aa][Mm][Ll]}] # YAML indent_size = 2 diff --git a/.github/workflows/CICD.yml b/.github/workflows/CICD.yml index aec424312..b47540ed9 100644 --- a/.github/workflows/CICD.yml +++ b/.github/workflows/CICD.yml @@ -1,10 +1,10 @@ name: CICD # spell-checker:ignore (acronyms) CICD MSVC musl -# spell-checker:ignore (env/flags) Awarnings Ccodegen Coverflow Cpanic RUSTDOCFLAGS RUSTFLAGS Zpanic +# spell-checker:ignore (env/flags) Awarnings Ccodegen Coverflow Cpanic Dwarnings RUSTDOCFLAGS RUSTFLAGS Zpanic # spell-checker:ignore (jargon) SHAs deps dequote softprops subshell toolchain # spell-checker:ignore (names) CodeCOV MacOS MinGW Peltoche rivy -# spell-checker:ignore (shell/tools) choco clippy dmake dpkg esac fakeroot gmake grcov halium lcov libssl mkdir popd printf pushd rustc rustfmt rustup shopt xargs +# spell-checker:ignore (shell/tools) choco clippy dmake dpkg esac fakeroot gmake grcov halium lcov libssl mkdir popd printf pushd rsync rustc rustfmt rustup shopt xargs # spell-checker:ignore (misc) aarch alnum armhf bindir busytest coreutils gnueabihf issuecomment maint nullglob onexitbegin onexitend pell runtest tempfile testsuite uutils DESTDIR sizemulti # ToDO: [2021-06; rivy] change from `cargo-tree` to `cargo tree` once MSRV is >= 1.45 @@ -340,6 +340,13 @@ jobs: ## Confirm MinSRV compatible 'Cargo.lock' # * 'Cargo.lock' is required to be in a format that `cargo` of MinSRV can interpret (eg, v1-format for MinSRV < v1.38) cargo fetch --locked --quiet || { echo "::error file=Cargo.lock::Incompatible (or out-of-date) 'Cargo.lock' file; update using \`cargo +${{ env.RUST_MIN_SRV }} update\`" ; exit 1 ; } + - name: Confirm MinSRV equivalence for '.clippy.toml' + shell: bash + run: | + ## Confirm MinSRV equivalence for '.clippy.toml' + # * ensure '.clippy.toml' MSRV configuration setting is equal to ${{ env.RUST_MIN_SRV }} + CLIPPY_MSRV=$(grep -P "(?i)^\s*msrv\s*=\s*" .clippy.toml | grep -oP "\d+([.]\d+)+") + if [ "${CLIPPY_MSRV}" != "${{ env.RUST_MIN_SRV }}" ]; then { echo "::error file=.clippy.toml::Incorrect MSRV configuration for clippy (found '${CLIPPY_MSRV}'; should be '${{ env.RUST_MIN_SRV }}'); update '.clippy.toml' with 'msrv = \"${{ env.RUST_MIN_SRV }}\"'" ; exit 1 ; } ; fi - name: Info shell: bash run: | diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml index b36bbcb33..e901d4cfe 100644 --- a/.github/workflows/GnuTests.yml +++ b/.github/workflows/GnuTests.yml @@ -1,6 +1,6 @@ name: GnuTests -# spell-checker:ignore (names) gnulib ; (utils) autopoint gperf pyinotify texinfo ; (vars) XPASS +# spell-checker:ignore (names) gnulib ; (jargon) submodules ; (people) Dawid Dziurla * dawidd ; (utils) autopoint chksum gperf pyinotify shopt texinfo ; (vars) FILESET XPASS on: [push, pull_request] @@ -9,23 +9,52 @@ jobs: name: Run GNU tests runs-on: ubuntu-latest steps: - - name: Checkout code uutil + - name: Initialize workflow variables + id: vars + shell: bash + run: | + ## VARs setup + outputs() { step_id="vars"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo ::set-output name=${var}::${!var}; done; } + # * config + path_GNU="gnu" + path_GNU_tests="${path_GNU}/tests" + path_UUTILS="uutils" + path_reference="reference" + outputs path_GNU path_GNU_tests path_reference path_UUTILS + # + repo_default_branch="${{ github.event.repository.default_branch }}" + repo_GNU_ref="v9.0" + repo_reference_branch="${{ github.event.repository.default_branch }}" + outputs repo_default_branch repo_GNU_ref repo_reference_branch + # + SUITE_LOG_FILE="${path_GNU_tests}/test-suite.log" + TEST_LOGS_GLOB="${path_GNU_tests}/**/*.log" ## note: not usable at bash CLI; [why] double globstar not enabled by default b/c MacOS includes only bash v3 which doesn't have double globstar support + TEST_FILESET_PREFIX='test-fileset-IDs.sha1#' + TEST_FILESET_SUFFIX='.txt' + TEST_SUMMARY_FILE='gnu-result.json' + TEST_FULL_SUMMARY_FILE='gnu-full-result.json' + outputs SUITE_LOG_FILE TEST_FILESET_PREFIX TEST_FILESET_SUFFIX TEST_LOGS_GLOB TEST_SUMMARY_FILE TEST_FULL_SUMMARY_FILE + - name: Checkout code (uutil) uses: actions/checkout@v2 with: - path: 'uutils' - - name: Checkout GNU coreutils + path: '${{ steps.vars.outputs.path_UUTILS }}' + - name: Checkout code (GNU coreutils) uses: actions/checkout@v2 with: repository: 'coreutils/coreutils' - path: 'gnu' - ref: v9.0 - - name: Checkout GNU coreutils library (gnulib) - uses: actions/checkout@v2 + path: '${{ steps.vars.outputs.path_GNU }}' + ref: ${{ steps.vars.outputs.repo_GNU_ref }} + submodules: recursive + - name: Retrieve reference artifacts + uses: dawidd6/action-download-artifact@v2 + # ref: + continue-on-error: true ## don't break the build for missing reference artifacts (may be expired or just not generated yet) with: - repository: 'coreutils/gnulib' - path: 'gnulib' - ref: 8e99f24c0931a38880c6ee9b8287c7da80b0036b - fetch-depth: 0 # gnu gets upset if gnulib is a shallow checkout + workflow: GnuTests.yml + branch: "${{ steps.vars.outputs.repo_reference_branch }}" + # workflow_conclusion: success ## (default); * but, if commit with failed GnuTests is merged into the default branch, future commits will all show regression errors in GnuTests CI until o/w fixed + workflow_conclusion: completed ## continually recalibrates to last commit of default branch with a successful GnuTests (ie, "self-heals" from GnuTest regressions, but needs more supervision for/of regressions) + path: "${{ steps.vars.outputs.path_reference }}" - name: Install `rust` toolchain uses: actions-rs/toolchain@v1 with: @@ -39,32 +68,55 @@ jobs: ## Install dependencies sudo apt-get update sudo apt-get install autoconf autopoint bison texinfo gperf gcc g++ gdb python-pyinotify jq + - name: Add various locales + shell: bash + run: | + echo "Before:" + locale -a + ## Some tests fail with 'cannot change locale (en_US.ISO-8859-1): No such file or directory' + ## Some others need a French locale + sudo locale-gen + sudo locale-gen fr_FR + sudo locale-gen fr_FR.UTF-8 + sudo update-locale + echo "After:" + locale -a - name: Build binaries shell: bash run: | ## Build binaries - cd uutils + cd '${{ steps.vars.outputs.path_UUTILS }}' bash util/build-gnu.sh - name: Run GNU tests shell: bash run: | - bash uutils/util/run-gnu-test.sh - - name: Extract testing info + path_GNU='${{ steps.vars.outputs.path_GNU }}' + path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}' + bash "${path_UUTILS}/util/run-gnu-test.sh" + - name: Extract testing info into JSON + shell: bash + run : | + path_UUTILS='${{ steps.vars.outputs.path_UUTILS }}' + python ${path_UUTILS}/util/gnu-json-result.py ${{ steps.vars.outputs.path_GNU_tests }} > ${{ steps.vars.outputs.TEST_FULL_SUMMARY_FILE }} + - name: Extract/summarize testing info + id: summary shell: bash run: | - ## Extract testing info - LOG_FILE=gnu/tests/test-suite.log - if test -f "$LOG_FILE" + ## Extract/summarize testing info + outputs() { step_id="summary"; for var in "$@" ; do echo steps.${step_id}.outputs.${var}="${!var}"; echo ::set-output name=${var}::${!var}; done; } + # + SUITE_LOG_FILE='${{ steps.vars.outputs.SUITE_LOG_FILE }}' + if test -f "${SUITE_LOG_FILE}" then - TOTAL=$(sed -n "s/.*# TOTAL: \(.*\)/\1/p" "$LOG_FILE"|tr -d '\r'|head -n1) - PASS=$(sed -n "s/.*# PASS: \(.*\)/\1/p" "$LOG_FILE"|tr -d '\r'|head -n1) - SKIP=$(sed -n "s/.*# SKIP: \(.*\)/\1/p" "$LOG_FILE"|tr -d '\r'|head -n1) - FAIL=$(sed -n "s/.*# FAIL: \(.*\)/\1/p" "$LOG_FILE"|tr -d '\r'|head -n1) - XPASS=$(sed -n "s/.*# XPASS: \(.*\)/\1/p" "$LOG_FILE"|tr -d '\r'|head -n1) - ERROR=$(sed -n "s/.*# ERROR: \(.*\)/\1/p" "$LOG_FILE"|tr -d '\r'|head -n1) + TOTAL=$(sed -n "s/.*# TOTAL: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) + PASS=$(sed -n "s/.*# PASS: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) + SKIP=$(sed -n "s/.*# SKIP: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) + FAIL=$(sed -n "s/.*# FAIL: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) + XPASS=$(sed -n "s/.*# XPASS: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) + ERROR=$(sed -n "s/.*# ERROR: \(.*\)/\1/p" "${SUITE_LOG_FILE}" | tr -d '\r' | head -n1) if [[ "$TOTAL" -eq 0 || "$TOTAL" -eq 1 ]]; then - echo "Error in the execution, failing early" - exit 1 + echo "::error ::Failed to parse test results from '${SUITE_LOG_FILE}'; failing early" + exit 1 fi output="GNU tests summary = TOTAL: $TOTAL / PASS: $PASS / FAIL: $FAIL / ERROR: $ERROR" echo "${output}" @@ -78,54 +130,70 @@ jobs: --arg fail "$FAIL" \ --arg xpass "$XPASS" \ --arg error "$ERROR" \ - '{($date): { sha: $sha, total: $total, pass: $pass, skip: $skip, fail: $fail, xpass: $xpass, error: $error, }}' > gnu-result.json + '{($date): { sha: $sha, total: $total, pass: $pass, skip: $skip, fail: $fail, xpass: $xpass, error: $error, }}' > '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' + HASH=$(sha1sum '${{ steps.vars.outputs.TEST_SUMMARY_FILE }}' | cut --delim=" " -f 1) + outputs HASH else - echo "::error ::Failed to get summary of test results" + echo "::error ::Failed to find summary of test results (missing '${SUITE_LOG_FILE}'); failing early" + exit 1 fi - - uses: actions/upload-artifact@v2 + - name: Reserve SHA1/ID of 'test-summary' + uses: actions/upload-artifact@v2 with: - name: test-report - path: gnu/tests/**/*.log - - uses: actions/upload-artifact@v2 + name: "${{ steps.summary.outputs.HASH }}" + path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}" + - name: Reserve test results summary + uses: actions/upload-artifact@v2 with: - name: gnu-result - path: gnu-result.json - - name: Download the result - uses: dawidd6/action-download-artifact@v2 + name: test-summary + path: "${{ steps.vars.outputs.TEST_SUMMARY_FILE }}" + - name: Reserve test logs + uses: actions/upload-artifact@v2 with: - workflow: GnuTests.yml - name: gnu-result - repo: uutils/coreutils - branch: main - path: dl - - name: Download the log - uses: dawidd6/action-download-artifact@v2 + name: test-logs + path: "${{ steps.vars.outputs.TEST_LOGS_GLOB }}" + - name: Upload full json results + uses: actions/upload-artifact@v2 with: - workflow: GnuTests.yml - name: test-report - repo: uutils/coreutils - branch: main - path: dl - - name: Compare failing tests against main + name: gnu-full-result.json + path: ${{ steps.vars.outputs.TEST_FULL_SUMMARY_FILE }} + - name: Compare test failures VS reference shell: bash run: | - OLD_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" dl/test-suite.log | sort) - NEW_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" gnu/tests/test-suite.log | sort) - for LINE in $OLD_FAILING - do - if ! grep -Fxq $LINE<<<"$NEW_FAILING"; then - echo "::warning ::Congrats! The gnu test $LINE is now passing!" - fi - done - for LINE in $NEW_FAILING - do - if ! grep -Fxq $LINE<<<"$OLD_FAILING" - then - echo "::error ::GNU test failed: $LINE. $LINE is passing on 'main'. Maybe you have to rebase?" - fi - done - - name: Compare against main results + have_new_failures="" + REF_LOG_FILE='${{ steps.vars.outputs.path_reference }}/test-logs/test-suite.log' + REF_SUMMARY_FILE='${{ steps.vars.outputs.path_reference }}/test-summary/gnu-result.json' + if test -f "${REF_LOG_FILE}"; then + echo "Reference SHA1/ID: $(sha1sum -- "${REF_SUMMARY_FILE}")" + REF_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${REF_LOG_FILE}" | sort) + NEW_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" '${{ steps.vars.outputs.path_GNU_tests }}/test-suite.log' | sort) + for LINE in ${REF_FAILING} + do + if ! grep -Fxq ${LINE}<<<"${NEW_FAILING}"; then + echo "::warning ::Congrats! The gnu test ${LINE} is now passing!" + fi + done + for LINE in ${NEW_FAILING} + do + if ! grep -Fxq ${LINE}<<<"${REF_FAILING}" + then + echo "::error ::GNU test failed: ${LINE}. ${LINE} is passing on '${{ steps.vars.outputs.repo_default_branch }}'. Maybe you have to rebase?" + have_new_failures="true" + fi + done + else + echo "::warning ::Skipping test failure comparison; no prior reference test logs are available." + fi + if test -n "${have_new_failures}" ; then exit -1 ; fi + - name: Compare test summary VS reference + if: success() || failure() # run regardless of prior step success/failure shell: bash run: | - mv dl/gnu-result.json main-gnu-result.json - python uutils/util/compare_gnu_result.py + REF_SUMMARY_FILE='${{ steps.vars.outputs.path_reference }}/test-summary/gnu-result.json' + if test -f "${REF_SUMMARY_FILE}"; then + echo "Reference SHA1/ID: $(sha1sum -- "${REF_SUMMARY_FILE}")" + mv "${REF_SUMMARY_FILE}" main-gnu-result.json + python uutils/util/compare_gnu_result.py + else + echo "::warning ::Skipping test summary comparison; no prior reference summary is available." + fi diff --git a/.rustfmt.toml b/.rustfmt.toml new file mode 100644 index 000000000..2a0c75141 --- /dev/null +++ b/.rustfmt.toml @@ -0,0 +1 @@ +# * using all default `cargo fmt`/`rustfmt` options diff --git a/.travis/redox-toolchain.sh b/.travis/redox-toolchain.sh index 83bc8fc45..d8b43b489 100755 --- a/.travis/redox-toolchain.sh +++ b/.travis/redox-toolchain.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh rustup target add x86_64-unknown-redox sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys AA12E97F0881517F diff --git a/.vscode/.gitattributes b/.vscode/.gitattributes new file mode 100644 index 000000000..c050fbbf3 --- /dev/null +++ b/.vscode/.gitattributes @@ -0,0 +1,2 @@ +# Configure GitHub to not mark comments in configuration files as errors +*.json linguist-language=jsonc diff --git a/.vscode/cSpell.json b/.vscode/cSpell.json index 95b6f0485..6dfb3b666 100644 --- a/.vscode/cSpell.json +++ b/.vscode/cSpell.json @@ -1,7 +1,12 @@ // `cspell` settings { - "version": "0.1", // Version of the setting file. Always 0.1 - "language": "en", // language - current active spelling language + // version of the setting file + "version": "0.2", + + // spelling language + "language": "en", + + // custom dictionaries "dictionaries": ["acronyms+names", "jargon", "people", "shell", "workspace"], "dictionaryDefinitions": [ { "name": "acronyms+names", "path": "./cspell.dictionaries/acronyms+names.wordlist.txt" }, @@ -10,10 +15,19 @@ { "name": "shell", "path": "./cspell.dictionaries/shell.wordlist.txt" }, { "name": "workspace", "path": "./cspell.dictionaries/workspace.wordlist.txt" } ], - // ignorePaths - a list of globs to specify which files are to be ignored - "ignorePaths": ["Cargo.lock", "target/**", "tests/**/fixtures/**", "src/uu/dd/test-resources/**", "vendor/**"], - // ignoreWords - a list of words to be ignored (even if they are in the flagWords) + + // files to ignore (globs supported) + "ignorePaths": [ + "Cargo.lock", + "target/**", + "tests/**/fixtures/**", + "src/uu/dd/test-resources/**", + "vendor/**" + ], + + // words to ignore (even if they are in the flagWords) "ignoreWords": [], - // words - list of words to be always considered correct + + // words to always consider correct "words": [] } diff --git a/.vscode/cspell.dictionaries/workspace.wordlist.txt b/.vscode/cspell.dictionaries/workspace.wordlist.txt index e41aba979..99ac20ea2 100644 --- a/.vscode/cspell.dictionaries/workspace.wordlist.txt +++ b/.vscode/cspell.dictionaries/workspace.wordlist.txt @@ -44,6 +44,7 @@ termsize termwidth textwrap thiserror +ureq walkdir winapi xattr diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 30b38bfa9..bd9dcf485 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -1,10 +1,13 @@ // spell-checker:ignore (misc) matklad // see for the documentation about the extensions.json format +// * +// "foxundermoon.shell-format" ~ shell script formatting ; note: ENABLE "Use EditorConfig" +// "matklad.rust-analyzer" ~ `rust` language support +// "streetsidesoftware.code-spell-checker" ~ `cspell` spell-checker support { - "recommendations": [ - // Rust language support. - "matklad.rust-analyzer", - // `cspell` spell-checker support - "streetsidesoftware.code-spell-checker" - ] + "recommendations": [ + "matklad.rust-analyzer", + "streetsidesoftware.code-spell-checker", + "foxundermoon.shell-format" + ] } diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..54df63a5b --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1 @@ +{ "cSpell.import": [".vscode/cspell.json"] } diff --git a/Cargo.lock b/Cargo.lock index cc7c3967b..244c56baa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,12 @@ version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "ahash" version = "0.4.7" @@ -50,6 +56,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" +[[package]] +name = "arrayvec" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8da52d66c7071e2e3fa2a1e5c6d088fec47b593032b254f5e980de8ea54454d6" + [[package]] name = "atty" version = "0.2.14" @@ -67,6 +79,12 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + [[package]] name = "bigdecimal" version = "0.3.0" @@ -123,10 +141,24 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "afa748e348ad3be8263be728124b24a24f268266f6f5d58af9d75f6a40b5c587" dependencies = [ "arrayref", - "arrayvec", + "arrayvec 0.5.2", "constant_time_eq", ] +[[package]] +name = "blake3" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a08e53fc5a564bb15bfe6fae56bd71522205f1f91893f9c0116edad6496c183f" +dependencies = [ + "arrayref", + "arrayvec 0.7.2", + "cc", + "cfg-if 1.0.0", + "constant_time_eq", + "digest", +] + [[package]] name = "block-buffer" version = "0.10.0" @@ -147,6 +179,12 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "bumpalo" +version = "3.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899" + [[package]] name = "byte-unit" version = "4.0.13" @@ -208,6 +246,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "chunked_transfer" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fff857943da45f546682664a79488be82e69e43c1a7a2307679ab9afb3a66d2e" + [[package]] name = "clang-sys" version = "1.3.0" @@ -292,6 +336,7 @@ dependencies = [ "conv", "filetime", "glob", + "hex-literal", "lazy_static", "libc", "nix 0.23.1", @@ -308,6 +353,7 @@ dependencies = [ "time", "unindent", "unix_socket", + "ureq", "users", "uu_arch", "uu_base32", @@ -411,6 +457,7 @@ dependencies = [ "uu_yes", "uucore", "walkdir", + "zip", ] [[package]] @@ -525,6 +572,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "crossbeam-channel" version = "0.5.2" @@ -670,6 +726,7 @@ dependencies = [ "block-buffer", "crypto-common", "generic-array", + "subtle", ] [[package]] @@ -770,12 +827,34 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "flate2" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f" +dependencies = [ + "cfg-if 1.0.0", + "crc32fast", + "libc", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "form_urlencoded" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +dependencies = [ + "matches", + "percent-encoding", +] + [[package]] name = "fs_extra" version = "1.2.0" @@ -882,6 +961,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6a22814455d41612f41161581c2883c0c6a1c41852729b17d5ed88f01e153aa" +[[package]] +name = "hex-literal" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ebdb29d2ea9ed0083cd8cece49bbd968021bd99b0849edb4a9a7ee0fdf6a4e0" + [[package]] name = "hostname" version = "0.3.1" @@ -899,6 +984,17 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "idna" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "if_rust_version" version = "1.0.0" @@ -939,6 +1035,15 @@ dependencies = [ "either", ] +[[package]] +name = "js-sys" +version = "0.3.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a38fc24e30fd564ce974c02bf1d337caddff65be6cc4735a1f7eab22a7440f04" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "keccak" version = "0.1.0" @@ -1016,6 +1121,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" +[[package]] +name = "matches" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" + [[package]] name = "md5" version = "0.3.8" @@ -1061,6 +1172,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +dependencies = [ + "adler", + "autocfg", +] + [[package]] name = "mio" version = "0.7.14" @@ -1347,6 +1468,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + [[package]] name = "phf" version = "0.10.1" @@ -1627,6 +1754,21 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53552c6c49e1e13f1a203ef0080ab3bbef0beb570a528993e83df057a9d9bba1" +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi 0.3.9", +] + [[package]] name = "rlimit" version = "0.4.0" @@ -1653,6 +1795,18 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustls" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b323592e3164322f5b193dc4302e4e36cd8d37158a712d664efae1a5c2791700" +dependencies = [ + "log", + "ring", + "sct", + "webpki", +] + [[package]] name = "same-file" version = "1.0.6" @@ -1668,6 +1822,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "selinux" version = "0.2.5" @@ -1716,19 +1880,15 @@ dependencies = [ [[package]] name = "sha1" -version = "0.6.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1da05c97445caa12d05e848c4a4fcbbea29e748ac28f7e80e9b010392063770" +checksum = "04cc229fb94bcb689ffc39bd4ded842f6ff76885efede7c6d1ffb62582878bea" dependencies = [ - "sha1_smol", + "cfg-if 1.0.0", + "cpufeatures", + "digest", ] -[[package]] -name = "sha1_smol" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae1a47186c03a32177042e55dbc5fd5aee900b8e0069a8d70fba96a9375cd012" - [[package]] name = "sha2" version = "0.10.1" @@ -1814,6 +1974,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -1850,6 +2016,12 @@ dependencies = [ "syn", ] +[[package]] +name = "subtle" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" + [[package]] name = "syn" version = "1.0.86" @@ -1979,6 +2151,21 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "tinyvec" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c1c1d5a42b6245520c249549ec267180beaffcc0615401ac8e31853d4b6d8d2" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + [[package]] name = "toml" version = "0.5.8" @@ -1994,6 +2181,12 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" +[[package]] +name = "unicode-bidi" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f" + [[package]] name = "unicode-linebreak" version = "0.1.2" @@ -2003,6 +2196,15 @@ dependencies = [ "regex", ] +[[package]] +name = "unicode-normalization" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-segmentation" version = "1.8.0" @@ -2043,6 +2245,41 @@ dependencies = [ "libc", ] +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "ureq" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9399fa2f927a3d327187cbd201480cee55bee6ac5d3c77dd27f0c6814cff16d5" +dependencies = [ + "base64", + "chunked_transfer", + "flate2", + "log", + "once_cell", + "rustls", + "url", + "webpki", + "webpki-roots", +] + +[[package]] +name = "url" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" +dependencies = [ + "form_urlencoded", + "idna", + "matches", + "percent-encoding", +] + [[package]] name = "users" version = "0.10.0" @@ -2375,6 +2612,7 @@ name = "uu_hashsum" version = "0.0.12" dependencies = [ "blake2b_simd", + "blake3", "clap 3.0.10", "digest", "hex", @@ -2444,6 +2682,7 @@ name = "uu_join" version = "0.0.12" dependencies = [ "clap 3.0.10", + "memchr 2.4.1", "uucore", ] @@ -2821,6 +3060,7 @@ name = "uu_split" version = "0.0.12" dependencies = [ "clap 3.0.10", + "memchr 2.4.1", "uucore", ] @@ -3138,6 +3378,89 @@ version = "0.10.2+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" +[[package]] +name = "wasm-bindgen" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b21c0df030f5a177f3cba22e9bc4322695ec43e7257d865302900290bcdedca" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote 1.0.14", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4203d69e40a52ee523b2529a773d5ffc1dc0071801c87b3d270b471b80ed01" +dependencies = [ + "quote 1.0.14", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa8a30d46208db204854cadbb5d4baf5fcf8071ba5bf48190c3e59937962ebc" +dependencies = [ + "proc-macro2", + "quote 1.0.14", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d958d035c4438e28c70e4321a2911302f10135ce78a9c7834c0cab4123d06a2" + +[[package]] +name = "web-sys" +version = "0.3.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c060b319f29dd25724f09a2ba1418f142f539b2be99fbf4d2d5a8f7330afb8eb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552ceb903e957524388c4d3475725ff2c8b7960922063af6ce53c9a43da07449" +dependencies = [ + "webpki", +] + [[package]] name = "which" version = "4.2.2" @@ -3215,3 +3538,15 @@ name = "z85" version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af896e93db81340b74b65f74276a99b210c086f3d34ed0abf433182a462af856" + +[[package]] +name = "zip" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93ab48844d61251bb3835145c521d88aa4031d7139e8485990f60ca911fa0815" +dependencies = [ + "byteorder", + "crc32fast", + "flate2", + "thiserror", +] diff --git a/Cargo.toml b/Cargo.toml index e9fbe42fb..4222f1749 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -252,6 +252,8 @@ lazy_static = { version="1.3" } textwrap = { version="0.14", features=["terminal_size"] } uucore = { version=">=0.0.11", package="uucore", path="src/uucore" } selinux = { version="0.2", optional = true } +ureq = "2.4.0" +zip = { version = "0.5.13", default_features=false, features=["deflate"] } # * uutils uu_test = { optional=true, version="0.0.12", package="uu_test", path="src/uu/test" } # @@ -372,13 +374,14 @@ libc = "0.2" pretty_assertions = "1" rand = "0.8" regex = "1.0" -sha1 = { version="0.6", features=["std"] } +sha1 = { version="0.10", features=["std"] } tempfile = "3.2.0" time = "0.1" unindent = "0.1" uucore = { version=">=0.0.11", package="uucore", path="src/uucore", features=["entries", "process"] } walkdir = "2.2" atty = "0.2" +hex-literal = "0.3.1" [target.'cfg(target_os = "linux")'.dev-dependencies] rlimit = "0.4.0" diff --git a/GNUmakefile b/GNUmakefile index 8f9a8cae4..b3278f9ee 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -62,6 +62,7 @@ PROGS := \ csplit \ cut \ date \ + dd \ df \ dircolors \ dirname \ diff --git a/README.md b/README.md index 28df6110f..d30fd4c05 100644 --- a/README.md +++ b/README.md @@ -349,6 +349,10 @@ $ make UTILS='UTILITY_1 UTILITY_2' RUNTEST_ARGS='-v' busytest ### Comparing with GNU +Below is the evolution of how many GNU tests uutils passes. A more detailed +breakdown of the GNU test results of the main branch can be found +[in the user manual](https://uutils.github.io/coreutils-docs/user/test_coverage.html). + ![Evolution over time](https://github.com/uutils/coreutils-tracking/blob/main/gnu-results.png?raw=true) To run locally: diff --git a/build.rs b/build.rs index f4d4acbc1..c4f135e64 100644 --- a/build.rs +++ b/build.rs @@ -116,6 +116,8 @@ pub fn main() { phf_map.entry("sha3-512sum", &map_value); phf_map.entry("shake128sum", &map_value); phf_map.entry("shake256sum", &map_value); + phf_map.entry("b2sum", &map_value); + phf_map.entry("b3sum", &map_value); tf.write_all( format!( "#[path=\"{dir}/test_{krate}.rs\"]\nmod test_{krate};\n", diff --git a/docs/Makefile b/docs/Makefile index f56df90fb..7372b3868 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,5 +1,4 @@ -UseGNU=gmake $* -all: - @$(UseGNU) -.DEFAULT: - @$(UseGNU) +clean: + rm -rf book + rm -f src/SUMMARY.md + rm -f src/utils/* diff --git a/docs/src/test_coverage.css b/docs/src/test_coverage.css new file mode 100644 index 000000000..37a658695 --- /dev/null +++ b/docs/src/test_coverage.css @@ -0,0 +1,46 @@ +:root { + --PASS: #44AF69; + --ERROR: #F8333C; + --FAIL: #F8333C; + --SKIP: #d3c994; +} +.PASS { + color: var(--PASS); +} +.ERROR { + color: var(--ERROR); +} +.FAIL { + color: var(--FAIL); +} +.SKIP { + color: var(--SKIP); +} +.testSummary { + display: inline-flex; + align-items: center; + justify-content: space-between; + width: 90%; +} +.progress { + width: 80%; + display: flex; + justify-content: right; + align-items: center; +} +.progress-bar { + height: 10px; + width: calc(100% - 15ch); + border-radius: 5px; +} +.result { + font-weight: bold; + width: 7ch; + display: inline-block; +} +.result-line { + margin: 8px; +} +.counts { + margin-right: 10px; +} \ No newline at end of file diff --git a/docs/src/test_coverage.js b/docs/src/test_coverage.js new file mode 100644 index 000000000..e601229af --- /dev/null +++ b/docs/src/test_coverage.js @@ -0,0 +1,82 @@ +// spell-checker:ignore hljs +function progressBar(totals) { + const bar = document.createElement("div"); + bar.className = "progress-bar"; + let totalTests = 0; + for (const [key, value] of Object.entries(totals)) { + totalTests += value; + } + const passPercentage = Math.round(100 * totals["PASS"] / totalTests); + const skipPercentage = passPercentage + Math.round(100 * totals["SKIP"] / totalTests); + + // The ternary expressions are used for some edge-cases where there are no failing test, + // but still a red (or beige) line shows up because of how CSS draws gradients. + bar.style = `background: linear-gradient( + to right, + var(--PASS) ${passPercentage}%` + + ( passPercentage === 100 ? ", var(--PASS)" : + `, var(--SKIP) ${passPercentage}%, + var(--SKIP) ${skipPercentage}%` + ) + + (skipPercentage === 100 ? ")" : ", var(--FAIL) 0)"); + + const progress = document.createElement("div"); + progress.className = "progress" + progress.innerHTML = ` + + ${totals["PASS"]} + / + ${totals["SKIP"]} + / + ${totals["FAIL"] + totals["ERROR"]} + + `; + progress.appendChild(bar); + return progress +} + +function parse_result(parent, obj) { + const totals = { + PASS: 0, + SKIP: 0, + FAIL: 0, + ERROR: 0, + }; + for (const [category, content] of Object.entries(obj)) { + if (typeof content === "string") { + const p = document.createElement("p"); + p.className = "result-line"; + totals[content]++; + p.innerHTML = `${content} ${category}`; + parent.appendChild(p); + } else { + const categoryName = document.createElement("code"); + categoryName.innerHTML = category; + categoryName.className = "hljs"; + + const details = document.createElement("details"); + const subtotals = parse_result(details, content); + for (const [subtotal, count] of Object.entries(subtotals)) { + totals[subtotal] += count; + } + const summaryDiv = document.createElement("div"); + summaryDiv.className = "testSummary"; + summaryDiv.appendChild(categoryName); + summaryDiv.appendChild(progressBar(subtotals)); + + const summary = document.createElement("summary"); + summary.appendChild(summaryDiv); + + details.appendChild(summary); + parent.appendChild(details); + } + } + return totals; +} + +fetch("https://raw.githubusercontent.com/uutils/coreutils-tracking/main/gnu-full-result.json") + .then((r) => r.json()) + .then((obj) => { + let parent = document.getElementById("test-cov"); + parse_result(parent, obj); + }); diff --git a/docs/src/test_coverage.md b/docs/src/test_coverage.md new file mode 100644 index 000000000..bf4c72129 --- /dev/null +++ b/docs/src/test_coverage.md @@ -0,0 +1,19 @@ +# GNU Test Coverage + +uutils is actively tested against the GNU coreutils test suite. The results +below are automatically updated every day. + +## Coverage per category + +Click on the categories to see the names of the tests. Green indicates a passing +test, yellow indicates a skipped test and red means that the test either failed +or resulted in an error. + + + + +
+ +## Progress over time + + diff --git a/src/bin/coreutils.rs b/src/bin/coreutils.rs index 901139edc..fcd86c93f 100644 --- a/src/bin/coreutils.rs +++ b/src/bin/coreutils.rs @@ -87,7 +87,7 @@ fn main() { }; if util == "completion" { - gen_completions(args, utils); + gen_completions(args, &utils); } match utils.get(util) { @@ -132,7 +132,7 @@ fn main() { /// Prints completions for the utility in the first parameter for the shell in the second parameter to stdout fn gen_completions( args: impl Iterator, - util_map: UtilityMap, + util_map: &UtilityMap, ) -> ! { let all_utilities: Vec<_> = std::iter::once("coreutils") .chain(util_map.keys().copied()) @@ -168,9 +168,9 @@ fn gen_completions( process::exit(0); } -fn gen_coreutils_app(util_map: UtilityMap) -> App<'static> { +fn gen_coreutils_app(util_map: &UtilityMap) -> App<'static> { let mut app = App::new("coreutils"); - for (_, (_, sub_app)) in &util_map { + for (_, (_, sub_app)) in util_map { app = app.subcommand(sub_app()); } app diff --git a/src/bin/uudoc.rs b/src/bin/uudoc.rs index 412a2dd48..33e5bf607 100644 --- a/src/bin/uudoc.rs +++ b/src/bin/uudoc.rs @@ -2,15 +2,27 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. +// spell-checker:ignore tldr use clap::App; use std::ffi::OsString; use std::fs::File; -use std::io::{self, Write}; +use std::io::Cursor; +use std::io::{self, Read, Seek, Write}; +use zip::ZipArchive; include!(concat!(env!("OUT_DIR"), "/uutils_map.rs")); fn main() -> io::Result<()> { + println!("Downloading tldr archive"); + let mut zip_reader = ureq::get("https://tldr.sh/assets/tldr.zip") + .call() + .unwrap() + .into_reader(); + let mut buffer = Vec::new(); + zip_reader.read_to_end(&mut buffer).unwrap(); + let mut tldr_zip = ZipArchive::new(Cursor::new(buffer)).unwrap(); + let utils = util_map::>>(); match std::fs::create_dir("docs/src/utils/") { Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => Ok(()), @@ -26,6 +38,7 @@ fn main() -> io::Result<()> { [Introduction](index.md)\n\ * [Installation](installation.md)\n\ * [Contributing](contributing.md)\n\ + * [GNU test coverage](test_coverage.md)\n\ \n\ # Reference\n\ * [Multi-call binary](multicall.md)\n", @@ -39,7 +52,7 @@ fn main() -> io::Result<()> { } let p = format!("docs/src/utils/{}.md", name); if let Ok(f) = File::create(&p) { - write_markdown(f, &mut app(), name)?; + write_markdown(f, &mut app(), name, &mut tldr_zip)?; println!("Wrote to '{}'", p); } else { println!("Error writing to {}", p); @@ -49,12 +62,18 @@ fn main() -> io::Result<()> { Ok(()) } -fn write_markdown(mut w: impl Write, app: &mut App, name: &str) -> io::Result<()> { +fn write_markdown( + mut w: impl Write, + app: &mut App, + name: &str, + tldr_zip: &mut zip::ZipArchive, +) -> io::Result<()> { write!(w, "# {}\n\n", name)?; write_version(&mut w, app)?; write_usage(&mut w, app, name)?; write_description(&mut w, app)?; - write_options(&mut w, app) + write_options(&mut w, app)?; + write_examples(&mut w, name, tldr_zip) } fn write_version(w: &mut impl Write, app: &App) -> io::Result<()> { @@ -67,7 +86,14 @@ fn write_version(w: &mut impl Write, app: &App) -> io::Result<()> { fn write_usage(w: &mut impl Write, app: &mut App, name: &str) -> io::Result<()> { writeln!(w, "\n```")?; - let mut usage: String = app.render_usage().lines().nth(1).unwrap().trim().into(); + let mut usage: String = app + .render_usage() + .lines() + .skip(1) + .map(|l| l.trim()) + .filter(|l| !l.is_empty()) + .collect::>() + .join("\n"); usage = usage.replace(app.get_name(), name); writeln!(w, "{}", usage)?; writeln!(w, "```") @@ -81,6 +107,51 @@ fn write_description(w: &mut impl Write, app: &App) -> io::Result<()> { } } +fn write_examples( + w: &mut impl Write, + name: &str, + tldr_zip: &mut zip::ZipArchive, +) -> io::Result<()> { + let content = if let Some(f) = get_zip_content(tldr_zip, &format!("pages/common/{}.md", name)) { + f + } else if let Some(f) = get_zip_content(tldr_zip, &format!("pages/linux/{}.md", name)) { + f + } else { + return Ok(()); + }; + + writeln!(w, "## Examples")?; + writeln!(w)?; + for line in content.lines().skip_while(|l| !l.starts_with('-')) { + if let Some(l) = line.strip_prefix("- ") { + writeln!(w, "{}", l)?; + } else if line.starts_with('`') { + writeln!(w, "```shell\n{}\n```", line.trim_matches('`'))?; + } else if line.is_empty() { + writeln!(w)?; + } else { + println!("Not sure what to do with this line:"); + println!("{}", line); + } + } + writeln!(w)?; + writeln!( + w, + "> The examples are provided by the [tldr-pages project](https://tldr.sh) under the [CC BY 4.0 License](https://github.com/tldr-pages/tldr/blob/main/LICENSE.md)." + )?; + writeln!(w, ">")?; + writeln!( + w, + "> Please note that, as uutils is a work in progress, some examples might fail." + ) +} + +fn get_zip_content(archive: &mut ZipArchive, name: &str) -> Option { + let mut s = String::new(); + archive.by_name(name).ok()?.read_to_string(&mut s).unwrap(); + Some(s) +} + fn write_options(w: &mut impl Write, app: &App) -> io::Result<()> { writeln!(w, "

Options

")?; write!(w, "
")?; @@ -130,7 +201,11 @@ fn write_options(w: &mut impl Write, app: &App) -> io::Result<()> { write!(w, "")?; } writeln!(w, "")?; - writeln!(w, "
\n\n{}\n\n
", arg.get_help().unwrap_or_default())?; + writeln!( + w, + "
\n\n{}\n\n
", + arg.get_help().unwrap_or_default().replace("\n", "
") + )?; } - writeln!(w, "
") + writeln!(w, "\n") } diff --git a/src/uu/base32/src/base32.rs b/src/uu/base32/src/base32.rs index 6d9759fa4..006a796f0 100644 --- a/src/uu/base32/src/base32.rs +++ b/src/uu/base32/src/base32.rs @@ -12,14 +12,14 @@ use uucore::{encoding::Format, error::UResult}; pub mod base_common; -static ABOUT: &str = " - With no FILE, or when FILE is -, read standard input. +static ABOUT: &str = "\ +With no FILE, or when FILE is -, read standard input. - The data are encoded as described for the base32 alphabet in RFC - 4648. When decoding, the input may contain newlines in addition - to the bytes of the formal base32 alphabet. Use --ignore-garbage - to attempt to recover from any other non-alphabet bytes in the - encoded stream. +The data are encoded as described for the base32 alphabet in RFC +4648. When decoding, the input may contain newlines in addition +to the bytes of the formal base32 alphabet. Use --ignore-garbage +to attempt to recover from any other non-alphabet bytes in the +encoded stream. "; fn usage() -> String { diff --git a/src/uu/base64/src/base64.rs b/src/uu/base64/src/base64.rs index 6d0192df9..20a9f55a5 100644 --- a/src/uu/base64/src/base64.rs +++ b/src/uu/base64/src/base64.rs @@ -13,14 +13,14 @@ use uucore::{encoding::Format, error::UResult}; use std::io::{stdin, Read}; -static ABOUT: &str = " - With no FILE, or when FILE is -, read standard input. +static ABOUT: &str = "\ +With no FILE, or when FILE is -, read standard input. - The data are encoded as described for the base64 alphabet in RFC - 3548. When decoding, the input may contain newlines in addition - to the bytes of the formal base64 alphabet. Use --ignore-garbage - to attempt to recover from any other non-alphabet bytes in the - encoded stream. +The data are encoded as described for the base64 alphabet in RFC +3548. When decoding, the input may contain newlines in addition +to the bytes of the formal base64 alphabet. Use --ignore-garbage +to attempt to recover from any other non-alphabet bytes in the +encoded stream. "; fn usage() -> String { diff --git a/src/uu/basenc/src/basenc.rs b/src/uu/basenc/src/basenc.rs index c21e224da..ef133b151 100644 --- a/src/uu/basenc/src/basenc.rs +++ b/src/uu/basenc/src/basenc.rs @@ -19,12 +19,12 @@ use uucore::{ use std::io::{stdin, Read}; -static ABOUT: &str = " - With no FILE, or when FILE is -, read standard input. +static ABOUT: &str = "\ +With no FILE, or when FILE is -, read standard input. - When decoding, the input may contain newlines in addition to the bytes of - the formal alphabet. Use --ignore-garbage to attempt to recover - from any other non-alphabet bytes in the encoded stream. +When decoding, the input may contain newlines in addition to the bytes of +the formal alphabet. Use --ignore-garbage to attempt to recover +from any other non-alphabet bytes in the encoded stream. "; const ENCODINGS: &[(&str, Format)] = &[ diff --git a/src/uu/cat/src/cat.rs b/src/uu/cat/src/cat.rs index e7fd31497..498e5e8ad 100644 --- a/src/uu/cat/src/cat.rs +++ b/src/uu/cat/src/cat.rs @@ -325,15 +325,15 @@ fn cat_path( state: &mut OutputState, out_info: Option<&FileInformation>, ) -> CatResult<()> { - if path == "-" { - let stdin = io::stdin(); - let mut handle = InputHandle { - reader: stdin, - is_interactive: atty::is(atty::Stream::Stdin), - }; - return cat_handle(&mut handle, options, state); - } match get_input_type(path)? { + InputType::StdIn => { + let stdin = io::stdin(); + let mut handle = InputHandle { + reader: stdin, + is_interactive: atty::is(atty::Stream::Stdin), + }; + cat_handle(&mut handle, options, state) + } InputType::Directory => Err(CatError::IsDirectory), #[cfg(unix)] InputType::Socket => { @@ -560,13 +560,12 @@ fn write_tab_to_end(mut in_buf: &[u8], writer: &mut W) -> usize { { Some(p) => { writer.write_all(&in_buf[..p]).unwrap(); - if in_buf[p] == b'\n' { - return count + p; - } else if in_buf[p] == b'\t' { + if in_buf[p] == b'\t' { writer.write_all(b"^I").unwrap(); in_buf = &in_buf[p + 1..]; count += p + 1; } else { + // b'\n' or b'\r' return count + p; } } @@ -589,10 +588,10 @@ fn write_nonprint_to_end(in_buf: &[u8], writer: &mut W, tab: &[u8]) -> 9 => writer.write_all(tab), 0..=8 | 10..=31 => writer.write_all(&[b'^', byte + 64]), 32..=126 => writer.write_all(&[byte]), - 127 => writer.write_all(&[b'^', byte - 64]), + 127 => writer.write_all(&[b'^', b'?']), 128..=159 => writer.write_all(&[b'M', b'-', b'^', byte - 64]), 160..=254 => writer.write_all(&[b'M', b'-', byte - 128]), - _ => writer.write_all(&[b'M', b'-', b'^', 63]), + _ => writer.write_all(&[b'M', b'-', b'^', b'?']), } .unwrap(); count += 1; diff --git a/src/uu/chroot/src/chroot.rs b/src/uu/chroot/src/chroot.rs index 179880b14..f656ed77d 100644 --- a/src/uu/chroot/src/chroot.rs +++ b/src/uu/chroot/src/chroot.rs @@ -201,12 +201,12 @@ fn set_main_group(group: &str) -> UResult<()> { } #[cfg(any(target_vendor = "apple", target_os = "freebsd"))] -fn set_groups(groups: Vec) -> libc::c_int { +fn set_groups(groups: &[libc::gid_t]) -> libc::c_int { unsafe { setgroups(groups.len() as libc::c_int, groups.as_ptr()) } } #[cfg(target_os = "linux")] -fn set_groups(groups: Vec) -> libc::c_int { +fn set_groups(groups: &[libc::gid_t]) -> libc::c_int { unsafe { setgroups(groups.len() as libc::size_t, groups.as_ptr()) } } @@ -220,7 +220,7 @@ fn set_groups_from_str(groups: &str) -> UResult<()> { }; groups_vec.push(gid); } - let err = set_groups(groups_vec); + let err = set_groups(&groups_vec); if err != 0 { return Err(ChrootError::SetGroupsFailed(Error::last_os_error()).into()); } diff --git a/src/uu/cp/src/cp.rs b/src/uu/cp/src/cp.rs index fa9a1df65..eeb7cf06f 100644 --- a/src/uu/cp/src/cp.rs +++ b/src/uu/cp/src/cp.rs @@ -1025,7 +1025,10 @@ fn copy_directory( if is_symlink && !options.dereference { copy_link(&path, &local_to_target, symlinked_files)?; } else if path.is_dir() && !local_to_target.exists() { - or_continue!(fs::create_dir_all(local_to_target)); + if target.is_file() { + return Err("cannot overwrite non-directory with directory".into()); + } + fs::create_dir_all(local_to_target)?; } else if !path.is_dir() { if preserve_hard_links { let mut found_hard_link = false; diff --git a/src/uu/dd/src/datastructures.rs b/src/uu/dd/src/datastructures.rs index 8fab1ffec..c9c89e858 100644 --- a/src/uu/dd/src/datastructures.rs +++ b/src/uu/dd/src/datastructures.rs @@ -7,50 +7,11 @@ // spell-checker:ignore ctable, outfile use std::error::Error; -use std::time; use uucore::error::UError; use crate::conversion_tables::*; -pub struct ProgUpdate { - pub read_stat: ReadStat, - pub write_stat: WriteStat, - pub duration: time::Duration, -} - -#[derive(Clone, Copy, Default)] -pub struct ReadStat { - pub reads_complete: u64, - pub reads_partial: u64, - pub records_truncated: u32, -} -impl std::ops::AddAssign for ReadStat { - fn add_assign(&mut self, other: Self) { - *self = Self { - reads_complete: self.reads_complete + other.reads_complete, - reads_partial: self.reads_partial + other.reads_partial, - records_truncated: self.records_truncated + other.records_truncated, - } - } -} - -#[derive(Clone, Copy)] -pub struct WriteStat { - pub writes_complete: u64, - pub writes_partial: u64, - pub bytes_total: u128, -} -impl std::ops::AddAssign for WriteStat { - fn add_assign(&mut self, other: Self) { - *self = Self { - writes_complete: self.writes_complete + other.writes_complete, - writes_partial: self.writes_partial + other.writes_partial, - bytes_total: self.bytes_total + other.bytes_total, - } - } -} - type Cbs = usize; /// Stores all Conv Flags that apply to the input @@ -116,15 +77,6 @@ pub struct OFlags { pub seek_bytes: bool, } -/// The value of the status cl-option. -/// Controls printing of transfer stats -#[derive(Copy, Clone, Debug, PartialEq)] -pub enum StatusLevel { - Progress, - Noxfer, - None, -} - /// The value of count=N /// Defaults to Reads(N) /// if iflag=count_bytes diff --git a/src/uu/dd/src/dd.rs b/src/uu/dd/src/dd.rs index 54e3190ce..c8004b893 100644 --- a/src/uu/dd/src/dd.rs +++ b/src/uu/dd/src/dd.rs @@ -5,10 +5,7 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore fname, tname, fpath, specfile, testfile, unspec, ifile, ofile, outfile, fullblock, urand, fileio, atoe, atoibm, behaviour, bmax, bremain, btotal, cflags, creat, ctable, ctty, datastructures, doesnt, etoa, fileout, fname, gnudd, iconvflags, nocache, noctty, noerror, nofollow, nolinks, nonblock, oconvflags, outfile, parseargs, rlen, rmax, rposition, rremain, rsofar, rstat, sigusr, sigval, wlen, wstat seekable - -#[cfg(test)] -mod dd_unit_tests; +// spell-checker:ignore fname, tname, fpath, specfile, testfile, unspec, ifile, ofile, outfile, fullblock, urand, fileio, atoe, atoibm, behaviour, bmax, bremain, cflags, creat, ctable, ctty, datastructures, doesnt, etoa, fileout, fname, gnudd, iconvflags, nocache, noctty, noerror, nofollow, nolinks, nonblock, oconvflags, outfile, parseargs, rlen, rmax, rposition, rremain, rsofar, rstat, sigusr, wlen, wstat seekable mod datastructures; use datastructures::*; @@ -19,29 +16,26 @@ use parseargs::Matches; mod conversion_tables; use conversion_tables::*; +mod progress; +use progress::{gen_prog_updater, ProgUpdate, ReadStat, StatusLevel, WriteStat}; + use std::cmp; use std::convert::TryInto; use std::env; -#[cfg(target_os = "linux")] -use std::error::Error; use std::fs::{File, OpenOptions}; use std::io::{self, Read, Seek, Write}; #[cfg(target_os = "linux")] use std::os::unix::fs::OpenOptionsExt; use std::path::Path; use std::sync::mpsc; -#[cfg(target_os = "linux")] -use std::sync::{atomic::AtomicUsize, atomic::Ordering, Arc}; use std::thread; use std::time; -use byte_unit::Byte; use clap::{crate_version, App, AppSettings, Arg, ArgMatches}; use gcd::Gcd; -#[cfg(target_os = "linux")] -use signal_hook::consts::signal; use uucore::display::Quotable; use uucore::error::{FromIo, UResult, USimpleError}; +use uucore::show_error; use uucore::InvalidEncodingHandling; const ABOUT: &str = "copy, and optionally convert, a file system resource"; @@ -80,9 +74,12 @@ impl Input { }; if let Some(amt) = skip { - let mut buf = vec![BUF_INIT_BYTE; amt]; - i.force_fill(&mut buf, amt) + let num_bytes_read = i + .force_fill(amt.try_into().unwrap()) .map_err_context(|| "failed to read input".to_string())?; + if num_bytes_read < amt { + show_error!("'standard input': cannot skip to specified offset"); + } } Ok(i) @@ -264,17 +261,19 @@ impl Input { }) } - /// Force-fills a buffer, ignoring zero-length reads which would otherwise be - /// interpreted as EOF. - /// Note: This will not return unless the source (eventually) produces - /// enough bytes to meet target_len. - fn force_fill(&mut self, buf: &mut [u8], target_len: usize) -> std::io::Result { - let mut base_idx = 0; - while base_idx < target_len { - base_idx += self.read(&mut buf[base_idx..target_len])?; - } - - Ok(base_idx) + /// Read the specified number of bytes from this reader. + /// + /// On success, this method returns the number of bytes read. If + /// this reader has fewer than `n` bytes available, then it reads + /// as many as possible. In that case, this method returns a + /// number less than `n`. + /// + /// # Errors + /// + /// If there is a problem reading. + fn force_fill(&mut self, n: u64) -> std::io::Result { + let mut buf = vec![]; + self.take(n).read_to_end(&mut buf) } } @@ -328,16 +327,13 @@ where let mut bytes_total = 0; for chunk in buf.chunks(self.obs) { - match self.write(chunk)? { - wlen if wlen < chunk.len() => { - writes_partial += 1; - bytes_total += wlen; - } - wlen => { - writes_complete += 1; - bytes_total += wlen; - } + let wlen = self.write(chunk)?; + if wlen < self.obs { + writes_partial += 1; + } else { + writes_complete += 1; } + bytes_total += wlen; } Ok(WriteStat { @@ -347,78 +343,111 @@ where }) } - fn dd_out(mut self, mut i: Input) -> UResult<()> { - let mut rstat = ReadStat { - reads_complete: 0, - reads_partial: 0, - records_truncated: 0, - }; - let mut wstat = WriteStat { - writes_complete: 0, - writes_partial: 0, - bytes_total: 0, - }; - let start = time::Instant::now(); - let bsize = calc_bsize(i.ibs, self.obs); - - let prog_tx = { - let (tx, rx) = mpsc::channel(); - thread::spawn(gen_prog_updater(rx, i.print_level)); - tx - }; - - while below_count_limit(&i.count, &rstat, &wstat) { - // Read/Write - let loop_bsize = calc_loop_bsize(&i.count, &rstat, &wstat, i.ibs, bsize); - match read_helper(&mut i, loop_bsize)? { - ( - ReadStat { - reads_complete: 0, - reads_partial: 0, - .. - }, - _, - ) => break, - (rstat_update, buf) => { - let wstat_update = self - .write_blocks(&buf) - .map_err_context(|| "failed to write output".to_string())?; - - rstat += rstat_update; - wstat += wstat_update; - } - }; - // Update Prog - prog_tx - .send(ProgUpdate { - read_stat: rstat, - write_stat: wstat, - duration: start.elapsed(), - }) - .map_err(|_| USimpleError::new(1, "failed to write output"))?; - } - - if self.cflags.fsync { - self.fsync() - .map_err_context(|| "failed to write output".to_string())?; - } else if self.cflags.fdatasync { - self.fdatasync() - .map_err_context(|| "failed to write output".to_string())?; - } - + /// Print the read/write statistics. + fn print_stats(&self, i: &Input, prog_update: &ProgUpdate) { match i.print_level { Some(StatusLevel::None) => {} - Some(StatusLevel::Noxfer) => print_io_lines(&ProgUpdate { - read_stat: rstat, - write_stat: wstat, - duration: start.elapsed(), - }), - Some(StatusLevel::Progress) | None => print_transfer_stats(&ProgUpdate { - read_stat: rstat, - write_stat: wstat, - duration: start.elapsed(), - }), + Some(StatusLevel::Noxfer) => prog_update.print_io_lines(), + Some(StatusLevel::Progress) | None => prog_update.print_transfer_stats(), } + } + + /// Flush the output to disk, if configured to do so. + fn sync(&mut self) -> std::io::Result<()> { + if self.cflags.fsync { + self.fsync() + } else if self.cflags.fdatasync { + self.fdatasync() + } else { + // Intentionally do nothing in this case. + Ok(()) + } + } + + /// Copy the given input data to this output, consuming both. + /// + /// This method contains the main loop for the `dd` program. Bytes + /// are read in blocks from `i` and written in blocks to this + /// output. Read/write statistics are reported to stderr as + /// configured by the `status` command-line argument. + /// + /// # Errors + /// + /// If there is a problem reading from the input or writing to + /// this output. + fn dd_out(mut self, mut i: Input) -> std::io::Result<()> { + // The read and write statistics. + // + // These objects are counters, initialized to zero. After each + // iteration of the main loop, each will be incremented by the + // number of blocks read and written, respectively. + let mut rstat = Default::default(); + let mut wstat = Default::default(); + + // The time at which the main loop starts executing. + // + // When `status=progress` is given on the command-line, the + // `dd` program reports its progress every second or so. Part + // of its report includes the throughput in bytes per second, + // which requires knowing how long the process has been + // running. + let start = time::Instant::now(); + + // A good buffer size for reading. + // + // This is an educated guess about a good buffer size based on + // the input and output block sizes. + let bsize = calc_bsize(i.ibs, self.obs); + + // Start a thread that reports transfer progress. + // + // When `status=progress` is given on the command-line, the + // `dd` program reports its progress every second or so. We + // perform this reporting in a new thread so as not to take + // any CPU time away from the actual reading and writing of + // data. We send a `ProgUpdate` from the transmitter `prog_tx` + // to the receives `rx`, and the receiver prints the transfer + // information. + let (prog_tx, rx) = mpsc::channel(); + thread::spawn(gen_prog_updater(rx, i.print_level)); + + // The main read/write loop. + // + // Each iteration reads blocks from the input and writes + // blocks to this output. Read/write statistics are updated on + // each iteration and cumulative statistics are reported to + // the progress reporting thread. + while below_count_limit(&i.count, &rstat, &wstat) { + // Read a block from the input then write the block to the output. + // + // As an optimization, make an educated guess about the + // best buffer size for reading based on the number of + // blocks already read and the number of blocks remaining. + let loop_bsize = calc_loop_bsize(&i.count, &rstat, &wstat, i.ibs, bsize); + let (rstat_update, buf) = read_helper(&mut i, loop_bsize)?; + if rstat_update.is_empty() { + break; + } + let wstat_update = self.write_blocks(&buf)?; + + // Update the read/write stats and inform the progress thread. + // + // If the receiver is disconnected, `send()` returns an + // error. Since it is just reporting progress and is not + // crucial to the operation of `dd`, let's just ignore the + // error. + rstat += rstat_update; + wstat += wstat_update; + let prog_update = ProgUpdate::new(rstat, wstat, start.elapsed()); + prog_tx.send(prog_update).unwrap_or(()); + } + + // Flush the output, if configured to do so. + self.sync()?; + + // Print the final read/write statistics. + let prog_update = ProgUpdate::new(rstat, wstat, start.elapsed()); + self.print_stats(&i, &prog_update); Ok(()) } } @@ -469,7 +498,6 @@ impl OutputTrait for Output { let mut opts = OpenOptions::new(); opts.write(true) .create(!cflags.nocreat) - .truncate(!cflags.notrunc) .create_new(cflags.excl) .append(oflags.append); @@ -489,13 +517,13 @@ impl OutputTrait for Output { let mut dst = open_dst(Path::new(&fname), &cflags, &oflags) .map_err_context(|| format!("failed to open {}", fname.quote()))?; - if let Some(amt) = seek { - let amt: u64 = amt - .try_into() - .map_err(|_| USimpleError::new(1, "failed to parse seek amount"))?; - dst.seek(io::SeekFrom::Start(amt)) - .map_err_context(|| "failed to seek in output file".to_string())?; + let i = seek.unwrap_or(0).try_into().unwrap(); + if !cflags.notrunc { + dst.set_len(i) + .map_err_context(|| "failed to truncate output file".to_string())?; } + dst.seek(io::SeekFrom::Start(i)) + .map_err_context(|| "failed to seek in output file".to_string())?; Ok(Self { dst, obs, cflags }) } else { @@ -698,7 +726,7 @@ fn conv_block_unblock_helper( } /// Read helper performs read operations common to all dd reads, and dispatches the buffer to relevant helper functions as dictated by the operations requested by the user. -fn read_helper(i: &mut Input, bsize: usize) -> UResult<(ReadStat, Vec)> { +fn read_helper(i: &mut Input, bsize: usize) -> std::io::Result<(ReadStat, Vec)> { // Local Predicate Fns ----------------------------------------------- fn is_conv(i: &Input) -> bool { i.cflags.ctable.is_some() @@ -719,12 +747,8 @@ fn read_helper(i: &mut Input, bsize: usize) -> UResult<(ReadStat, Ve // Read let mut buf = vec![BUF_INIT_BYTE; bsize]; let mut rstat = match i.cflags.sync { - Some(ch) => i - .fill_blocks(&mut buf, ch) - .map_err_context(|| "failed to write output".to_string())?, - _ => i - .fill_consecutive(&mut buf) - .map_err_context(|| "failed to write output".to_string())?, + Some(ch) => i.fill_blocks(&mut buf, ch)?, + _ => i.fill_consecutive(&mut buf)?, }; // Return early if no data if rstat.reads_complete == 0 && rstat.reads_partial == 0 { @@ -736,116 +760,13 @@ fn read_helper(i: &mut Input, bsize: usize) -> UResult<(ReadStat, Ve perform_swab(&mut buf); } if is_conv(i) || is_block(i) || is_unblock(i) { - let buf = conv_block_unblock_helper(buf, i, &mut rstat)?; + let buf = conv_block_unblock_helper(buf, i, &mut rstat).unwrap(); Ok((rstat, buf)) } else { Ok((rstat, buf)) } } -// Print io lines of a status update: -// + records in -// + records out -fn print_io_lines(update: &ProgUpdate) { - eprintln!( - "{}+{} records in", - update.read_stat.reads_complete, update.read_stat.reads_partial - ); - if update.read_stat.records_truncated > 0 { - eprintln!("{} truncated records", update.read_stat.records_truncated); - } - eprintln!( - "{}+{} records out", - update.write_stat.writes_complete, update.write_stat.writes_partial - ); -} -// Print the progress line of a status update: -// bytes (, ) copied,