From 96d3a95a3920b0931a87c167b29e603779e617e3 Mon Sep 17 00:00:00 2001 From: Terts Diepraam Date: Sun, 30 Jan 2022 14:32:08 +0100 Subject: [PATCH 01/78] test: fix wsl executable permission --- tests/by-util/test_test.rs | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/tests/by-util/test_test.rs b/tests/by-util/test_test.rs index db74265a4..8500d63ed 100644 --- a/tests/by-util/test_test.rs +++ b/tests/by-util/test_test.rs @@ -440,7 +440,27 @@ fn test_file_is_not_writable() { #[test] fn test_file_is_not_executable() { - new_ucmd!().args(&["!", "-x", "regular_file"]).succeeds(); + #[cfg(unix)] + let (at, mut ucmd) = at_and_ucmd!(); + #[cfg(not(unix))] + let (_, mut ucmd) = at_and_ucmd!(); + + // WSL creates executable files by default, so if we are on unix, make sure + // to set make it non-executable. + // Files on other targets are non-executable by default. + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let metadata = std::fs::metadata(at.plus("regular_file")).unwrap(); + let mut permissions = metadata.permissions(); + + // The conversion is useless on some platforms and casts from u16 to + // u32 on others + #[allow(clippy::useless_conversion)] + permissions.set_mode(permissions.mode() & !u32::from(libc::S_IXUSR)); + std::fs::set_permissions(at.plus("regular_file"), permissions).unwrap(); + } + ucmd.args(&["!", "-x", "regular_file"]).succeeds(); } #[test] From be6287e3e30e14b6c9c455217846cb3acdc9c282 Mon Sep 17 00:00:00 2001 From: Narasimha Prasanna HN Date: Tue, 1 Feb 2022 17:37:04 +0530 Subject: [PATCH 02/78] Fix: Avoid infinite recursive copies when source and destination directories are same or source is a prefix of destination --- src/uu/cp/src/cp.rs | 17 +++++++++++++++ tests/by-util/test_cp.rs | 45 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/src/uu/cp/src/cp.rs b/src/uu/cp/src/cp.rs index f8ce6f241..938ecfe03 100644 --- a/src/uu/cp/src/cp.rs +++ b/src/uu/cp/src/cp.rs @@ -969,6 +969,16 @@ fn copy_directory( return copy_file(root, target, options, symlinked_files); } + // check if root is a prefix of target + if path_has_prefix(target, root)? { + return Err(format!( + "cannot copy a directory, {}, into itself, {}", + root.quote(), + target.quote() + ) + .into()); + } + let current_dir = env::current_dir().unwrap_or_else(|e| crash!(1, "failed to get current directory {}", e)); @@ -1570,6 +1580,13 @@ pub fn paths_refer_to_same_file(p1: &Path, p2: &Path) -> io::Result { Ok(pathbuf1 == pathbuf2) } +pub fn path_has_prefix(p1: &Path, p2: &Path) -> io::Result { + let pathbuf1 = canonicalize(p1, MissingHandling::Normal, ResolveMode::Logical)?; + let pathbuf2 = canonicalize(p2, MissingHandling::Normal, ResolveMode::Logical)?; + + Ok(pathbuf1.starts_with(pathbuf2)) +} + #[test] fn test_cp_localize_to_target() { assert!( diff --git a/tests/by-util/test_cp.rs b/tests/by-util/test_cp.rs index 92637dfbe..0a4dfd16d 100644 --- a/tests/by-util/test_cp.rs +++ b/tests/by-util/test_cp.rs @@ -1444,3 +1444,48 @@ fn test_cp_archive_on_nonexistent_file() { "cp: cannot stat 'nonexistent_file.txt': No such file or directory (os error 2)", ); } + +#[test] +fn test_dir_recursive_copy() { + let scene = TestScenario::new(util_name!()); + let at = &scene.fixtures; + + at.mkdir("parent1"); + at.mkdir("parent2"); + at.mkdir("parent1/child"); + at.mkdir("parent2/child1"); + at.mkdir("parent2/child1/child2"); + at.mkdir("parent2/child1/child2/child3"); + + // case-1: copy parent1 -> parent1: should fail + scene + .ucmd() + .arg("-R") + .arg("parent1") + .arg("parent1") + .fails() + .stderr_contains("cannot copy a directory"); + // case-2: copy parent1 -> parent1/child should fail + scene + .ucmd() + .arg("-R") + .arg("parent1") + .arg("parent1/child") + .fails() + .stderr_contains("cannot copy a directory"); + // case-3: copy parent1/child -> parent2 should pass + scene + .ucmd() + .arg("-R") + .arg("parent1/child") + .arg("parent2") + .succeeds(); + // case-4: copy parent2/child1/ -> parent2/child1/child2/child3 + scene + .ucmd() + .arg("-R") + .arg("parent2/child1/") + .arg("parent2/child1/child2/child3") + .fails() + .stderr_contains("cannot copy a directory"); +} From f39b861469c8d7598741864f2602c15a5496287b Mon Sep 17 00:00:00 2001 From: Hanif Ariffin Date: Sat, 5 Feb 2022 19:12:05 +0800 Subject: [PATCH 03/78] Refactor padding calculations into a function Signed-off-by: Hanif Ariffin --- src/uu/ls/src/ls.rs | 184 +++++++++++++++++++++++--------------------- 1 file changed, 97 insertions(+), 87 deletions(-) diff --git a/src/uu/ls/src/ls.rs b/src/uu/ls/src/ls.rs index 7fdec53f0..e86744955 100644 --- a/src/uu/ls/src/ls.rs +++ b/src/uu/ls/src/ls.rs @@ -1275,9 +1275,9 @@ only ignore '.' and '..'.", ) } -/// Represents a Path along with it's associated data -/// Any data that will be reused several times makes sense to be added to this structure -/// Caching data here helps eliminate redundant syscalls to fetch same information +/// Represents a Path along with it's associated data. +/// Any data that will be reused several times makes sense to be added to this structure. +/// Caching data here helps eliminate redundant syscalls to fetch same information. #[derive(Debug)] struct PathData { // Result got from symlink_metadata() or metadata() based on config @@ -1678,92 +1678,10 @@ fn display_items(items: &[PathData], config: &Config, out: &mut BufWriter, +) -> PaddingCollection { + let ( + mut longest_inode_len, + mut longest_link_count_len, + mut longest_uname_len, + mut longest_group_len, + mut longest_context_len, + mut longest_size_len, + mut longest_major_len, + mut longest_minor_len, + ) = (1, 1, 1, 1, 1, 1, 1, 1); + + for item in items { + let context_len = item.security_context.len(); + let (link_count_len, uname_len, group_len, size_len, major_len, minor_len, inode_len) = + display_dir_entry_size(item, config, out); + longest_inode_len = inode_len.max(longest_inode_len); + longest_link_count_len = link_count_len.max(longest_link_count_len); + longest_uname_len = uname_len.max(longest_uname_len); + longest_group_len = group_len.max(longest_group_len); + if config.context { + longest_context_len = context_len.max(longest_context_len); + } + if items.len() == 1usize { + longest_size_len = 0usize; + longest_major_len = 0usize; + longest_minor_len = 0usize; + } else { + longest_major_len = major_len.max(longest_major_len); + longest_minor_len = minor_len.max(longest_minor_len); + longest_size_len = size_len + .max(longest_size_len) + .max(longest_major_len + longest_minor_len + 2usize); + } + } + + PaddingCollection { + longest_inode_len, + longest_link_count_len, + longest_uname_len, + longest_group_len, + longest_context_len, + longest_size_len, + longest_major_len, + longest_minor_len, + } +} + +#[cfg(not(unix))] +fn calculate_paddings( + items: &[PathData], + config: &Config, + out: &mut BufWriter, +) -> PaddingCollection { + let ( + mut longest_link_count_len, + mut longest_uname_len, + mut longest_group_len, + mut longest_context_len, + mut longest_size_len, + ) = (1, 1, 1, 1, 1); + + for item in items { + let context_len = item.security_context.len(); + let (link_count_len, uname_len, group_len, size_len, _major_len, _minor_len, _inode_len) = + display_dir_entry_size(item, config, out); + longest_link_count_len = link_count_len.max(longest_link_count_len); + longest_uname_len = uname_len.max(longest_uname_len); + longest_group_len = group_len.max(longest_group_len); + if config.context { + longest_context_len = context_len.max(longest_context_len); + } + longest_size_len = size_len.max(longest_size_len); + } + + PaddingCollection { + longest_inode_len, + longest_link_count_len, + longest_uname_len, + longest_group_len, + longest_context_len, + longest_size_len, + longest_major_len, + longest_minor_len, + } +} From e35b93156ab31cb7e563ddbb342df4aa52787ef4 Mon Sep 17 00:00:00 2001 From: Hanif Ariffin Date: Sat, 5 Feb 2022 19:30:39 +0800 Subject: [PATCH 04/78] Propagate all write and (most) flush errors Signed-off-by: Hanif Ariffin --- src/uu/ls/src/ls.rs | 127 +++++++++++++++++++++++--------------------- 1 file changed, 67 insertions(+), 60 deletions(-) diff --git a/src/uu/ls/src/ls.rs b/src/uu/ls/src/ls.rs index e86744955..f118a7594 100644 --- a/src/uu/ls/src/ls.rs +++ b/src/uu/ls/src/ls.rs @@ -1379,7 +1379,8 @@ impl PathData { // if not, check if we can use Path metadata match get_metadata(self.p_buf.as_path(), self.must_dereference) { Err(err) => { - let _ = out.flush(); + // FIXME: A bit tricky to propagate the result here + out.flush().unwrap(); let errno = err.raw_os_error().unwrap_or(1i32); // a bad fd will throw an error when dereferenced, // but GNU will not throw an error until a bad fd "dir" @@ -1443,7 +1444,7 @@ fn list(locs: Vec<&Path>, config: &Config) -> UResult<()> { sort_entries(&mut files, config, &mut out); sort_entries(&mut dirs, config, &mut out); - display_items(&files, config, &mut out); + display_items(&files, config, &mut out)?; for (pos, path_data) in dirs.iter().enumerate() { // Do read_dir call here to match GNU semantics by printing @@ -1451,7 +1452,7 @@ fn list(locs: Vec<&Path>, config: &Config) -> UResult<()> { let read_dir = match fs::read_dir(&path_data.p_buf) { Err(err) => { // flush stdout buffer before the error to preserve formatting and order - let _ = out.flush(); + out.flush()?; show!(LsError::IOErrorContext(err, path_data.p_buf.clone())); continue; } @@ -1461,12 +1462,12 @@ fn list(locs: Vec<&Path>, config: &Config) -> UResult<()> { // Print dir heading - name... 'total' comes after error display if initial_locs_len > 1 || config.recursive { if pos.eq(&0usize) && files.is_empty() { - let _ = writeln!(out, "{}:", path_data.p_buf.display()); + writeln!(out, "{}:", path_data.p_buf.display())?; } else { - let _ = writeln!(out, "\n{}:", path_data.p_buf.display()); + writeln!(out, "\n{}:", path_data.p_buf.display())?; } } - enter_directory(path_data, read_dir, config, &mut out); + enter_directory(path_data, read_dir, config, &mut out)?; } Ok(()) @@ -1540,7 +1541,7 @@ fn enter_directory( read_dir: ReadDir, config: &Config, out: &mut BufWriter, -) { +) -> UResult<()> { // Create vec of entries with initial dot files let mut entries: Vec = if config.files == Files::All { vec![ @@ -1570,7 +1571,7 @@ fn enter_directory( let dir_entry = match raw_entry { Ok(path) => path, Err(err) => { - let _ = out.flush(); + out.flush()?; show!(LsError::IOError(err)); continue; } @@ -1588,10 +1589,10 @@ fn enter_directory( // Print total after any error display if config.format == Format::Long { - display_total(&entries, config, out); + display_total(&entries, config, out)?; } - display_items(&entries, config, out); + display_items(&entries, config, out)?; if config.recursive { for e in entries @@ -1603,17 +1604,19 @@ fn enter_directory( { match fs::read_dir(&e.p_buf) { Err(err) => { - let _ = out.flush(); + out.flush()?; show!(LsError::IOErrorContext(err, e.p_buf.clone())); continue; } Ok(rd) => { - let _ = writeln!(out, "\n{}:", e.p_buf.display()); - enter_directory(e, rd, config, out); + writeln!(out, "\n{}:", e.p_buf.display())?; + enter_directory(e, rd, config, out)?; } } } } + + Ok(()) } fn get_metadata(p_buf: &Path, dereference: bool) -> std::io::Result { @@ -1661,7 +1664,7 @@ fn pad_right(string: &str, count: usize) -> String { format!("{:) { +fn display_total(items: &[PathData], config: &Config, out: &mut BufWriter) -> UResult<()> { let mut total_size = 0; for item in items { total_size += item @@ -1669,19 +1672,19 @@ fn display_total(items: &[PathData], config: &Config, out: &mut BufWriter) { +fn display_items(items: &[PathData], config: &Config, out: &mut BufWriter) -> UResult<()> { // `-Z`, `--context`: // Display the SELinux security context or '?' if none is found. When used with the `-l` // option, print the security context to the left of the size column. if config.format == Format::Long { let padding_collection = calculate_padding_collection(items, config, out); - for item in items { - display_item_long(item, &padding_collection, config, out); + display_item_long(item, &padding_collection, config, out)?; } } else { let mut longest_context_len = 1; @@ -1718,13 +1721,13 @@ fn display_items(items: &[PathData], config: &Config, out: &mut BufWriter display_grid(names, config.width, Direction::TopToBottom, out), - Format::Across => display_grid(names, config.width, Direction::LeftToRight, out), + Format::Columns => display_grid(names, config.width, Direction::TopToBottom, out)?, + Format::Across => display_grid(names, config.width, Direction::LeftToRight, out)?, Format::Commas => { let mut current_col = 0; let mut names = names; if let Some(name) = names.next() { - let _ = write!(out, "{}", name.contents); + write!(out, "{}", name.contents)?; current_col = name.width as u16 + 2; } for name in names { @@ -1732,25 +1735,27 @@ fn display_items(items: &[PathData], config: &Config, out: &mut BufWriter config.width { current_col = name_width + 2; - let _ = write!(out, ",\n{}", name.contents); + write!(out, ",\n{}", name.contents)?; } else { current_col += name_width + 2; - let _ = write!(out, ", {}", name.contents); + write!(out, ", {}", name.contents)?; } } // Current col is never zero again if names have been printed. // So we print a newline. if current_col > 0 { - let _ = writeln!(out,); + writeln!(out,)?; } } _ => { for name in names { - let _ = writeln!(out, "{}", name.contents); + writeln!(out, "{}", name.contents)?; } } - } + }; } + + Ok(()) } fn get_block_size(md: &Metadata, config: &Config) -> u64 { @@ -1769,7 +1774,6 @@ fn get_block_size(md: &Metadata, config: &Config) -> u64 { #[cfg(not(unix))] { - let _ = config; // no way to get block size for windows, fall-back to file size md.len() } @@ -1780,19 +1784,19 @@ fn display_grid( width: u16, direction: Direction, out: &mut BufWriter, -) { +) -> UResult<()> { if width == 0 { // If the width is 0 we print one single line let mut printed_something = false; for name in names { if printed_something { - let _ = write!(out, " "); + write!(out, " ")?; } printed_something = true; - let _ = write!(out, "{}", name.contents); + write!(out, "{}", name.contents)?; } if printed_something { - let _ = writeln!(out); + writeln!(out)?; } } else { let mut grid = Grid::new(GridOptions { @@ -1806,14 +1810,15 @@ fn display_grid( match grid.fit_into_width(width as usize) { Some(output) => { - let _ = write!(out, "{}", output); + write!(out, "{}", output)?; } // Width is too small for the grid, so we fit it in one column None => { - let _ = write!(out, "{}", grid.fit_into_columns(1)); + write!(out, "{}", grid.fit_into_columns(1))?; } } } + Ok(()) } /// This writes to the BufWriter out a single string of the output of `ls -l`. @@ -1849,20 +1854,20 @@ fn display_item_long( padding: &PaddingCollection, config: &Config, out: &mut BufWriter, -) { +) -> UResult<()> { if let Some(md) = item.md(out) { #[cfg(unix)] { if config.inode { - let _ = write!( + write!( out, "{} ", pad_left(&get_inode(md), padding.longest_inode_len), - ); + )?; } } - let _ = write!( + write!( out, "{}{} {}", display_permissions(md, true), @@ -1874,48 +1879,48 @@ fn display_item_long( "" }, pad_left(&display_symlink_count(md), padding.longest_link_count_len), - ); + )?; if config.long.owner { - let _ = write!( + write!( out, " {}", pad_right(&display_uname(md, config), padding.longest_uname_len), - ); + )?; } if config.long.group { - let _ = write!( + write!( out, " {}", pad_right(&display_group(md, config), padding.longest_group_len), - ); + )?; } if config.context { - let _ = write!( + write!( out, " {}", pad_right(&item.security_context, padding.longest_context_len), - ); + )?; } // Author is only different from owner on GNU/Hurd, so we reuse // the owner, since GNU/Hurd is not currently supported by Rust. if config.long.author { - let _ = write!( + write!( out, " {}", pad_right(&display_uname(md, config), padding.longest_uname_len), - ); + )?; } match display_size_or_rdev(md, config) { SizeOrDeviceId::Size(size) => { - let _ = write!(out, " {}", pad_left(&size, padding.longest_size_len),); + write!(out, " {}", pad_left(&size, padding.longest_size_len),)?; } SizeOrDeviceId::Device(major, minor) => { - let _ = write!( + write!( out, " {}, {}", pad_left( @@ -1936,19 +1941,19 @@ fn display_item_long( #[cfg(unix)] padding.longest_minor_len, ), - ); + )?; } }; let dfn = display_file_name(item, config, None, 0, out).contents; - let _ = writeln!(out, " {} {}", display_date(md, config), dfn); + writeln!(out, " {} {}", display_date(md, config), dfn)?; } else { // this 'else' is expressly for the case of a dangling symlink/restricted file #[cfg(unix)] { if config.inode { - let _ = write!(out, "{} ", pad_left("?", padding.longest_inode_len),); + write!(out, "{} ", pad_left("?", padding.longest_inode_len),)?; } } @@ -1985,7 +1990,7 @@ fn display_item_long( } }; - let _ = write!( + write!( out, "{}{} {}", format_args!("{}?????????", leading_char), @@ -1997,41 +2002,43 @@ fn display_item_long( "" }, pad_left("?", padding.longest_link_count_len), - ); + )?; if config.long.owner { - let _ = write!(out, " {}", pad_right("?", padding.longest_uname_len)); + write!(out, " {}", pad_right("?", padding.longest_uname_len))?; } if config.long.group { - let _ = write!(out, " {}", pad_right("?", padding.longest_group_len)); + write!(out, " {}", pad_right("?", padding.longest_group_len))?; } if config.context { - let _ = write!( + write!( out, " {}", pad_right(&item.security_context, padding.longest_context_len) - ); + )?; } // Author is only different from owner on GNU/Hurd, so we reuse // the owner, since GNU/Hurd is not currently supported by Rust. if config.long.author { - let _ = write!(out, " {}", pad_right("?", padding.longest_uname_len)); + write!(out, " {}", pad_right("?", padding.longest_uname_len))?; } let dfn = display_file_name(item, config, None, 0, out).contents; let date_len = 12; - let _ = writeln!( + writeln!( out, " {} {} {}", pad_left("?", padding.longest_size_len), pad_left("?", date_len), dfn, - ); + )?; } + + Ok(()) } #[cfg(unix)] From 519e82240a1bb7367bc76e69cd32b484913edcc3 Mon Sep 17 00:00:00 2001 From: Hanif Ariffin Date: Sat, 5 Feb 2022 23:32:44 +0800 Subject: [PATCH 05/78] Revert "Refactor padding calculations into a function" This reverts commit f39b861469c8d7598741864f2602c15a5496287b. Signed-off-by: Hanif Ariffin --- src/uu/ls/src/ls.rs | 185 +++++++++++++++++++++----------------------- 1 file changed, 88 insertions(+), 97 deletions(-) diff --git a/src/uu/ls/src/ls.rs b/src/uu/ls/src/ls.rs index f118a7594..aba59fe7e 100644 --- a/src/uu/ls/src/ls.rs +++ b/src/uu/ls/src/ls.rs @@ -1275,9 +1275,9 @@ only ignore '.' and '..'.", ) } -/// Represents a Path along with it's associated data. -/// Any data that will be reused several times makes sense to be added to this structure. -/// Caching data here helps eliminate redundant syscalls to fetch same information. +/// Represents a Path along with it's associated data +/// Any data that will be reused several times makes sense to be added to this structure +/// Caching data here helps eliminate redundant syscalls to fetch same information #[derive(Debug)] struct PathData { // Result got from symlink_metadata() or metadata() based on config @@ -1682,9 +1682,92 @@ fn display_items(items: &[PathData], config: &Config, out: &mut BufWriter, -) -> PaddingCollection { - let ( - mut longest_inode_len, - mut longest_link_count_len, - mut longest_uname_len, - mut longest_group_len, - mut longest_context_len, - mut longest_size_len, - mut longest_major_len, - mut longest_minor_len, - ) = (1, 1, 1, 1, 1, 1, 1, 1); - - for item in items { - let context_len = item.security_context.len(); - let (link_count_len, uname_len, group_len, size_len, major_len, minor_len, inode_len) = - display_dir_entry_size(item, config, out); - longest_inode_len = inode_len.max(longest_inode_len); - longest_link_count_len = link_count_len.max(longest_link_count_len); - longest_uname_len = uname_len.max(longest_uname_len); - longest_group_len = group_len.max(longest_group_len); - if config.context { - longest_context_len = context_len.max(longest_context_len); - } - if items.len() == 1usize { - longest_size_len = 0usize; - longest_major_len = 0usize; - longest_minor_len = 0usize; - } else { - longest_major_len = major_len.max(longest_major_len); - longest_minor_len = minor_len.max(longest_minor_len); - longest_size_len = size_len - .max(longest_size_len) - .max(longest_major_len + longest_minor_len + 2usize); - } - } - - PaddingCollection { - longest_inode_len, - longest_link_count_len, - longest_uname_len, - longest_group_len, - longest_context_len, - longest_size_len, - longest_major_len, - longest_minor_len, - } -} - -#[cfg(not(unix))] -fn calculate_paddings( - items: &[PathData], - config: &Config, - out: &mut BufWriter, -) -> PaddingCollection { - let ( - mut longest_link_count_len, - mut longest_uname_len, - mut longest_group_len, - mut longest_context_len, - mut longest_size_len, - ) = (1, 1, 1, 1, 1); - - for item in items { - let context_len = item.security_context.len(); - let (link_count_len, uname_len, group_len, size_len, _major_len, _minor_len, _inode_len) = - display_dir_entry_size(item, config, out); - longest_link_count_len = link_count_len.max(longest_link_count_len); - longest_uname_len = uname_len.max(longest_uname_len); - longest_group_len = group_len.max(longest_group_len); - if config.context { - longest_context_len = context_len.max(longest_context_len); - } - longest_size_len = size_len.max(longest_size_len); - } - - PaddingCollection { - longest_inode_len, - longest_link_count_len, - longest_uname_len, - longest_group_len, - longest_context_len, - longest_size_len, - longest_major_len, - longest_minor_len, - } -} From 78847e2ad098196b028f553641806a9c765daf36 Mon Sep 17 00:00:00 2001 From: Hanif Ariffin Date: Sat, 5 Feb 2022 23:40:23 +0800 Subject: [PATCH 06/78] Undo a small change that was meant to silence clippy Signed-off-by: Hanif Ariffin --- src/uu/ls/src/ls.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/uu/ls/src/ls.rs b/src/uu/ls/src/ls.rs index aba59fe7e..19acf36b5 100644 --- a/src/uu/ls/src/ls.rs +++ b/src/uu/ls/src/ls.rs @@ -1857,6 +1857,8 @@ fn get_block_size(md: &Metadata, config: &Config) -> u64 { #[cfg(not(unix))] { + // Silence linter warning about `config` being unused for windows. + let _ = config; // no way to get block size for windows, fall-back to file size md.len() } From 9ce9a4405280cb9272900c9c113aa93b9c6cad47 Mon Sep 17 00:00:00 2001 From: Hanif Bin Ariffin Date: Sun, 6 Feb 2022 10:26:01 +0800 Subject: [PATCH 07/78] Silencing clippy about unused variables Signed-off-by: Hanif Bin Ariffin --- src/uu/ls/src/ls.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/uu/ls/src/ls.rs b/src/uu/ls/src/ls.rs index aba59fe7e..d78813569 100644 --- a/src/uu/ls/src/ls.rs +++ b/src/uu/ls/src/ls.rs @@ -1857,6 +1857,7 @@ fn get_block_size(md: &Metadata, config: &Config) -> u64 { #[cfg(not(unix))] { + let _ = config; // no way to get block size for windows, fall-back to file size md.len() } From 66733ca99491dd7486300f1db1cb7fc2ba1e29da Mon Sep 17 00:00:00 2001 From: Andreas Molzer Date: Thu, 3 Feb 2022 23:44:43 +0100 Subject: [PATCH 08/78] seq: Add difficult cases to test suite --- tests/by-util/test_seq.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/by-util/test_seq.rs b/tests/by-util/test_seq.rs index 5adbc292e..ad3086b03 100644 --- a/tests/by-util/test_seq.rs +++ b/tests/by-util/test_seq.rs @@ -81,6 +81,33 @@ fn test_rejects_non_floats() { .usage_error("invalid floating point argument: 'foo'"); } +#[test] +fn test_accepts_option_argument_directly() { + new_ucmd!() + .arg("-s,") + .arg("2") + .succeeds() + .stdout_is("1,2\n"); +} + +#[test] +fn test_option_with_detected_negative_argument() { + new_ucmd!() + .arg("-s,") + .args(&["-1", "2"]) + .succeeds() + .stdout_is("-1,0,1,2\n"); +} + +#[test] +fn test_negative_number_as_separator() { + new_ucmd!() + .arg("-s") + .args(&["-1", "2"]) + .succeeds() + .stdout_is("1-12\n"); +} + #[test] fn test_invalid_float() { new_ucmd!() From a2e93299186be1911d1173cff8ea7f6163fa8f0f Mon Sep 17 00:00:00 2001 From: Andreas Molzer Date: Sun, 6 Feb 2022 03:46:31 +0100 Subject: [PATCH 09/78] seq: Allow option to receive immediate arguments WIP: this needs to be adjusted --- src/uu/seq/src/seq.rs | 2 +- tests/by-util/test_seq.rs | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/uu/seq/src/seq.rs b/src/uu/seq/src/seq.rs index af961a493..ec437dd40 100644 --- a/src/uu/seq/src/seq.rs +++ b/src/uu/seq/src/seq.rs @@ -146,7 +146,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { pub fn uu_app<'a>() -> App<'a> { App::new(uucore::util_name()) .setting(AppSettings::TrailingVarArg) - .setting(AppSettings::AllowHyphenValues) + .setting(AppSettings::AllowNegativeNumbers) .setting(AppSettings::InferLongArgs) .version(crate_version!()) .about(ABOUT) diff --git a/tests/by-util/test_seq.rs b/tests/by-util/test_seq.rs index ad3086b03..a18cbc2ad 100644 --- a/tests/by-util/test_seq.rs +++ b/tests/by-util/test_seq.rs @@ -20,14 +20,14 @@ fn test_hex_rejects_sign_after_identifier() { .args(&["-0x-123ABC"]) .fails() .no_stdout() - .stderr_contains("invalid floating point argument: '-0x-123ABC'") - .stderr_contains("for more information."); + .stderr_contains("which wasn't expected, or isn't valid in this context") + .stderr_contains("For more information try --help"); new_ucmd!() .args(&["-0x+123ABC"]) .fails() .no_stdout() - .stderr_contains("invalid floating point argument: '-0x+123ABC'") - .stderr_contains("for more information."); + .stderr_contains("which wasn't expected, or isn't valid in this context") + .stderr_contains("For more information try --help"); } #[test] From e77c8ff38154d378c2f2f7463977fc73b6485344 Mon Sep 17 00:00:00 2001 From: xxyzz Date: Sun, 13 Feb 2022 10:17:02 +0800 Subject: [PATCH 10/78] df: `-t` may appear more than once and doesn't support delimiter --- src/uu/df/src/df.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/uu/df/src/df.rs b/src/uu/df/src/df.rs index 90f1b0c9a..02cfca012 100644 --- a/src/uu/df/src/df.rs +++ b/src/uu/df/src/df.rs @@ -381,7 +381,7 @@ pub fn uu_app<'a>() -> App<'a> { .long("type") .allow_invalid_utf8(true) .takes_value(true) - .use_delimiter(true) + .multiple_occurrences(true) .help("limit listing to file systems of type TYPE"), ) .arg( From c849b8722f303b5e6c773ef5c61fb0490089d58f Mon Sep 17 00:00:00 2001 From: xxyzz Date: Sun, 13 Feb 2022 10:18:40 +0800 Subject: [PATCH 11/78] df: `--output` option conflicts with `-i`, `-P`, `-T` --- src/uu/df/src/df.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/uu/df/src/df.rs b/src/uu/df/src/df.rs index 02cfca012..be33238ff 100644 --- a/src/uu/df/src/df.rs +++ b/src/uu/df/src/df.rs @@ -338,6 +338,7 @@ pub fn uu_app<'a>() -> App<'a> { Arg::new(OPT_INODES) .short('i') .long("inodes") + .conflicts_with(OPT_OUTPUT) .help("list inode information instead of block usage"), ) .arg(Arg::new(OPT_KILO).short('k').help("like --block-size=1K")) @@ -367,6 +368,7 @@ pub fn uu_app<'a>() -> App<'a> { Arg::new(OPT_PORTABILITY) .short('P') .long("portability") + .conflicts_with(OPT_OUTPUT) .help("use the POSIX output format"), ) .arg( @@ -388,6 +390,7 @@ pub fn uu_app<'a>() -> App<'a> { Arg::new(OPT_PRINT_TYPE) .short('T') .long("print-type") + .conflicts_with(OPT_OUTPUT) .help("print file system type"), ) .arg( From 18b11cb2cf9d9f5115f9d56e44e8ffbb4255642d Mon Sep 17 00:00:00 2001 From: xxyzz Date: Thu, 3 Feb 2022 18:03:48 +0800 Subject: [PATCH 12/78] Create coverage report for GNU tests --- .github/workflows/GnuTests.yml | 76 ++++++++++++++++++++++++++++++++++ DEVELOPER_INSTRUCTIONS.md | 4 +- 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml index e57204213..4d8e1db19 100644 --- a/.github/workflows/GnuTests.yml +++ b/.github/workflows/GnuTests.yml @@ -182,3 +182,79 @@ jobs: else echo "::warning ::Skipping test summary comparison; no prior reference summary is available." fi + + gnu_coverage: + name: Run GNU tests with coverage + runs-on: ubuntu-latest + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} + steps: + - name: Checkout code uutil + uses: actions/checkout@v2 + with: + path: 'uutils' + - name: Checkout GNU coreutils + uses: actions/checkout@v2 + with: + repository: 'coreutils/coreutils' + path: 'gnu' + ref: 'v9.0' + submodules: recursive + - name: Install `rust` toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: nightly + default: true + profile: minimal # minimal component installation (ie, no documentation) + components: rustfmt + - name: Install dependencies + run: | + sudo apt update + sudo apt install autoconf autopoint bison texinfo gperf gcc g++ gdb python-pyinotify jq valgrind libexpect-perl -y + - name: Add various locales + run: | + echo "Before:" + locale -a + ## Some tests fail with 'cannot change locale (en_US.ISO-8859-1): No such file or directory' + ## Some others need a French locale + sudo locale-gen + sudo locale-gen fr_FR + sudo locale-gen fr_FR.UTF-8 + sudo update-locale + echo "After:" + locale -a + - name: Build binaries + env: + CARGO_INCREMENTAL: "0" + RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Copt-level=0 -Clink-dead-code -Coverflow-checks=off -Zpanic_abort_tests -Cpanic=abort" + RUSTDOCFLAGS: "-Cpanic=abort" + run: | + cd uutils + UU_MAKE_PROFILE=debug bash util/build-gnu.sh + - name: Run GNU tests + run: bash uutils/util/run-gnu-test.sh + - name: "`grcov` ~ install" + uses: actions-rs/install@v0.1 + with: + crate: grcov + version: latest + use-tool-cache: false + - name: Generate coverage data (via `grcov`) + id: coverage + run: | + ## Generate coverage data + cd uutils + COVERAGE_REPORT_DIR="target/debug" + COVERAGE_REPORT_FILE="${COVERAGE_REPORT_DIR}/lcov.info" + mkdir -p "${COVERAGE_REPORT_DIR}" + # display coverage files + grcov . --output-type files --ignore build.rs --ignore "vendor/*" --ignore "/*" --ignore "[a-zA-Z]:/*" --excl-br-line "^\s*((debug_)?assert(_eq|_ne)?!|#\[derive\()" | sort --unique + # generate coverage report + grcov . --output-type lcov --output-path "${COVERAGE_REPORT_FILE}" --branch --ignore build.rs --ignore "vendor/*" --ignore "/*" --ignore "[a-zA-Z]:/*" --excl-br-line "^\s*((debug_)?assert(_eq|_ne)?!|#\[derive\()" + echo ::set-output name=report::${COVERAGE_REPORT_FILE} + - name: Upload coverage results (to Codecov.io) + uses: codecov/codecov-action@v2 + with: + file: ${{ steps.coverage.outputs.report }} + flags: gnutests + name: gnutests + working-directory: uutils diff --git a/DEVELOPER_INSTRUCTIONS.md b/DEVELOPER_INSTRUCTIONS.md index 027d4dca1..c007fba7e 100644 --- a/DEVELOPER_INSTRUCTIONS.md +++ b/DEVELOPER_INSTRUCTIONS.md @@ -21,7 +21,7 @@ Running GNU tests At the end you should have uutils, gnu and gnulib checked out next to each other. - Run `cd uutils && ./util/build-gnu.sh && cd ..` to get everything ready (this may take a while) -- Finally, you can run `tests with bash uutils/util/run-gnu-test.sh `. Instead of `` insert the test you want to run, e.g. `tests/misc/wc-proc`. +- Finally, you can run tests with `bash uutils/util/run-gnu-test.sh `. Instead of `` insert the test you want to run, e.g. `tests/misc/wc-proc.sh`. Code Coverage Report Generation @@ -33,7 +33,7 @@ Code coverage report can be generated using [grcov](https://github.com/mozilla/g ### Using Nightly Rust -To generate [gcov-based](https://github.com/mozilla/grcov#example-how-to-generate-gcda-files-for-cc) coverage report +To generate [gcov-based](https://github.com/mozilla/grcov#example-how-to-generate-gcda-files-for-a-rust-project) coverage report ```bash $ export CARGO_INCREMENTAL=0 From 1dbd474339b90bb072d1a78be32379c00cd36339 Mon Sep 17 00:00:00 2001 From: xxyzz Date: Fri, 4 Feb 2022 18:34:09 +0800 Subject: [PATCH 13/78] There are four GNU tests require valgrind --- .github/workflows/GnuTests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml index 4d8e1db19..1990bfbd3 100644 --- a/.github/workflows/GnuTests.yml +++ b/.github/workflows/GnuTests.yml @@ -66,7 +66,7 @@ jobs: run: | ## Install dependencies sudo apt-get update - sudo apt-get install autoconf autopoint bison texinfo gperf gcc g++ gdb python-pyinotify jq + sudo apt-get install autoconf autopoint bison texinfo gperf gcc g++ gdb python-pyinotify jq valgrind - name: Add various locales shell: bash run: | From ce02eae14bfdff5c0f85b2b6f80e9d0dc77eaedf Mon Sep 17 00:00:00 2001 From: xxyzz Date: Fri, 4 Feb 2022 19:17:30 +0800 Subject: [PATCH 14/78] tests/misc/tty-eof.pl requires Perl's Expect package >=1.11 --- .github/workflows/GnuTests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml index 1990bfbd3..79e0878ac 100644 --- a/.github/workflows/GnuTests.yml +++ b/.github/workflows/GnuTests.yml @@ -66,7 +66,7 @@ jobs: run: | ## Install dependencies sudo apt-get update - sudo apt-get install autoconf autopoint bison texinfo gperf gcc g++ gdb python-pyinotify jq valgrind + sudo apt-get install autoconf autopoint bison texinfo gperf gcc g++ gdb python-pyinotify jq valgrind libexpect-perl - name: Add various locales shell: bash run: | From 1ccf94e4fa8dd334a33e4174043acbeb57730802 Mon Sep 17 00:00:00 2001 From: xxyzz Date: Mon, 14 Feb 2022 14:04:07 +0800 Subject: [PATCH 15/78] Run 33 GNU tests that require root --- util/run-gnu-test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/util/run-gnu-test.sh b/util/run-gnu-test.sh index 4ff266833..360807013 100755 --- a/util/run-gnu-test.sh +++ b/util/run-gnu-test.sh @@ -31,6 +31,8 @@ export RUST_BACKTRACE=1 if test -n "$1"; then # if set, run only the test passed export RUN_TEST="TESTS=$1" +elif test -n "$CI"; then + sudo make -j "$(nproc)" check-root SUBDIRS=. RUN_EXPENSIVE_TESTS=yes RUN_VERY_EXPENSIVE_TESTS=yes VERBOSE=no gl_public_submodule_commit="" srcdir="${path_GNU}" TEST_SUITE_LOG="tests/test-suite-root.log" || : fi # * timeout used to kill occasionally errant/"stuck" processes (note: 'release' testing takes ~1 hour; 'debug' testing takes ~2.5 hours) From aa4c5aea50bac22393f4ba1d4eee8b7800a74fe9 Mon Sep 17 00:00:00 2001 From: Jeffrey Finkelstein Date: Sun, 30 Jan 2022 21:35:43 -0500 Subject: [PATCH 16/78] split: refactor to add SuffixType enum Refactor the code to use a `SuffixType` enumeration with two members, `Alphabetic` and `NumericDecimal`, representing the two currently supported ways of producing filename suffixes. This prepares the code to more easily support other formats, like numeric hexadecimal. --- src/uu/split/src/filenames.rs | 62 ++++++++++++++++++++++++----------- src/uu/split/src/split.rs | 20 ++++++++--- 2 files changed, 58 insertions(+), 24 deletions(-) diff --git a/src/uu/split/src/filenames.rs b/src/uu/split/src/filenames.rs index 3e2db3606..1b89190c7 100644 --- a/src/uu/split/src/filenames.rs +++ b/src/uu/split/src/filenames.rs @@ -13,12 +13,13 @@ //! //! ```rust,ignore //! use crate::filenames::FilenameIterator; +//! use crate::filenames::SuffixType; //! //! let prefix = "chunk_".to_string(); //! let suffix = ".txt".to_string(); //! let width = 2; -//! let use_numeric_suffix = false; -//! let it = FilenameIterator::new(prefix, suffix, width, use_numeric_suffix); +//! let suffix_type = SuffixType::Alphabetic; +//! let it = FilenameIterator::new(prefix, suffix, width, suffix_type); //! //! assert_eq!(it.next().unwrap(), "chunk_aa.txt"); //! assert_eq!(it.next().unwrap(), "chunk_ab.txt"); @@ -28,6 +29,26 @@ use crate::number::DynamicWidthNumber; use crate::number::FixedWidthNumber; use crate::number::Number; +/// The format to use for suffixes in the filename for each output chunk. +#[derive(Clone, Copy)] +pub enum SuffixType { + /// Lowercase ASCII alphabetic characters. + Alphabetic, + + /// Decimal numbers. + NumericDecimal, +} + +impl SuffixType { + /// The radix to use when representing the suffix string as digits. + fn radix(&self) -> u8 { + match self { + SuffixType::Alphabetic => 26, + SuffixType::NumericDecimal => 10, + } + } +} + /// Compute filenames from a given index. /// /// This iterator yields filenames for use with ``split``. @@ -42,8 +63,8 @@ use crate::number::Number; /// width in characters. In that case, after the iterator yields each /// string of that width, the iterator is exhausted. /// -/// Finally, if `use_numeric_suffix` is `true`, then numbers will be -/// used instead of lowercase ASCII alphabetic characters. +/// Finally, `suffix_type` controls which type of suffix to produce, +/// alphabetic or numeric. /// /// # Examples /// @@ -52,28 +73,30 @@ use crate::number::Number; /// /// ```rust,ignore /// use crate::filenames::FilenameIterator; +/// use crate::filenames::SuffixType; /// /// let prefix = "chunk_".to_string(); /// let suffix = ".txt".to_string(); /// let width = 2; -/// let use_numeric_suffix = false; -/// let it = FilenameIterator::new(prefix, suffix, width, use_numeric_suffix); +/// let suffix_type = SuffixType::Alphabetic; +/// let it = FilenameIterator::new(prefix, suffix, width, suffix_type); /// /// assert_eq!(it.next().unwrap(), "chunk_aa.txt"); /// assert_eq!(it.next().unwrap(), "chunk_ab.txt"); /// assert_eq!(it.next().unwrap(), "chunk_ac.txt"); /// ``` /// -/// For numeric filenames, set `use_numeric_suffix` to `true`: +/// For numeric filenames, use `SuffixType::NumericDecimal`: /// /// ```rust,ignore /// use crate::filenames::FilenameIterator; +/// use crate::filenames::SuffixType; /// /// let prefix = "chunk_".to_string(); /// let suffix = ".txt".to_string(); /// let width = 2; -/// let use_numeric_suffix = true; -/// let it = FilenameIterator::new(prefix, suffix, width, use_numeric_suffix); +/// let suffix_type = SuffixType::NumericDecimal; +/// let it = FilenameIterator::new(prefix, suffix, width, suffix_type); /// /// assert_eq!(it.next().unwrap(), "chunk_00.txt"); /// assert_eq!(it.next().unwrap(), "chunk_01.txt"); @@ -91,9 +114,9 @@ impl<'a> FilenameIterator<'a> { prefix: &'a str, additional_suffix: &'a str, suffix_length: usize, - use_numeric_suffix: bool, + suffix_type: SuffixType, ) -> FilenameIterator<'a> { - let radix = if use_numeric_suffix { 10 } else { 26 }; + let radix = suffix_type.radix(); let number = if suffix_length == 0 { Number::DynamicWidth(DynamicWidthNumber::new(radix)) } else { @@ -130,39 +153,40 @@ impl<'a> Iterator for FilenameIterator<'a> { mod tests { use crate::filenames::FilenameIterator; + use crate::filenames::SuffixType; #[test] fn test_filename_iterator_alphabetic_fixed_width() { - let mut it = FilenameIterator::new("chunk_", ".txt", 2, false); + let mut it = FilenameIterator::new("chunk_", ".txt", 2, SuffixType::Alphabetic); assert_eq!(it.next().unwrap(), "chunk_aa.txt"); assert_eq!(it.next().unwrap(), "chunk_ab.txt"); assert_eq!(it.next().unwrap(), "chunk_ac.txt"); - let mut it = FilenameIterator::new("chunk_", ".txt", 2, false); + let mut it = FilenameIterator::new("chunk_", ".txt", 2, SuffixType::Alphabetic); assert_eq!(it.nth(26 * 26 - 1).unwrap(), "chunk_zz.txt"); assert_eq!(it.next(), None); } #[test] fn test_filename_iterator_numeric_fixed_width() { - let mut it = FilenameIterator::new("chunk_", ".txt", 2, true); + let mut it = FilenameIterator::new("chunk_", ".txt", 2, SuffixType::NumericDecimal); assert_eq!(it.next().unwrap(), "chunk_00.txt"); assert_eq!(it.next().unwrap(), "chunk_01.txt"); assert_eq!(it.next().unwrap(), "chunk_02.txt"); - let mut it = FilenameIterator::new("chunk_", ".txt", 2, true); + let mut it = FilenameIterator::new("chunk_", ".txt", 2, SuffixType::NumericDecimal); assert_eq!(it.nth(10 * 10 - 1).unwrap(), "chunk_99.txt"); assert_eq!(it.next(), None); } #[test] fn test_filename_iterator_alphabetic_dynamic_width() { - let mut it = FilenameIterator::new("chunk_", ".txt", 0, false); + let mut it = FilenameIterator::new("chunk_", ".txt", 0, SuffixType::Alphabetic); assert_eq!(it.next().unwrap(), "chunk_aa.txt"); assert_eq!(it.next().unwrap(), "chunk_ab.txt"); assert_eq!(it.next().unwrap(), "chunk_ac.txt"); - let mut it = FilenameIterator::new("chunk_", ".txt", 0, false); + let mut it = FilenameIterator::new("chunk_", ".txt", 0, SuffixType::Alphabetic); assert_eq!(it.nth(26 * 25 - 1).unwrap(), "chunk_yz.txt"); assert_eq!(it.next().unwrap(), "chunk_zaaa.txt"); assert_eq!(it.next().unwrap(), "chunk_zaab.txt"); @@ -170,12 +194,12 @@ mod tests { #[test] fn test_filename_iterator_numeric_dynamic_width() { - let mut it = FilenameIterator::new("chunk_", ".txt", 0, true); + let mut it = FilenameIterator::new("chunk_", ".txt", 0, SuffixType::NumericDecimal); assert_eq!(it.next().unwrap(), "chunk_00.txt"); assert_eq!(it.next().unwrap(), "chunk_01.txt"); assert_eq!(it.next().unwrap(), "chunk_02.txt"); - let mut it = FilenameIterator::new("chunk_", ".txt", 0, true); + let mut it = FilenameIterator::new("chunk_", ".txt", 0, SuffixType::NumericDecimal); assert_eq!(it.nth(10 * 9 - 1).unwrap(), "chunk_89.txt"); assert_eq!(it.next().unwrap(), "chunk_9000.txt"); assert_eq!(it.next().unwrap(), "chunk_9001.txt"); diff --git a/src/uu/split/src/split.rs b/src/uu/split/src/split.rs index 29559f8b8..dbb537c96 100644 --- a/src/uu/split/src/split.rs +++ b/src/uu/split/src/split.rs @@ -12,6 +12,7 @@ mod number; mod platform; use crate::filenames::FilenameIterator; +use crate::filenames::SuffixType; use clap::{crate_version, App, AppSettings, Arg, ArgMatches}; use std::env; use std::fmt; @@ -250,13 +251,22 @@ impl Strategy { } } +/// Parse the suffix type from the command-line arguments. +fn suffix_type_from(matches: &ArgMatches) -> SuffixType { + if matches.occurrences_of(OPT_NUMERIC_SUFFIXES) > 0 { + SuffixType::NumericDecimal + } else { + SuffixType::Alphabetic + } +} + /// Parameters that control how a file gets split. /// /// You can convert an [`ArgMatches`] instance into a [`Settings`] /// instance by calling [`Settings::from`]. struct Settings { prefix: String, - numeric_suffix: bool, + suffix_type: SuffixType, suffix_length: usize, additional_suffix: String, input: String, @@ -324,7 +334,7 @@ impl Settings { suffix_length: suffix_length_str .parse() .map_err(|_| SettingsError::SuffixLength(suffix_length_str.to_string()))?, - numeric_suffix: matches.occurrences_of(OPT_NUMERIC_SUFFIXES) > 0, + suffix_type: suffix_type_from(matches), additional_suffix, verbose: matches.occurrences_of("verbose") > 0, strategy: Strategy::from(matches).map_err(SettingsError::Strategy)?, @@ -384,7 +394,7 @@ impl<'a> ByteChunkWriter<'a> { &settings.prefix, &settings.additional_suffix, settings.suffix_length, - settings.numeric_suffix, + settings.suffix_type, ); let filename = filename_iterator.next()?; if settings.verbose { @@ -512,7 +522,7 @@ impl<'a> LineChunkWriter<'a> { &settings.prefix, &settings.additional_suffix, settings.suffix_length, - settings.numeric_suffix, + settings.suffix_type, ); let filename = filename_iterator.next()?; if settings.verbose { @@ -604,7 +614,7 @@ where &settings.prefix, &settings.additional_suffix, settings.suffix_length, - settings.numeric_suffix, + settings.suffix_type, ); // Create one writer for each chunk. This will create each From 891c5d1ffaa7643eeed3e4ef095fe62c2a95d35d Mon Sep 17 00:00:00 2001 From: Jeffrey Finkelstein Date: Sun, 16 Jan 2022 10:36:26 -0500 Subject: [PATCH 17/78] split: add SuffixType::NumericHexadecimal Add a `NumericHexadecimal` member to the `SuffixType` enum so that a future commit can add support for hexadecimal filename suffixes to the `split` program. --- src/uu/split/src/filenames.rs | 6 +- src/uu/split/src/number.rs | 119 +++++++++++++++++++++++++++------- 2 files changed, 101 insertions(+), 24 deletions(-) diff --git a/src/uu/split/src/filenames.rs b/src/uu/split/src/filenames.rs index 1b89190c7..0121ba87f 100644 --- a/src/uu/split/src/filenames.rs +++ b/src/uu/split/src/filenames.rs @@ -37,6 +37,9 @@ pub enum SuffixType { /// Decimal numbers. NumericDecimal, + + /// Hexadecimal numbers. + NumericHexadecimal, } impl SuffixType { @@ -45,6 +48,7 @@ impl SuffixType { match self { SuffixType::Alphabetic => 26, SuffixType::NumericDecimal => 10, + SuffixType::NumericHexadecimal => 16, } } } @@ -86,7 +90,7 @@ impl SuffixType { /// assert_eq!(it.next().unwrap(), "chunk_ac.txt"); /// ``` /// -/// For numeric filenames, use `SuffixType::NumericDecimal`: +/// For decimal numeric filenames, use `SuffixType::NumericDecimal`: /// /// ```rust,ignore /// use crate::filenames::FilenameIterator; diff --git a/src/uu/split/src/number.rs b/src/uu/split/src/number.rs index ef3ccbc4b..d5427e2ca 100644 --- a/src/uu/split/src/number.rs +++ b/src/uu/split/src/number.rs @@ -40,13 +40,19 @@ impl Error for Overflow {} /// specifically for the `split` program. See the /// [`DynamicWidthNumber`] documentation for more information. /// -/// Numbers of radix 10 are displayable and rendered as decimal -/// numbers (for example, "00" or "917"). Numbers of radix 26 are -/// displayable and rendered as lowercase ASCII alphabetic characters -/// (for example, "aa" or "zax"). Numbers of other radices cannot be -/// displayed. The display of a [`DynamicWidthNumber`] includes a -/// prefix whose length depends on the width of the number. See the -/// [`DynamicWidthNumber`] documentation for more information. +/// Numbers of radix +/// +/// * 10 are displayable and rendered as decimal numbers (for example, +/// "00" or "917"), +/// * 16 are displayable and rendered as hexadecimal numbers (for example, +/// "00" or "e7f"), +/// * 26 are displayable and rendered as lowercase ASCII alphabetic +/// characters (for example, "aa" or "zax"). +/// +/// Numbers of other radices cannot be displayed. The display of a +/// [`DynamicWidthNumber`] includes a prefix whose length depends on +/// the width of the number. See the [`DynamicWidthNumber`] +/// documentation for more information. /// /// The digits of a number are accessible via the [`Number::digits`] /// method. The digits are represented as a [`Vec`] with the most @@ -169,12 +175,12 @@ impl Display for Number { /// /// # Displaying /// -/// This number is only displayable if `radix` is 10 or `radix` is -/// 26. If `radix` is 10, then the digits are concatenated and -/// displayed as a fixed-width decimal number. If `radix` is 26, then -/// each digit is translated to the corresponding lowercase ASCII -/// alphabetic character (that is, 'a', 'b', 'c', etc.) and -/// concatenated. +/// This number is only displayable if `radix` is 10, 26, or 26. If +/// `radix` is 10 or 16, then the digits are concatenated and +/// displayed as a fixed-width decimal or hexadecimal number, +/// respectively. If `radix` is 26, then each digit is translated to +/// the corresponding lowercase ASCII alphabetic character (that is, +/// 'a', 'b', 'c', etc.) and concatenated. #[derive(Clone)] pub struct FixedWidthNumber { radix: u8, @@ -228,6 +234,14 @@ impl Display for FixedWidthNumber { let digits: String = self.digits.iter().map(|d| (b'0' + d) as char).collect(); write!(f, "{}", digits) } + 16 => { + let digits: String = self + .digits + .iter() + .map(|d| (if *d < 10 { b'0' + d } else { b'a' + (d - 10) }) as char) + .collect(); + write!(f, "{}", digits) + } 26 => { let digits: String = self.digits.iter().map(|d| (b'a' + d) as char).collect(); write!(f, "{}", digits) @@ -264,14 +278,15 @@ impl Display for FixedWidthNumber { /// /// # Displaying /// -/// This number is only displayable if `radix` is 10 or `radix` is -/// 26. If `radix` is 10, then the digits are concatenated and -/// displayed as a fixed-width decimal number with a prefix of `n - 2` -/// instances of the character '9', where `n` is the number of digits. -/// If `radix` is 26, then each digit is translated to the -/// corresponding lowercase ASCII alphabetic character (that is, 'a', -/// 'b', 'c', etc.) and concatenated with a prefix of `n - 2` -/// instances of the character 'z'. +/// This number is only displayable if `radix` is 10, 16, or 26. If +/// `radix` is 10 or 16, then the digits are concatenated and +/// displayed as a fixed-width decimal or hexadecimal number, +/// respectively, with a prefix of `n - 2` instances of the character +/// '9' of 'f', respectively, where `n` is the number of digits. If +/// `radix` is 26, then each digit is translated to the corresponding +/// lowercase ASCII alphabetic character (that is, 'a', 'b', 'c', +/// etc.) and concatenated with a prefix of `n - 2` instances of the +/// character 'z'. /// /// This notion of displaying the number is specific to the `split` /// program. @@ -349,6 +364,21 @@ impl Display for DynamicWidthNumber { digits = digits, ) } + 16 => { + let num_fill_chars = self.digits.len() - 2; + let digits: String = self + .digits + .iter() + .map(|d| (if *d < 10 { b'0' + d } else { b'a' + (d - 10) }) as char) + .collect(); + write!( + f, + "{empty:f { let num_fill_chars = self.digits.len() - 2; let digits: String = self.digits.iter().map(|d| (b'a' + d) as char).collect(); @@ -424,7 +454,7 @@ mod tests { } #[test] - fn test_dynamic_width_number_display_numeric() { + fn test_dynamic_width_number_display_numeric_decimal() { fn num(n: usize) -> Number { let mut number = Number::DynamicWidth(DynamicWidthNumber::new(10)); for _ in 0..n { @@ -444,6 +474,30 @@ mod tests { assert_eq!(format!("{}", num(10 * 99 + 1)), "990001"); } + #[test] + fn test_dynamic_width_number_display_numeric_hexadecimal() { + fn num(n: usize) -> Number { + let mut number = Number::DynamicWidth(DynamicWidthNumber::new(16)); + for _ in 0..n { + number.increment().unwrap() + } + number + } + + assert_eq!(format!("{}", num(0)), "00"); + assert_eq!(format!("{}", num(15)), "0f"); + assert_eq!(format!("{}", num(16)), "10"); + assert_eq!(format!("{}", num(17)), "11"); + assert_eq!(format!("{}", num(18)), "12"); + + assert_eq!(format!("{}", num(16 * 15 - 1)), "ef"); + assert_eq!(format!("{}", num(16 * 15)), "f000"); + assert_eq!(format!("{}", num(16 * 15 + 1)), "f001"); + assert_eq!(format!("{}", num(16 * 255 - 1)), "feff"); + assert_eq!(format!("{}", num(16 * 255)), "ff0000"); + assert_eq!(format!("{}", num(16 * 255 + 1)), "ff0001"); + } + #[test] fn test_fixed_width_number_increment() { let mut n = Number::FixedWidth(FixedWidthNumber::new(3, 2)); @@ -493,7 +547,7 @@ mod tests { } #[test] - fn test_fixed_width_number_display_numeric() { + fn test_fixed_width_number_display_numeric_decimal() { fn num(n: usize) -> Result { let mut number = Number::FixedWidth(FixedWidthNumber::new(10, 2)); for _ in 0..n { @@ -510,4 +564,23 @@ mod tests { assert_eq!(format!("{}", num(10 * 10 - 1).unwrap()), "99"); assert!(num(10 * 10).is_err()); } + + #[test] + fn test_fixed_width_number_display_numeric_hexadecimal() { + fn num(n: usize) -> Result { + let mut number = Number::FixedWidth(FixedWidthNumber::new(16, 2)); + for _ in 0..n { + number.increment()?; + } + Ok(number) + } + + assert_eq!(format!("{}", num(0).unwrap()), "00"); + assert_eq!(format!("{}", num(15).unwrap()), "0f"); + assert_eq!(format!("{}", num(17).unwrap()), "11"); + assert_eq!(format!("{}", num(16 * 15 - 1).unwrap()), "ef"); + assert_eq!(format!("{}", num(16 * 15).unwrap()), "f0"); + assert_eq!(format!("{}", num(16 * 16 - 1).unwrap()), "ff"); + assert!(num(16 * 16).is_err()); + } } From 4470430c894579c452e44fb933f883e2487c84cc Mon Sep 17 00:00:00 2001 From: Jeffrey Finkelstein Date: Sun, 16 Jan 2022 10:41:52 -0500 Subject: [PATCH 18/78] split: add support for -x option (hex suffixes) Add support for the `-x` command-line option to `split`. This option causes `split` to produce filenames with hexadecimal suffixes instead of the default alphabetic suffixes. --- src/uu/split/src/split.rs | 11 +++++++++ tests/by-util/test_split.rs | 24 ++++++++++++++++++- .../split/twohundredfortyonebytes.txt | 1 + 3 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/split/twohundredfortyonebytes.txt diff --git a/src/uu/split/src/split.rs b/src/uu/split/src/split.rs index dbb537c96..70de45ce2 100644 --- a/src/uu/split/src/split.rs +++ b/src/uu/split/src/split.rs @@ -32,6 +32,7 @@ static OPT_ADDITIONAL_SUFFIX: &str = "additional-suffix"; static OPT_FILTER: &str = "filter"; static OPT_NUMBER: &str = "number"; static OPT_NUMERIC_SUFFIXES: &str = "numeric-suffixes"; +static OPT_HEX_SUFFIXES: &str = "hex-suffixes"; static OPT_SUFFIX_LENGTH: &str = "suffix-length"; static OPT_DEFAULT_SUFFIX_LENGTH: &str = "0"; static OPT_VERBOSE: &str = "verbose"; @@ -143,6 +144,14 @@ pub fn uu_app<'a>() -> App<'a> { .default_value(OPT_DEFAULT_SUFFIX_LENGTH) .help("use suffixes of length N (default 2)"), ) + .arg( + Arg::new(OPT_HEX_SUFFIXES) + .short('x') + .long(OPT_HEX_SUFFIXES) + .takes_value(true) + .default_missing_value("0") + .help("use hex suffixes starting at 0, not alphabetic"), + ) .arg( Arg::new(OPT_VERBOSE) .long(OPT_VERBOSE) @@ -255,6 +264,8 @@ impl Strategy { fn suffix_type_from(matches: &ArgMatches) -> SuffixType { if matches.occurrences_of(OPT_NUMERIC_SUFFIXES) > 0 { SuffixType::NumericDecimal + } else if matches.occurrences_of(OPT_HEX_SUFFIXES) > 0 { + SuffixType::NumericHexadecimal } else { SuffixType::Alphabetic } diff --git a/tests/by-util/test_split.rs b/tests/by-util/test_split.rs index b5d799d72..846d483b2 100644 --- a/tests/by-util/test_split.rs +++ b/tests/by-util/test_split.rs @@ -2,7 +2,7 @@ // * // * For the full copyright and license information, please view the LICENSE // * file that was distributed with this source code. -// spell-checker:ignore xzaaa sixhundredfiftyonebytes ninetyonebytes asciilowercase fghij klmno pqrst uvwxyz fivelines +// spell-checker:ignore xzaaa sixhundredfiftyonebytes ninetyonebytes asciilowercase fghij klmno pqrst uvwxyz fivelines twohundredfortyonebytes extern crate rand; extern crate regex; @@ -409,6 +409,28 @@ fn test_numeric_dynamic_suffix_length() { assert_eq!(file_read(&at, "x9000"), "a"); } +#[test] +fn test_hex_dynamic_suffix_length() { + let (at, mut ucmd) = at_and_ucmd!(); + // Split into chunks of one byte each, use hexadecimal digits + // instead of letters as file suffixes. + // + // The input file has (16^2) - 16 + 1 = 241 bytes. This is just + // enough to force `split` to dynamically increase the length of + // the filename for the very last chunk. + // + // x00, x01, x02, ..., xed, xee, xef, xf000 + // + ucmd.args(&["-x", "-b", "1", "twohundredfortyonebytes.txt"]) + .succeeds(); + for i in 0..240 { + let filename = format!("x{:02x}", i); + let contents = file_read(&at, &filename); + assert_eq!(contents, "a"); + } + assert_eq!(file_read(&at, "xf000"), "a"); +} + #[test] fn test_suffixes_exhausted() { new_ucmd!() diff --git a/tests/fixtures/split/twohundredfortyonebytes.txt b/tests/fixtures/split/twohundredfortyonebytes.txt new file mode 100644 index 000000000..10a53a61c --- /dev/null +++ b/tests/fixtures/split/twohundredfortyonebytes.txt @@ -0,0 +1 @@ +aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \ No newline at end of file From 6c3fc7b214d974397548bbbc6f7fd7dea960b09a Mon Sep 17 00:00:00 2001 From: ndd7xv Date: Tue, 15 Feb 2022 01:35:28 -0500 Subject: [PATCH 19/78] split: throw error when # chunks > # filenames from suffix length --- src/uu/split/src/filenames.rs | 2 +- src/uu/split/src/split.rs | 28 +++++++++++++++++++++++----- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/uu/split/src/filenames.rs b/src/uu/split/src/filenames.rs index 0121ba87f..426c76226 100644 --- a/src/uu/split/src/filenames.rs +++ b/src/uu/split/src/filenames.rs @@ -44,7 +44,7 @@ pub enum SuffixType { impl SuffixType { /// The radix to use when representing the suffix string as digits. - fn radix(&self) -> u8 { + pub fn radix(&self) -> u8 { match self { SuffixType::Alphabetic => 26, SuffixType::NumericDecimal => 10, diff --git a/src/uu/split/src/split.rs b/src/uu/split/src/split.rs index 70de45ce2..a7a49af00 100644 --- a/src/uu/split/src/split.rs +++ b/src/uu/split/src/split.rs @@ -293,11 +293,14 @@ enum SettingsError { Strategy(StrategyError), /// Invalid suffix length parameter. - SuffixLength(String), + SuffixNotParsable(String), /// Suffix contains a directory separator, which is not allowed. SuffixContainsSeparator(String), + /// Suffix is not large enough to split into specified chunks + SuffixTooSmall(usize), + /// The `--filter` option is not supported on Windows. #[cfg(windows)] NotSupported, @@ -317,7 +320,8 @@ impl fmt::Display for SettingsError { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { Self::Strategy(e) => e.fmt(f), - Self::SuffixLength(s) => write!(f, "invalid suffix length: {}", s.quote()), + Self::SuffixNotParsable(s) => write!(f, "invalid suffix length: {}", s.quote()), + Self::SuffixTooSmall(i) => write!(f, "the suffix length needs to be at least {}", i), Self::SuffixContainsSeparator(s) => write!( f, "invalid suffix {}, contains directory separator", @@ -340,15 +344,29 @@ impl Settings { if additional_suffix.contains('/') { return Err(SettingsError::SuffixContainsSeparator(additional_suffix)); } + let strategy = Strategy::from(matches).map_err(SettingsError::Strategy)?; + let suffix_type = suffix_type_from(matches); let suffix_length_str = matches.value_of(OPT_SUFFIX_LENGTH).unwrap(); + let suffix_length: usize = suffix_length_str + .parse() + .map_err(|_| SettingsError::SuffixNotParsable(suffix_length_str.to_string()))?; + if let Strategy::Number(chunks) = strategy { + if suffix_length != 0 { + let required_suffix_length = + (chunks as f64).log(suffix_type.radix() as f64).ceil() as usize; + if suffix_length < required_suffix_length { + return Err(SettingsError::SuffixTooSmall(required_suffix_length)); + } + } + } let result = Self { suffix_length: suffix_length_str .parse() - .map_err(|_| SettingsError::SuffixLength(suffix_length_str.to_string()))?, - suffix_type: suffix_type_from(matches), + .map_err(|_| SettingsError::SuffixNotParsable(suffix_length_str.to_string()))?, + suffix_type, additional_suffix, verbose: matches.occurrences_of("verbose") > 0, - strategy: Strategy::from(matches).map_err(SettingsError::Strategy)?, + strategy, input: matches.value_of(ARG_INPUT).unwrap().to_owned(), prefix: matches.value_of(ARG_PREFIX).unwrap().to_owned(), filter: matches.value_of(OPT_FILTER).map(|s| s.to_owned()), From c16c06ea0d15c46082db9205d5d461f937e778f1 Mon Sep 17 00:00:00 2001 From: xxyzz Date: Thu, 17 Feb 2022 13:43:59 +0800 Subject: [PATCH 20/78] df: add output option's valid field names --- src/uu/df/src/df.rs | 12 +++++++++--- tests/by-util/test_df.rs | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/src/uu/df/src/df.rs b/src/uu/df/src/df.rs index be33238ff..cb7a84e20 100644 --- a/src/uu/df/src/df.rs +++ b/src/uu/df/src/df.rs @@ -5,6 +5,7 @@ // // For the full copyright and license information, please view the LICENSE file // that was distributed with this source code. +// spell-checker:ignore itotal iused iavail ipcent pcent mod table; use uucore::error::UResult; @@ -46,6 +47,10 @@ static OPT_SYNC: &str = "sync"; static OPT_TYPE: &str = "type"; static OPT_PRINT_TYPE: &str = "print-type"; static OPT_EXCLUDE_TYPE: &str = "exclude-type"; +static OUTPUT_FIELD_LIST: [&str; 12] = [ + "source", "fstype", "itotal", "iused", "iavail", "ipcent", "size", "used", "avail", "pcent", + "file", "target", +]; /// Store names of file systems as a selector. /// Note: `exclude` takes priority over `include`. @@ -338,7 +343,6 @@ pub fn uu_app<'a>() -> App<'a> { Arg::new(OPT_INODES) .short('i') .long("inodes") - .conflicts_with(OPT_OUTPUT) .help("list inode information instead of block usage"), ) .arg(Arg::new(OPT_KILO).short('k').help("like --block-size=1K")) @@ -359,6 +363,10 @@ pub fn uu_app<'a>() -> App<'a> { .long("output") .takes_value(true) .use_delimiter(true) + .possible_values(OUTPUT_FIELD_LIST) + .default_missing_values(&OUTPUT_FIELD_LIST) + .default_values(&["source", "size", "used", "avail", "pcent", "target"]) + .conflicts_with_all(&[OPT_INODES, OPT_PORTABILITY, OPT_PRINT_TYPE]) .help( "use the output format defined by FIELD_LIST,\ or print all fields if FIELD_LIST is omitted.", @@ -368,7 +376,6 @@ pub fn uu_app<'a>() -> App<'a> { Arg::new(OPT_PORTABILITY) .short('P') .long("portability") - .conflicts_with(OPT_OUTPUT) .help("use the POSIX output format"), ) .arg( @@ -390,7 +397,6 @@ pub fn uu_app<'a>() -> App<'a> { Arg::new(OPT_PRINT_TYPE) .short('T') .long("print-type") - .conflicts_with(OPT_OUTPUT) .help("print file system type"), ) .arg( diff --git a/tests/by-util/test_df.rs b/tests/by-util/test_df.rs index 00f1ecf3a..3af02428e 100644 --- a/tests/by-util/test_df.rs +++ b/tests/by-util/test_df.rs @@ -58,4 +58,23 @@ fn test_order_same() { assert_eq!(output1, output2); } +#[test] +fn test_output_conflict_options() { + for option in ["-i", "-T", "-P"] { + new_ucmd!().arg("--output=source").arg(option).fails(); + } +} + +#[test] +fn test_output_option() { + new_ucmd!().arg("--output").succeeds(); + new_ucmd!().arg("--output=source,target").succeeds(); + new_ucmd!().arg("--output=invalid_option").fails(); +} + +#[test] +fn test_type_option() { + new_ucmd!().args(&["-t", "ext4", "-t", "ext3"]).succeeds(); +} + // ToDO: more tests... From 494d709e0f05bdd18d5782a6d1fce565397c844f Mon Sep 17 00:00:00 2001 From: ndd7xv Date: Wed, 16 Feb 2022 23:48:12 -0500 Subject: [PATCH 21/78] split: small tweaks to wording changes `SuffixType` enums to have better names and hex suffix help to be consistent with numeric suffix help --- src/uu/split/src/filenames.rs | 20 ++++++++++---------- src/uu/split/src/split.rs | 6 +++--- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/uu/split/src/filenames.rs b/src/uu/split/src/filenames.rs index 426c76226..31742194d 100644 --- a/src/uu/split/src/filenames.rs +++ b/src/uu/split/src/filenames.rs @@ -36,10 +36,10 @@ pub enum SuffixType { Alphabetic, /// Decimal numbers. - NumericDecimal, + Decimal, /// Hexadecimal numbers. - NumericHexadecimal, + Hexadecimal, } impl SuffixType { @@ -47,8 +47,8 @@ impl SuffixType { pub fn radix(&self) -> u8 { match self { SuffixType::Alphabetic => 26, - SuffixType::NumericDecimal => 10, - SuffixType::NumericHexadecimal => 16, + SuffixType::Decimal => 10, + SuffixType::Hexadecimal => 16, } } } @@ -90,7 +90,7 @@ impl SuffixType { /// assert_eq!(it.next().unwrap(), "chunk_ac.txt"); /// ``` /// -/// For decimal numeric filenames, use `SuffixType::NumericDecimal`: +/// For decimal numeric filenames, use `SuffixType::Decimal`: /// /// ```rust,ignore /// use crate::filenames::FilenameIterator; @@ -99,7 +99,7 @@ impl SuffixType { /// let prefix = "chunk_".to_string(); /// let suffix = ".txt".to_string(); /// let width = 2; -/// let suffix_type = SuffixType::NumericDecimal; +/// let suffix_type = SuffixType::Decimal; /// let it = FilenameIterator::new(prefix, suffix, width, suffix_type); /// /// assert_eq!(it.next().unwrap(), "chunk_00.txt"); @@ -173,12 +173,12 @@ mod tests { #[test] fn test_filename_iterator_numeric_fixed_width() { - let mut it = FilenameIterator::new("chunk_", ".txt", 2, SuffixType::NumericDecimal); + let mut it = FilenameIterator::new("chunk_", ".txt", 2, SuffixType::Decimal); assert_eq!(it.next().unwrap(), "chunk_00.txt"); assert_eq!(it.next().unwrap(), "chunk_01.txt"); assert_eq!(it.next().unwrap(), "chunk_02.txt"); - let mut it = FilenameIterator::new("chunk_", ".txt", 2, SuffixType::NumericDecimal); + let mut it = FilenameIterator::new("chunk_", ".txt", 2, SuffixType::Decimal); assert_eq!(it.nth(10 * 10 - 1).unwrap(), "chunk_99.txt"); assert_eq!(it.next(), None); } @@ -198,12 +198,12 @@ mod tests { #[test] fn test_filename_iterator_numeric_dynamic_width() { - let mut it = FilenameIterator::new("chunk_", ".txt", 0, SuffixType::NumericDecimal); + let mut it = FilenameIterator::new("chunk_", ".txt", 0, SuffixType::Decimal); assert_eq!(it.next().unwrap(), "chunk_00.txt"); assert_eq!(it.next().unwrap(), "chunk_01.txt"); assert_eq!(it.next().unwrap(), "chunk_02.txt"); - let mut it = FilenameIterator::new("chunk_", ".txt", 0, SuffixType::NumericDecimal); + let mut it = FilenameIterator::new("chunk_", ".txt", 0, SuffixType::Decimal); assert_eq!(it.nth(10 * 9 - 1).unwrap(), "chunk_89.txt"); assert_eq!(it.next().unwrap(), "chunk_9000.txt"); assert_eq!(it.next().unwrap(), "chunk_9001.txt"); diff --git a/src/uu/split/src/split.rs b/src/uu/split/src/split.rs index a7a49af00..cde0f8e55 100644 --- a/src/uu/split/src/split.rs +++ b/src/uu/split/src/split.rs @@ -150,7 +150,7 @@ pub fn uu_app<'a>() -> App<'a> { .long(OPT_HEX_SUFFIXES) .takes_value(true) .default_missing_value("0") - .help("use hex suffixes starting at 0, not alphabetic"), + .help("use hex suffixes instead of alphabetic"), ) .arg( Arg::new(OPT_VERBOSE) @@ -263,9 +263,9 @@ impl Strategy { /// Parse the suffix type from the command-line arguments. fn suffix_type_from(matches: &ArgMatches) -> SuffixType { if matches.occurrences_of(OPT_NUMERIC_SUFFIXES) > 0 { - SuffixType::NumericDecimal + SuffixType::Decimal } else if matches.occurrences_of(OPT_HEX_SUFFIXES) > 0 { - SuffixType::NumericHexadecimal + SuffixType::Hexadecimal } else { SuffixType::Alphabetic } From f57e3470ae452e23157a375aaf58c9692617a696 Mon Sep 17 00:00:00 2001 From: Terts Diepraam Date: Thu, 17 Feb 2022 18:29:05 +0100 Subject: [PATCH 22/78] docs: add examples from tldr-pages --- docs/.gitignore | 1 + docs/Makefile | 4 +++- src/bin/uudoc.rs | 50 +++++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/docs/.gitignore b/docs/.gitignore index f2b5c7168..c836306f5 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -1,3 +1,4 @@ book src/utils src/SUMMARY.md +tldr/ diff --git a/docs/Makefile b/docs/Makefile index dd700bcb0..23901b755 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,4 +1,6 @@ +# spell-checker:ignore tldr clean: - rm -rf _build + rm -rf book rm -f src/SUMMARY.md rm -f src/utils/* + rm -rf tldr diff --git a/src/bin/uudoc.rs b/src/bin/uudoc.rs index 71bbb2684..5658f491c 100644 --- a/src/bin/uudoc.rs +++ b/src/bin/uudoc.rs @@ -2,15 +2,33 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. +// spell-checker:ignore tldr use clap::App; use std::ffi::OsString; use std::fs::File; -use std::io::{self, Write}; +use std::io::{self, Read, Write}; +use std::process::Command; include!(concat!(env!("OUT_DIR"), "/uutils_map.rs")); fn main() -> io::Result<()> { + let _ = std::fs::create_dir("docs/tldr"); + println!("Downloading tldr archive"); + Command::new("curl") + .arg("https://tldr.sh/assets/tldr.zip") + .arg("--output") + .arg("docs/tldr/tldr.zip") + .output()?; + + println!("Unzipping tldr archive"); + Command::new("unzip") + .arg("-o") + .arg("docs/tldr/tldr.zip") + .arg("-d") + .arg("docs/tldr") + .output()?; + let utils = util_map::>>(); match std::fs::create_dir("docs/src/utils/") { Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => Ok(()), @@ -55,6 +73,7 @@ fn write_markdown(mut w: impl Write, app: &mut App, name: &str) -> io::Result<() write_version(&mut w, app)?; write_usage(&mut w, app, name)?; write_description(&mut w, app)?; + write_examples(&mut w, name)?; write_options(&mut w, app) } @@ -82,6 +101,35 @@ fn write_description(w: &mut impl Write, app: &App) -> io::Result<()> { } } +fn write_examples(w: &mut impl Write, name: &str) -> io::Result<()> { + if let Ok(mut file) = std::fs::File::open(format!("docs/tldr/pages/common/{}.md", name)) + .or_else(|_| std::fs::File::open(format!("docs/tldr/pages/linux/{}.md", name))) + { + let mut content = String::new(); + file.read_to_string(&mut content)?; + + writeln!(w, "## Examples")?; + writeln!(w)?; + for line in content.lines().skip_while(|l| !l.starts_with('-')) { + if let Some(l) = line.strip_prefix("- ") { + writeln!(w, "{}", l)?; + } else if line.starts_with('`') { + writeln!(w, "```shell\n{}\n```", line.trim_matches('`'))?; + } else if line.is_empty() { + writeln!(w)?; + } else { + println!("Not sure what to do with this line:"); + println!("{}", line); + } + } + writeln!(w)?; + writeln!(w, "> The examples are provided by the [tldr-pages project](https://tldr.sh) under the [CC BY 4.0 License](https://github.com/tldr-pages/tldr/blob/main/LICENSE.md).")?; + } else { + println!("No examples found for: {}", name); + } + Ok(()) +} + fn write_options(w: &mut impl Write, app: &App) -> io::Result<()> { writeln!(w, "

Options

")?; write!(w, "
")?; From 0af2c9bafbdc7f9ebf799bed230da832d5bb5cd4 Mon Sep 17 00:00:00 2001 From: Roy Ivy III Date: Sun, 13 Feb 2022 21:46:45 -0600 Subject: [PATCH 23/78] maint/CICD ~ (GnuTests) display sub-step test comparison failures more prominently --- .github/workflows/GnuTests.yml | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml index eef8567c7..e901d4cfe 100644 --- a/.github/workflows/GnuTests.yml +++ b/.github/workflows/GnuTests.yml @@ -160,34 +160,38 @@ jobs: - name: Compare test failures VS reference shell: bash run: | + have_new_failures="" REF_LOG_FILE='${{ steps.vars.outputs.path_reference }}/test-logs/test-suite.log' REF_SUMMARY_FILE='${{ steps.vars.outputs.path_reference }}/test-summary/gnu-result.json' if test -f "${REF_LOG_FILE}"; then - echo "Reference SHA1/ID (of '${REF_SUMMARY_FILE}'): $(sha1sum -- "${REF_SUMMARY_FILE}")" + echo "Reference SHA1/ID: $(sha1sum -- "${REF_SUMMARY_FILE}")" REF_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" "${REF_LOG_FILE}" | sort) NEW_FAILING=$(sed -n "s/^FAIL: \([[:print:]]\+\).*/\1/p" '${{ steps.vars.outputs.path_GNU_tests }}/test-suite.log' | sort) - for LINE in $REF_FAILING + for LINE in ${REF_FAILING} do - if ! grep -Fxq $LINE<<<"$NEW_FAILING"; then - echo "::warning ::Congrats! The gnu test $LINE is now passing!" + if ! grep -Fxq ${LINE}<<<"${NEW_FAILING}"; then + echo "::warning ::Congrats! The gnu test ${LINE} is now passing!" fi done - for LINE in $NEW_FAILING + for LINE in ${NEW_FAILING} do - if ! grep -Fxq $LINE<<<"$REF_FAILING" + if ! grep -Fxq ${LINE}<<<"${REF_FAILING}" then - echo "::error ::GNU test failed: $LINE. $LINE is passing on '${{ steps.vars.outputs.repo_default_branch }}'. Maybe you have to rebase?" + echo "::error ::GNU test failed: ${LINE}. ${LINE} is passing on '${{ steps.vars.outputs.repo_default_branch }}'. Maybe you have to rebase?" + have_new_failures="true" fi done else echo "::warning ::Skipping test failure comparison; no prior reference test logs are available." fi + if test -n "${have_new_failures}" ; then exit -1 ; fi - name: Compare test summary VS reference + if: success() || failure() # run regardless of prior step success/failure shell: bash run: | REF_SUMMARY_FILE='${{ steps.vars.outputs.path_reference }}/test-summary/gnu-result.json' if test -f "${REF_SUMMARY_FILE}"; then - echo "Reference SHA1/ID (of '${REF_SUMMARY_FILE}'): $(sha1sum -- "${REF_SUMMARY_FILE}")" + echo "Reference SHA1/ID: $(sha1sum -- "${REF_SUMMARY_FILE}")" mv "${REF_SUMMARY_FILE}" main-gnu-result.json python uutils/util/compare_gnu_result.py else From d6424bb354dced58cf314377db00ce2e6a847e99 Mon Sep 17 00:00:00 2001 From: Roy Ivy III Date: Wed, 16 Feb 2022 10:16:42 -0600 Subject: [PATCH 24/78] maint/util ~ remove extra/redundant factor tests for 'debug' builds - avoids "Terminated" timeout errors when using longer running 'debug' `factor` --- util/build-gnu.sh | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/util/build-gnu.sh b/util/build-gnu.sh index 2ab23ddc7..bb8d6e522 100755 --- a/util/build-gnu.sh +++ b/util/build-gnu.sh @@ -70,19 +70,38 @@ sed -i 's|^"\$@|/usr/bin/timeout 600 "\$@|' build-aux/test-driver sed -i "s/^[[:blank:]]*PATH=.*/ PATH='${UU_BUILD_DIR//\//\\/}\$(PATH_SEPARATOR)'\"\$\$PATH\" \\\/" Makefile sed -i 's| tr | /usr/bin/tr |' tests/init.sh make -j "$(nproc)" -# Generate the factor tests, so they can be fixed -# Used to be 36. Reduced to 20 to decrease the log size -for i in {00..20}; do - make "tests/factor/t${i}.sh" -done - -# strip the long stuff -for i in {21..36}; do +first=00 +if test ${UU_MAKE_PROFILE} != "debug"; then + # Generate the factor tests, so they can be fixed + # * reduced to 20 to decrease log size (down from 36 expected by GNU) + # * only for 'release', skipped for 'debug' as redundant and too time consuming (causing timeout errors) + seq=$( + i=${first} + while test "$i" -le 20; do + printf '%02d ' $i + i=$(($i + 1)) + done + ) + for i in ${seq}; do + make "tests/factor/t${i}.sh" + done + sed -i -e 's|^seq |/usr/bin/seq |' -e 's|sha1sum |/usr/bin/sha1sum |' tests/factor/t*sh + first=21 +fi +# strip all (debug) or just the longer (release) factor tests from Makefile +seq=$( + i=${first} + while test "$i" -le 36; do + printf '%02d ' $i + i=$(($i + 1)) + done +) +for i in ${seq}; do + echo "strip t${i}.sh from Makefile" sed -i -e "s/\$(tf)\/t${i}.sh//g" Makefile done grep -rl 'path_prepend_' tests/* | xargs sed -i 's| path_prepend_ ./src||' -sed -i -e 's|^seq |/usr/bin/seq |' -e 's|sha1sum |/usr/bin/sha1sum |' tests/factor/t*sh # Remove tests checking for --version & --help # Not really interesting for us and logs are too big From 6718d97f97e092b6d6a0ec5edb72497ba1b85be1 Mon Sep 17 00:00:00 2001 From: Jeffrey Finkelstein Date: Wed, 9 Feb 2022 21:41:33 -0500 Subject: [PATCH 25/78] split: add support for -e argument Add the `-e` flag, which indicates whether to elide (that is, remove) empty files that would have been created by the `-n` option. The `-n` command-line argument gives a specific number of chunks into which the input files will be split. If the number of chunks is greater than the number of bytes, then empty files will be created for the excess chunks. But if `-e` is given, then empty files will not be created. For example, contrast $ printf 'a\n' > f && split -e -n 3 f && cat xaa xab xac a cat: xac: No such file or directory with $ printf 'a\n' > f && split -n 3 f && cat xaa xab xac a --- src/uu/split/src/split.rs | 36 ++++++++++++++++++++++++++++- tests/by-util/test_split.rs | 28 +++++++++++++++++++++- tests/fixtures/split/threebytes.txt | 1 + 3 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 tests/fixtures/split/threebytes.txt diff --git a/src/uu/split/src/split.rs b/src/uu/split/src/split.rs index 70de45ce2..7ff08d37d 100644 --- a/src/uu/split/src/split.rs +++ b/src/uu/split/src/split.rs @@ -39,6 +39,7 @@ static OPT_VERBOSE: &str = "verbose"; //The ---io-blksize parameter is consumed and ignored. //The parameter is included to make GNU coreutils tests pass. static OPT_IO_BLKSIZE: &str = "-io-blksize"; +static OPT_ELIDE_EMPTY_FILES: &str = "elide-empty-files"; static ARG_INPUT: &str = "input"; static ARG_PREFIX: &str = "prefix"; @@ -128,6 +129,13 @@ pub fn uu_app<'a>() -> App<'a> { "write to shell COMMAND file name is $FILE (Currently not implemented for Windows)", ), ) + .arg( + Arg::new(OPT_ELIDE_EMPTY_FILES) + .long(OPT_ELIDE_EMPTY_FILES) + .short('e') + .takes_value(false) + .help("do not generate empty output files with '-n'"), + ) .arg( Arg::new(OPT_NUMERIC_SUFFIXES) .short('d') @@ -285,6 +293,16 @@ struct Settings { filter: Option, strategy: Strategy, verbose: bool, + + /// Whether to *not* produce empty files when using `-n`. + /// + /// The `-n` command-line argument gives a specific number of + /// chunks into which the input files will be split. If the number + /// of chunks is greater than the number of bytes, and this is + /// `false`, then empty files will be created for the excess + /// chunks. If this is `false`, then empty files will not be + /// created. + elide_empty_files: bool, } /// An error when parsing settings from command-line arguments. @@ -352,6 +370,7 @@ impl Settings { input: matches.value_of(ARG_INPUT).unwrap().to_owned(), prefix: matches.value_of(ARG_PREFIX).unwrap().to_owned(), filter: matches.value_of(OPT_FILTER).map(|s| s.to_owned()), + elide_empty_files: matches.is_present(OPT_ELIDE_EMPTY_FILES), }; #[cfg(windows)] if result.filter.is_some() { @@ -616,9 +635,24 @@ where { // Get the size of the input file in bytes and compute the number // of bytes per chunk. + // + // If the requested number of chunks exceeds the number of bytes + // in the file *and* the `elide_empty_files` parameter is enabled, + // then behave as if the number of chunks was set to the number of + // bytes in the file. This ensures that we don't write empty + // files. Otherwise, just write the `num_chunks - num_bytes` empty + // files. let metadata = metadata(&settings.input).unwrap(); let num_bytes = metadata.len(); - let chunk_size = (num_bytes / (num_chunks as u64)) as usize; + let will_have_empty_files = settings.elide_empty_files && num_chunks as u64 > num_bytes; + let (num_chunks, chunk_size) = if will_have_empty_files { + let num_chunks = num_bytes as usize; + let chunk_size = 1; + (num_chunks, chunk_size) + } else { + let chunk_size = ((num_bytes / (num_chunks as u64)) as usize).max(1); + (num_chunks, chunk_size) + }; // This object is responsible for creating the filename for each chunk. let mut filename_iterator = FilenameIterator::new( diff --git a/tests/by-util/test_split.rs b/tests/by-util/test_split.rs index 846d483b2..9454687ac 100644 --- a/tests/by-util/test_split.rs +++ b/tests/by-util/test_split.rs @@ -2,7 +2,7 @@ // * // * For the full copyright and license information, please view the LICENSE // * file that was distributed with this source code. -// spell-checker:ignore xzaaa sixhundredfiftyonebytes ninetyonebytes asciilowercase fghij klmno pqrst uvwxyz fivelines twohundredfortyonebytes +// spell-checker:ignore xzaaa sixhundredfiftyonebytes ninetyonebytes threebytes asciilowercase fghij klmno pqrst uvwxyz fivelines twohundredfortyonebytes extern crate rand; extern crate regex; @@ -526,3 +526,29 @@ fn test_include_newlines() { at.open("xac").read_to_string(&mut s).unwrap(); assert_eq!(s, "5\n"); } + +#[test] +fn test_allow_empty_files() { + let (at, mut ucmd) = at_and_ucmd!(); + ucmd.args(&["-n", "4", "threebytes.txt"]) + .succeeds() + .no_stdout() + .no_stderr(); + assert_eq!(at.read("xaa"), "a"); + assert_eq!(at.read("xab"), "b"); + assert_eq!(at.read("xac"), "c"); + assert_eq!(at.read("xad"), ""); +} + +#[test] +fn test_elide_empty_files() { + let (at, mut ucmd) = at_and_ucmd!(); + ucmd.args(&["-e", "-n", "4", "threebytes.txt"]) + .succeeds() + .no_stdout() + .no_stderr(); + assert_eq!(at.read("xaa"), "a"); + assert_eq!(at.read("xab"), "b"); + assert_eq!(at.read("xac"), "c"); + assert!(!at.plus("xad").exists()); +} diff --git a/tests/fixtures/split/threebytes.txt b/tests/fixtures/split/threebytes.txt new file mode 100644 index 000000000..f2ba8f84a --- /dev/null +++ b/tests/fixtures/split/threebytes.txt @@ -0,0 +1 @@ +abc \ No newline at end of file From 89f428b44f704b0fb330a6477a9a71836e751090 Mon Sep 17 00:00:00 2001 From: Jeffrey Finkelstein Date: Sat, 12 Feb 2022 22:34:39 -0500 Subject: [PATCH 26/78] dd: remove spurious zero multiplier warning Fix a bug in which `dd` was inappropriately showing a warning about a "0x" multiplier when there was no "x" character in the argument. --- src/uu/dd/src/parseargs.rs | 34 +++++++++++++++++++++------------- tests/by-util/test_dd.rs | 16 ++++++++++++++++ 2 files changed, 37 insertions(+), 13 deletions(-) diff --git a/src/uu/dd/src/parseargs.rs b/src/uu/dd/src/parseargs.rs index 1425cae01..028deab74 100644 --- a/src/uu/dd/src/parseargs.rs +++ b/src/uu/dd/src/parseargs.rs @@ -325,6 +325,14 @@ impl std::str::FromStr for StatusLevel { } } +fn show_zero_multiplier_warning() { + show_warning!( + "{} is a zero multiplier; use {} if that is intended", + "0x".quote(), + "00x".quote() + ); +} + /// Parse bytes using str::parse, then map error if needed. fn parse_bytes_only(s: &str) -> Result { s.parse() @@ -357,13 +365,6 @@ fn parse_bytes_only(s: &str) -> Result { /// assert_eq!(parse_bytes_no_x("2k").unwrap(), 2 * 1024); /// ``` fn parse_bytes_no_x(s: &str) -> Result { - if s == "0" { - show_warning!( - "{} is a zero multiplier; use {} if that is intended", - "0x".quote(), - "00x".quote() - ); - } let (num, multiplier) = match (s.find('c'), s.rfind('w'), s.rfind('b')) { (None, None, None) => match uucore::parse_size::parse_size(s) { Ok(n) => (n, 1), @@ -401,13 +402,20 @@ fn parse_bytes_with_opt_multiplier(s: &str) -> Result { // Split on the 'x' characters. Each component will be parsed // individually, then multiplied together. - let mut total = 1; - for part in s.split('x') { - let num = parse_bytes_no_x(part).map_err(|e| e.with_arg(s.to_string()))?; - total *= num; + let parts: Vec<&str> = s.split('x').collect(); + if parts.len() == 1 { + parse_bytes_no_x(parts[0]).map_err(|e| e.with_arg(s.to_string())) + } else { + let mut total = 1; + for part in parts { + if part == "0" { + show_zero_multiplier_warning(); + } + let num = parse_bytes_no_x(part).map_err(|e| e.with_arg(s.to_string()))?; + total *= num; + } + Ok(total) } - - Ok(total) } pub fn parse_ibs(matches: &Matches) -> Result { diff --git a/tests/by-util/test_dd.rs b/tests/by-util/test_dd.rs index 58c577a7d..22daec60c 100644 --- a/tests/by-util/test_dd.rs +++ b/tests/by-util/test_dd.rs @@ -201,6 +201,13 @@ fn test_x_multiplier() { #[test] fn test_zero_multiplier_warning() { for arg in ["count", "seek", "skip"] { + new_ucmd!() + .args(&[format!("{}=0", arg).as_str(), "status=none"]) + .pipe_in("") + .succeeds() + .no_stdout() + .no_stderr(); + new_ucmd!() .args(&[format!("{}=00x1", arg).as_str(), "status=none"]) .pipe_in("") @@ -1063,3 +1070,12 @@ fn test_all_valid_ascii_ebcdic_ascii_roundtrip_conv_test() { .succeeds() .stdout_is_fixture_bytes("all-valid-ascii-chars-37eff01866ba3f538421b30b7cbefcac.test"); } + +#[test] +fn test_skip_zero() { + new_ucmd!() + .args(&["skip=0", "status=noxfer"]) + .succeeds() + .no_stdout() + .stderr_is("0+0 records in\n0+0 records out\n"); +} From 1e12c46c246d58115a33b5f1279251ea866df9a8 Mon Sep 17 00:00:00 2001 From: ndd7xv Date: Thu, 10 Feb 2022 19:24:16 -0500 Subject: [PATCH 27/78] uucore(memo): replace err_conv with result --- src/uu/printf/src/printf.rs | 2 +- src/uu/seq/src/seq.rs | 12 ++- src/uucore/src/lib/features/memo.rs | 15 +-- src/uucore/src/lib/features/tokenize/sub.rs | 94 +++++++++++++------ src/uucore/src/lib/features/tokenize/token.rs | 4 +- .../lib/features/tokenize/unescaped_text.rs | 6 +- 6 files changed, 90 insertions(+), 43 deletions(-) diff --git a/src/uu/printf/src/printf.rs b/src/uu/printf/src/printf.rs index a30d18c53..5732d4ced 100644 --- a/src/uu/printf/src/printf.rs +++ b/src/uu/printf/src/printf.rs @@ -284,7 +284,7 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { None => vec![], }; - memo::Memo::run_all(format_string, &values[..]); + memo::Memo::run_all(format_string, &values[..])?; Ok(()) } diff --git a/src/uu/seq/src/seq.rs b/src/uu/seq/src/seq.rs index 3646effc1..28524c55c 100644 --- a/src/uu/seq/src/seq.rs +++ b/src/uu/seq/src/seq.rs @@ -5,6 +5,7 @@ // TODO: Support -f flag // spell-checker:ignore (ToDO) istr chiter argptr ilen extendedbigdecimal extendedbigint numberparse use std::io::{stdout, ErrorKind, Write}; +use std::process::exit; use clap::{crate_version, App, AppSettings, Arg}; use num_traits::Zero; @@ -12,6 +13,7 @@ use num_traits::Zero; use uucore::error::FromIo; use uucore::error::UResult; use uucore::memo::Memo; +use uucore::show; mod error; mod extendedbigdecimal; @@ -287,7 +289,10 @@ fn print_seq( match format { Some(f) => { let s = format!("{}", value); - Memo::run_all(f, &[s]); + if let Err(x) = Memo::run_all(f, &[s]) { + show!(x); + exit(1); + } } None => write_value_float( &mut stdout, @@ -349,7 +354,10 @@ fn print_seq_integers( match format { Some(f) => { let s = format!("{}", value); - Memo::run_all(f, &[s]); + if let Err(x) = Memo::run_all(f, &[s]) { + show!(x); + exit(1); + } } None => write_value_int(&mut stdout, &value, padding, pad, is_first_iteration)?, } diff --git a/src/uucore/src/lib/features/memo.rs b/src/uucore/src/lib/features/memo.rs index fd57c33b5..0e3a6376c 100644 --- a/src/uucore/src/lib/features/memo.rs +++ b/src/uucore/src/lib/features/memo.rs @@ -6,6 +6,7 @@ //! that prints tokens. use crate::display::Quotable; +use crate::error::UResult; use crate::features::tokenize::sub::Sub; use crate::features::tokenize::token::{Token, Tokenizer}; use crate::features::tokenize::unescaped_text::UnescapedText; @@ -26,17 +27,17 @@ fn warn_excess_args(first_arg: &str) { } impl Memo { - pub fn new(pf_string: &str, pf_args_it: &mut Peekable>) -> Self { + pub fn new(pf_string: &str, pf_args_it: &mut Peekable>) -> UResult { let mut pm = Self { tokens: Vec::new() }; let mut tmp_token: Option>; let mut it = put_back_n(pf_string.chars()); let mut has_sub = false; loop { - tmp_token = UnescapedText::from_it(&mut it, pf_args_it); + tmp_token = UnescapedText::from_it(&mut it, pf_args_it)?; if let Some(x) = tmp_token { pm.tokens.push(x); } - tmp_token = Sub::from_it(&mut it, pf_args_it); + tmp_token = Sub::from_it(&mut it, pf_args_it)?; if let Some(x) = tmp_token { if !has_sub { has_sub = true; @@ -64,19 +65,19 @@ impl Memo { } } } - pm + Ok(pm) } pub fn apply(&self, pf_args_it: &mut Peekable>) { for tkn in &self.tokens { tkn.print(pf_args_it); } } - pub fn run_all(pf_string: &str, pf_args: &[String]) { + pub fn run_all(pf_string: &str, pf_args: &[String]) -> UResult<()> { let mut arg_it = pf_args.iter().peekable(); - let pm = Self::new(pf_string, &mut arg_it); + let pm = Self::new(pf_string, &mut arg_it)?; loop { if arg_it.peek().is_none() { - break; + return Ok(()); } pm.apply(&mut arg_it); } diff --git a/src/uucore/src/lib/features/tokenize/sub.rs b/src/uucore/src/lib/features/tokenize/sub.rs index ac471ae3e..a1c2f3807 100644 --- a/src/uucore/src/lib/features/tokenize/sub.rs +++ b/src/uucore/src/lib/features/tokenize/sub.rs @@ -5,7 +5,7 @@ //! it is created by Sub's implementation of the Tokenizer trait //! Subs which have numeric field chars make use of the num_format //! submodule -use crate::show_error; +use crate::error::{UResult, UUsageError}; use itertools::{put_back_n, PutBackN}; use std::iter::Peekable; use std::process::exit; @@ -20,11 +20,6 @@ use super::unescaped_text::UnescapedText; const EXIT_ERR: i32 = 1; -fn err_conv(sofar: &str) { - show_error!("%{}: invalid conversion specification", sofar); - exit(EXIT_ERR); -} - fn convert_asterisk_arg_int(asterisk_arg: &str) -> isize { // this is a costly way to parse the // args used for asterisk values into integers @@ -116,14 +111,14 @@ impl SubParser { fn from_it( it: &mut PutBackN, args: &mut Peekable>, - ) -> Option> { + ) -> UResult>> { let mut parser = Self::new(); - if parser.sub_vals_retrieved(it) { + if parser.sub_vals_retrieved(it)? { let t: Box = Self::build_token(parser); t.print(args); - Some(t) + Ok(Some(t)) } else { - None + Ok(None) } } fn build_token(parser: Self) -> Box { @@ -151,9 +146,9 @@ impl SubParser { )); t } - fn sub_vals_retrieved(&mut self, it: &mut PutBackN) -> bool { - if !Self::successfully_eat_prefix(it, &mut self.text_so_far) { - return false; + fn sub_vals_retrieved(&mut self, it: &mut PutBackN) -> UResult { + if !Self::successfully_eat_prefix(it, &mut self.text_so_far)? { + return Ok(false); } // this fn in particular is much longer than it needs to be // .could get a lot @@ -177,7 +172,10 @@ impl SubParser { '-' | '*' | '0'..='9' => { if !self.past_decimal { if self.min_width_is_asterisk || self.specifiers_found { - err_conv(&self.text_so_far); + return Err(UUsageError::new( + 1, + format!("%{}: invalid conversion specification", &self.text_so_far), + )); } if self.min_width_tmp.is_none() { self.min_width_tmp = Some(String::new()); @@ -185,7 +183,13 @@ impl SubParser { match self.min_width_tmp.as_mut() { Some(x) => { if (ch == '-' || ch == '*') && !x.is_empty() { - err_conv(&self.text_so_far); + return Err(UUsageError::new( + 1, + format!( + "%{}: invalid conversion specification", + &self.text_so_far + ), + )); } if ch == '*' { self.min_width_is_asterisk = true; @@ -200,7 +204,10 @@ impl SubParser { // second field should never have a // negative value if self.second_field_is_asterisk || ch == '-' || self.specifiers_found { - err_conv(&self.text_so_far); + return Err(UUsageError::new( + 1, + format!("%{}: invalid conversion specification", &self.text_so_far), + )); } if self.second_field_tmp.is_none() { self.second_field_tmp = Some(String::new()); @@ -208,7 +215,13 @@ impl SubParser { match self.second_field_tmp.as_mut() { Some(x) => { if ch == '*' && !x.is_empty() { - err_conv(&self.text_so_far); + return Err(UUsageError::new( + 1, + format!( + "%{}: invalid conversion specification", + &self.text_so_far + ), + )); } if ch == '*' { self.second_field_is_asterisk = true; @@ -225,7 +238,10 @@ impl SubParser { if !self.past_decimal { self.past_decimal = true; } else { - err_conv(&self.text_so_far); + return Err(UUsageError::new( + 1, + format!("%{}: invalid conversion specification", &self.text_so_far), + )); } } x if legal_fields.binary_search(&x).is_ok() => { @@ -242,18 +258,24 @@ impl SubParser { } } _ => { - err_conv(&self.text_so_far); + return Err(UUsageError::new( + 1, + format!("%{}: invalid conversion specification", &self.text_so_far), + )); } } } if self.field_char.is_none() { - err_conv(&self.text_so_far); + return Err(UUsageError::new( + 1, + format!("%{}: invalid conversion specification", &self.text_so_far), + )); } let field_char_retrieved = self.field_char.unwrap(); if self.past_decimal && self.second_field_tmp.is_none() { self.second_field_tmp = Some(String::from("0")); } - self.validate_field_params(field_char_retrieved); + self.validate_field_params(field_char_retrieved)?; // if the dot is provided without a second field // printf interprets it as 0. if let Some(x) = self.second_field_tmp.as_mut() { @@ -262,9 +284,12 @@ impl SubParser { } } - true + Ok(true) } - fn successfully_eat_prefix(it: &mut PutBackN, text_so_far: &mut String) -> bool { + fn successfully_eat_prefix( + it: &mut PutBackN, + text_so_far: &mut String, + ) -> UResult { // get next two chars, // if they're '%%' we're not tokenizing it // else put chars back @@ -274,12 +299,14 @@ impl SubParser { match n_ch { Some(x) => { it.put_back(x); - true + Ok(true) } None => { text_so_far.push('%'); - err_conv(text_so_far); - false + return Err(UUsageError::new( + 1, + format!("%{}: invalid conversion specification", &text_so_far[..]), + )); } } } else { @@ -289,10 +316,10 @@ impl SubParser { if let Some(x) = preface { it.put_back(x); }; - false + Ok(false) } } - fn validate_field_params(&self, field_char: char) { + fn validate_field_params(&self, field_char: char) -> UResult<()> { // check for illegal combinations here when possible vs // on each application so we check less per application // to do: move these checks to Sub::new @@ -304,8 +331,15 @@ impl SubParser { || self.past_decimal || self.second_field_tmp.is_some())) { - err_conv(&self.text_so_far); + // invalid string substitution + // to do: include information about an invalid + // string substitution + return Err(UUsageError::new( + 1, + format!("%{}: invalid conversion specification", &self.text_so_far), + )); } + Ok(()) } } @@ -313,7 +347,7 @@ impl token::Tokenizer for Sub { fn from_it( it: &mut PutBackN, args: &mut Peekable>, - ) -> Option> { + ) -> UResult>> { SubParser::from_it(it, args) } } diff --git a/src/uucore/src/lib/features/tokenize/token.rs b/src/uucore/src/lib/features/tokenize/token.rs index 1d8ee5ead..6a25b620f 100644 --- a/src/uucore/src/lib/features/tokenize/token.rs +++ b/src/uucore/src/lib/features/tokenize/token.rs @@ -4,6 +4,8 @@ use std::iter::Peekable; use std::slice::Iter; use std::str::Chars; +use crate::error::UResult; + // A token object is an object that can print the expected output // of a contiguous segment of the format string, and // requires at most 1 argument @@ -27,5 +29,5 @@ pub trait Tokenizer { fn from_it( it: &mut PutBackN, args: &mut Peekable>, - ) -> Option>; + ) -> UResult>>; } diff --git a/src/uucore/src/lib/features/tokenize/unescaped_text.rs b/src/uucore/src/lib/features/tokenize/unescaped_text.rs index 0ec721b48..d186dbc26 100644 --- a/src/uucore/src/lib/features/tokenize/unescaped_text.rs +++ b/src/uucore/src/lib/features/tokenize/unescaped_text.rs @@ -13,6 +13,8 @@ use std::process::exit; use std::slice::Iter; use std::str::Chars; +use crate::error::UResult; + use super::token; const EXIT_OK: i32 = 0; @@ -266,8 +268,8 @@ impl token::Tokenizer for UnescapedText { fn from_it( it: &mut PutBackN, _: &mut Peekable>, - ) -> Option> { - Self::from_it_core(it, false) + ) -> UResult>> { + Ok(Self::from_it_core(it, false)) } } impl token::Token for UnescapedText { From f04f22b0123505137091134c962bb02915b4ead1 Mon Sep 17 00:00:00 2001 From: ndd7xv Date: Sun, 13 Feb 2022 17:28:33 -0500 Subject: [PATCH 28/78] uucore(memo): refactor error propogation with new SubError enum --- src/uucore/src/lib/features/tokenize/sub.rs | 76 +++++++++------------ 1 file changed, 33 insertions(+), 43 deletions(-) diff --git a/src/uucore/src/lib/features/tokenize/sub.rs b/src/uucore/src/lib/features/tokenize/sub.rs index a1c2f3807..f8b5b5caf 100644 --- a/src/uucore/src/lib/features/tokenize/sub.rs +++ b/src/uucore/src/lib/features/tokenize/sub.rs @@ -5,8 +5,10 @@ //! it is created by Sub's implementation of the Tokenizer trait //! Subs which have numeric field chars make use of the num_format //! submodule -use crate::error::{UResult, UUsageError}; +use crate::error::{UError, UResult}; use itertools::{put_back_n, PutBackN}; +use std::error::Error; +use std::fmt::Display; use std::iter::Peekable; use std::process::exit; use std::slice::Iter; @@ -20,6 +22,23 @@ use super::unescaped_text::UnescapedText; const EXIT_ERR: i32 = 1; +#[derive(Debug)] +pub enum SubError { + InvalidSpec(String), +} + +impl Display for SubError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { + match self { + Self::InvalidSpec(s) => write!(f, "%{}: invalid conversion specification", s), + } + } +} + +impl Error for SubError {} + +impl UError for SubError {} + fn convert_asterisk_arg_int(asterisk_arg: &str) -> isize { // this is a costly way to parse the // args used for asterisk values into integers @@ -172,10 +191,7 @@ impl SubParser { '-' | '*' | '0'..='9' => { if !self.past_decimal { if self.min_width_is_asterisk || self.specifiers_found { - return Err(UUsageError::new( - 1, - format!("%{}: invalid conversion specification", &self.text_so_far), - )); + return Err(SubError::InvalidSpec(self.text_so_far.clone()).into()); } if self.min_width_tmp.is_none() { self.min_width_tmp = Some(String::new()); @@ -183,13 +199,9 @@ impl SubParser { match self.min_width_tmp.as_mut() { Some(x) => { if (ch == '-' || ch == '*') && !x.is_empty() { - return Err(UUsageError::new( - 1, - format!( - "%{}: invalid conversion specification", - &self.text_so_far - ), - )); + return Err( + SubError::InvalidSpec(self.text_so_far.clone()).into() + ); } if ch == '*' { self.min_width_is_asterisk = true; @@ -204,10 +216,7 @@ impl SubParser { // second field should never have a // negative value if self.second_field_is_asterisk || ch == '-' || self.specifiers_found { - return Err(UUsageError::new( - 1, - format!("%{}: invalid conversion specification", &self.text_so_far), - )); + return Err(SubError::InvalidSpec(self.text_so_far.clone()).into()); } if self.second_field_tmp.is_none() { self.second_field_tmp = Some(String::new()); @@ -215,13 +224,9 @@ impl SubParser { match self.second_field_tmp.as_mut() { Some(x) => { if ch == '*' && !x.is_empty() { - return Err(UUsageError::new( - 1, - format!( - "%{}: invalid conversion specification", - &self.text_so_far - ), - )); + return Err( + SubError::InvalidSpec(self.text_so_far.clone()).into() + ); } if ch == '*' { self.second_field_is_asterisk = true; @@ -238,10 +243,7 @@ impl SubParser { if !self.past_decimal { self.past_decimal = true; } else { - return Err(UUsageError::new( - 1, - format!("%{}: invalid conversion specification", &self.text_so_far), - )); + return Err(SubError::InvalidSpec(self.text_so_far.clone()).into()); } } x if legal_fields.binary_search(&x).is_ok() => { @@ -258,18 +260,12 @@ impl SubParser { } } _ => { - return Err(UUsageError::new( - 1, - format!("%{}: invalid conversion specification", &self.text_so_far), - )); + return Err(SubError::InvalidSpec(self.text_so_far.clone()).into()); } } } if self.field_char.is_none() { - return Err(UUsageError::new( - 1, - format!("%{}: invalid conversion specification", &self.text_so_far), - )); + return Err(SubError::InvalidSpec(self.text_so_far.clone()).into()); } let field_char_retrieved = self.field_char.unwrap(); if self.past_decimal && self.second_field_tmp.is_none() { @@ -303,10 +299,7 @@ impl SubParser { } None => { text_so_far.push('%'); - return Err(UUsageError::new( - 1, - format!("%{}: invalid conversion specification", &text_so_far[..]), - )); + Err(SubError::InvalidSpec(text_so_far.clone()).into()) } } } else { @@ -334,10 +327,7 @@ impl SubParser { // invalid string substitution // to do: include information about an invalid // string substitution - return Err(UUsageError::new( - 1, - format!("%{}: invalid conversion specification", &self.text_so_far), - )); + return Err(SubError::InvalidSpec(self.text_so_far.clone()).into()); } Ok(()) } From 69a94e412bcb2f71772884bc11556720fa9bca8f Mon Sep 17 00:00:00 2001 From: Terts Diepraam Date: Thu, 17 Feb 2022 22:24:41 +0100 Subject: [PATCH 29/78] docs: use rust libraries for downloading and unzipping tldr --- .../workspace.wordlist.txt | 1 + Cargo.lock | 302 ++++++++++++++++++ Cargo.toml | 2 + docs/.gitignore | 1 - docs/Makefile | 2 - src/bin/uudoc.rs | 103 +++--- 6 files changed, 364 insertions(+), 47 deletions(-) diff --git a/.vscode/cspell.dictionaries/workspace.wordlist.txt b/.vscode/cspell.dictionaries/workspace.wordlist.txt index e41aba979..99ac20ea2 100644 --- a/.vscode/cspell.dictionaries/workspace.wordlist.txt +++ b/.vscode/cspell.dictionaries/workspace.wordlist.txt @@ -44,6 +44,7 @@ termsize termwidth textwrap thiserror +ureq walkdir winapi xattr diff --git a/Cargo.lock b/Cargo.lock index 90b71d2a7..244c56baa 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,12 @@ version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "ahash" version = "0.4.7" @@ -73,6 +79,12 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + [[package]] name = "bigdecimal" version = "0.3.0" @@ -167,6 +179,12 @@ dependencies = [ "regex-automata", ] +[[package]] +name = "bumpalo" +version = "3.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a45a46ab1f2412e53d3a0ade76ffad2025804294569aae387231a0cd6e0899" + [[package]] name = "byte-unit" version = "4.0.13" @@ -228,6 +246,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "chunked_transfer" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fff857943da45f546682664a79488be82e69e43c1a7a2307679ab9afb3a66d2e" + [[package]] name = "clang-sys" version = "1.3.0" @@ -329,6 +353,7 @@ dependencies = [ "time", "unindent", "unix_socket", + "ureq", "users", "uu_arch", "uu_base32", @@ -432,6 +457,7 @@ dependencies = [ "uu_yes", "uucore", "walkdir", + "zip", ] [[package]] @@ -546,6 +572,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "crossbeam-channel" version = "0.5.2" @@ -792,12 +827,34 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "flate2" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f" +dependencies = [ + "cfg-if 1.0.0", + "crc32fast", + "libc", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "form_urlencoded" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" +dependencies = [ + "matches", + "percent-encoding", +] + [[package]] name = "fs_extra" version = "1.2.0" @@ -927,6 +984,17 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "idna" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" +dependencies = [ + "matches", + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "if_rust_version" version = "1.0.0" @@ -967,6 +1035,15 @@ dependencies = [ "either", ] +[[package]] +name = "js-sys" +version = "0.3.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a38fc24e30fd564ce974c02bf1d337caddff65be6cc4735a1f7eab22a7440f04" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "keccak" version = "0.1.0" @@ -1044,6 +1121,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffbee8634e0d45d258acb448e7eaab3fce7a0a467395d4d9f228e3c1f01fb2e4" +[[package]] +name = "matches" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f" + [[package]] name = "md5" version = "0.3.8" @@ -1089,6 +1172,16 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +dependencies = [ + "adler", + "autocfg", +] + [[package]] name = "mio" version = "0.7.14" @@ -1375,6 +1468,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" +[[package]] +name = "percent-encoding" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" + [[package]] name = "phf" version = "0.10.1" @@ -1655,6 +1754,21 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53552c6c49e1e13f1a203ef0080ab3bbef0beb570a528993e83df057a9d9bba1" +[[package]] +name = "ring" +version = "0.16.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +dependencies = [ + "cc", + "libc", + "once_cell", + "spin", + "untrusted", + "web-sys", + "winapi 0.3.9", +] + [[package]] name = "rlimit" version = "0.4.0" @@ -1681,6 +1795,18 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustls" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b323592e3164322f5b193dc4302e4e36cd8d37158a712d664efae1a5c2791700" +dependencies = [ + "log", + "ring", + "sct", + "webpki", +] + [[package]] name = "same-file" version = "1.0.6" @@ -1696,6 +1822,16 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "sct" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "selinux" version = "0.2.5" @@ -1838,6 +1974,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "spin" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" + [[package]] name = "stable_deref_trait" version = "1.2.0" @@ -2009,6 +2151,21 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "tinyvec" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c1c1d5a42b6245520c249549ec267180beaffcc0615401ac8e31853d4b6d8d2" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" + [[package]] name = "toml" version = "0.5.8" @@ -2024,6 +2181,12 @@ version = "1.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987" +[[package]] +name = "unicode-bidi" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a01404663e3db436ed2746d9fefef640d868edae3cceb81c3b8d5732fda678f" + [[package]] name = "unicode-linebreak" version = "0.1.2" @@ -2033,6 +2196,15 @@ dependencies = [ "regex", ] +[[package]] +name = "unicode-normalization" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" +dependencies = [ + "tinyvec", +] + [[package]] name = "unicode-segmentation" version = "1.8.0" @@ -2073,6 +2245,41 @@ dependencies = [ "libc", ] +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "ureq" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9399fa2f927a3d327187cbd201480cee55bee6ac5d3c77dd27f0c6814cff16d5" +dependencies = [ + "base64", + "chunked_transfer", + "flate2", + "log", + "once_cell", + "rustls", + "url", + "webpki", + "webpki-roots", +] + +[[package]] +name = "url" +version = "2.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" +dependencies = [ + "form_urlencoded", + "idna", + "matches", + "percent-encoding", +] + [[package]] name = "users" version = "0.10.0" @@ -3171,6 +3378,89 @@ version = "0.10.2+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" +[[package]] +name = "wasm-bindgen" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25f1af7423d8588a3d840681122e72e6a24ddbcb3f0ec385cac0d12d24256c06" +dependencies = [ + "cfg-if 1.0.0", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b21c0df030f5a177f3cba22e9bc4322695ec43e7257d865302900290bcdedca" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote 1.0.14", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4203d69e40a52ee523b2529a773d5ffc1dc0071801c87b3d270b471b80ed01" +dependencies = [ + "quote 1.0.14", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa8a30d46208db204854cadbb5d4baf5fcf8071ba5bf48190c3e59937962ebc" +dependencies = [ + "proc-macro2", + "quote 1.0.14", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d958d035c4438e28c70e4321a2911302f10135ce78a9c7834c0cab4123d06a2" + +[[package]] +name = "web-sys" +version = "0.3.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c060b319f29dd25724f09a2ba1418f142f539b2be99fbf4d2d5a8f7330afb8eb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" +dependencies = [ + "ring", + "untrusted", +] + +[[package]] +name = "webpki-roots" +version = "0.22.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552ceb903e957524388c4d3475725ff2c8b7960922063af6ce53c9a43da07449" +dependencies = [ + "webpki", +] + [[package]] name = "which" version = "4.2.2" @@ -3248,3 +3538,15 @@ name = "z85" version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af896e93db81340b74b65f74276a99b210c086f3d34ed0abf433182a462af856" + +[[package]] +name = "zip" +version = "0.5.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93ab48844d61251bb3835145c521d88aa4031d7139e8485990f60ca911fa0815" +dependencies = [ + "byteorder", + "crc32fast", + "flate2", + "thiserror", +] diff --git a/Cargo.toml b/Cargo.toml index 336729813..4222f1749 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -252,6 +252,8 @@ lazy_static = { version="1.3" } textwrap = { version="0.14", features=["terminal_size"] } uucore = { version=">=0.0.11", package="uucore", path="src/uucore" } selinux = { version="0.2", optional = true } +ureq = "2.4.0" +zip = { version = "0.5.13", default_features=false, features=["deflate"] } # * uutils uu_test = { optional=true, version="0.0.12", package="uu_test", path="src/uu/test" } # diff --git a/docs/.gitignore b/docs/.gitignore index c836306f5..f2b5c7168 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -1,4 +1,3 @@ book src/utils src/SUMMARY.md -tldr/ diff --git a/docs/Makefile b/docs/Makefile index 23901b755..7372b3868 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,6 +1,4 @@ -# spell-checker:ignore tldr clean: rm -rf book rm -f src/SUMMARY.md rm -f src/utils/* - rm -rf tldr diff --git a/src/bin/uudoc.rs b/src/bin/uudoc.rs index 5658f491c..9e075ccd4 100644 --- a/src/bin/uudoc.rs +++ b/src/bin/uudoc.rs @@ -7,27 +7,21 @@ use clap::App; use std::ffi::OsString; use std::fs::File; -use std::io::{self, Read, Write}; -use std::process::Command; +use std::io::Cursor; +use std::io::{self, Read, Seek, Write}; +use zip::ZipArchive; include!(concat!(env!("OUT_DIR"), "/uutils_map.rs")); fn main() -> io::Result<()> { - let _ = std::fs::create_dir("docs/tldr"); println!("Downloading tldr archive"); - Command::new("curl") - .arg("https://tldr.sh/assets/tldr.zip") - .arg("--output") - .arg("docs/tldr/tldr.zip") - .output()?; - - println!("Unzipping tldr archive"); - Command::new("unzip") - .arg("-o") - .arg("docs/tldr/tldr.zip") - .arg("-d") - .arg("docs/tldr") - .output()?; + let mut zip_reader = ureq::get("https://tldr.sh/assets/tldr.zip") + .call() + .unwrap() + .into_reader(); + let mut buffer = Vec::new(); + zip_reader.read_to_end(&mut buffer).unwrap(); + let mut tldr_zip = ZipArchive::new(Cursor::new(buffer)).unwrap(); let utils = util_map::>>(); match std::fs::create_dir("docs/src/utils/") { @@ -58,7 +52,7 @@ fn main() -> io::Result<()> { } let p = format!("docs/src/utils/{}.md", name); if let Ok(f) = File::create(&p) { - write_markdown(f, &mut app(), name)?; + write_markdown(f, &mut app(), name, &mut tldr_zip)?; println!("Wrote to '{}'", p); } else { println!("Error writing to {}", p); @@ -68,12 +62,17 @@ fn main() -> io::Result<()> { Ok(()) } -fn write_markdown(mut w: impl Write, app: &mut App, name: &str) -> io::Result<()> { +fn write_markdown( + mut w: impl Write, + app: &mut App, + name: &str, + tldr_zip: &mut zip::ZipArchive, +) -> io::Result<()> { write!(w, "# {}\n\n", name)?; write_version(&mut w, app)?; write_usage(&mut w, app, name)?; write_description(&mut w, app)?; - write_examples(&mut w, name)?; + write_examples(&mut w, name, tldr_zip)?; write_options(&mut w, app) } @@ -101,33 +100,49 @@ fn write_description(w: &mut impl Write, app: &App) -> io::Result<()> { } } -fn write_examples(w: &mut impl Write, name: &str) -> io::Result<()> { - if let Ok(mut file) = std::fs::File::open(format!("docs/tldr/pages/common/{}.md", name)) - .or_else(|_| std::fs::File::open(format!("docs/tldr/pages/linux/{}.md", name))) - { - let mut content = String::new(); - file.read_to_string(&mut content)?; - - writeln!(w, "## Examples")?; - writeln!(w)?; - for line in content.lines().skip_while(|l| !l.starts_with('-')) { - if let Some(l) = line.strip_prefix("- ") { - writeln!(w, "{}", l)?; - } else if line.starts_with('`') { - writeln!(w, "```shell\n{}\n```", line.trim_matches('`'))?; - } else if line.is_empty() { - writeln!(w)?; - } else { - println!("Not sure what to do with this line:"); - println!("{}", line); - } - } - writeln!(w)?; - writeln!(w, "> The examples are provided by the [tldr-pages project](https://tldr.sh) under the [CC BY 4.0 License](https://github.com/tldr-pages/tldr/blob/main/LICENSE.md).")?; +fn write_examples( + w: &mut impl Write, + name: &str, + tldr_zip: &mut zip::ZipArchive, +) -> io::Result<()> { + let content = if let Some(f) = get_zip_content(tldr_zip, &format!("pages/common/{}.md", name)) { + f + } else if let Some(f) = get_zip_content(tldr_zip, &format!("pages/linux/{}.md", name)) { + f } else { - println!("No examples found for: {}", name); + return Ok(()); + }; + + writeln!(w, "## Examples")?; + writeln!(w)?; + for line in content.lines().skip_while(|l| !l.starts_with('-')) { + if let Some(l) = line.strip_prefix("- ") { + writeln!(w, "{}", l)?; + } else if line.starts_with('`') { + writeln!(w, "```shell\n{}\n```", line.trim_matches('`'))?; + } else if line.is_empty() { + writeln!(w)?; + } else { + println!("Not sure what to do with this line:"); + println!("{}", line); + } } - Ok(()) + writeln!(w)?; + writeln!( + w, + "> The examples are provided by the [tldr-pages project](https://tldr.sh) under the [CC BY 4.0 License](https://github.com/tldr-pages/tldr/blob/main/LICENSE.md)." + )?; + writeln!(w, ">")?; + writeln!( + w, + "> Please note that, as uutils is a work in progress, some examples might fail." + ) +} + +fn get_zip_content(archive: &mut ZipArchive, name: &str) -> Option { + let mut s = String::new(); + archive.by_name(name).ok()?.read_to_string(&mut s).unwrap(); + Some(s) } fn write_options(w: &mut impl Write, app: &App) -> io::Result<()> { From 766c85fb5efa2ff2bd252c25f314925bc4730943 Mon Sep 17 00:00:00 2001 From: Jeffrey Finkelstein Date: Thu, 17 Feb 2022 22:35:30 -0500 Subject: [PATCH 30/78] dd: correct order and phrasing of truncated record Place the "truncated records" line below the "records out" line in the status report produced by `dd` and properly handle the singularization of the word "record" in the case of 1 truncated record. This matches the behavior of GNU `dd`. For example $ printf "ab" | dd cbs=1 conv=block status=noxfer > /dev/null 0+1 records in 0+1 records out 1 truncated record $ printf "ab\ncd\n" | dd cbs=1 conv=block status=noxfer > /dev/null 0+1 records in 0+1 records out 2 truncated records --- src/uu/dd/src/dd.rs | 8 +++++--- tests/by-util/test_dd.rs | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/uu/dd/src/dd.rs b/src/uu/dd/src/dd.rs index b24e18049..b38671a9a 100644 --- a/src/uu/dd/src/dd.rs +++ b/src/uu/dd/src/dd.rs @@ -779,13 +779,15 @@ fn print_io_lines(update: &ProgUpdate) { "{}+{} records in", update.read_stat.reads_complete, update.read_stat.reads_partial ); - if update.read_stat.records_truncated > 0 { - eprintln!("{} truncated records", update.read_stat.records_truncated); - } eprintln!( "{}+{} records out", update.write_stat.writes_complete, update.write_stat.writes_partial ); + match update.read_stat.records_truncated { + 0 => {} + 1 => eprintln!("1 truncated record"), + n => eprintln!("{} truncated records", n), + } } // Print the progress line of a status update: // bytes (, ) copied,
") + writeln!(w, "\n") } From be0b77e7a73dded31deb35f48a8027285ec77cba Mon Sep 17 00:00:00 2001 From: Sylvestre Ledru Date: Sat, 19 Feb 2022 12:47:29 +0100 Subject: [PATCH 33/78] when help item has an \n, translate it to a
example: https://uutils.github.io/coreutils-docs/user/utils/ls.html / classify --- src/bin/uudoc.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/bin/uudoc.rs b/src/bin/uudoc.rs index 4020b2787..33e5bf607 100644 --- a/src/bin/uudoc.rs +++ b/src/bin/uudoc.rs @@ -201,7 +201,11 @@ fn write_options(w: &mut impl Write, app: &App) -> io::Result<()> { write!(w, "")?; } writeln!(w, "")?; - writeln!(w, "
\n\n{}\n\n
", arg.get_help().unwrap_or_default())?; + writeln!( + w, + "
\n\n{}\n\n
", + arg.get_help().unwrap_or_default().replace("\n", "
") + )?; } writeln!(w, "\n") } From 6900638ac6a84a9cd4313df93ef3502dcfa30dd2 Mon Sep 17 00:00:00 2001 From: Jeffrey Finkelstein Date: Wed, 16 Feb 2022 21:32:38 -0500 Subject: [PATCH 34/78] dd: don't error when outfile is /dev/null Prevent `dd` from terminating with an error when given the command-line argument `of=/dev/null`. This commit allows the call to `File::set_len()` to result in an error without causing the process to terminate prematurely. --- src/uu/dd/src/dd.rs | 11 +++++++++-- tests/by-util/test_dd.rs | 7 +++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/uu/dd/src/dd.rs b/src/uu/dd/src/dd.rs index b38671a9a..1ce64bb78 100644 --- a/src/uu/dd/src/dd.rs +++ b/src/uu/dd/src/dd.rs @@ -521,10 +521,17 @@ impl OutputTrait for Output { let mut dst = open_dst(Path::new(&fname), &cflags, &oflags) .map_err_context(|| format!("failed to open {}", fname.quote()))?; + // Seek to the index in the output file, truncating if requested. + // + // Calling `set_len()` may result in an error (for + // example, when calling it on `/dev/null`), but we don't + // want to terminate the process when that happens. + // Instead, we suppress the error by calling + // `Result::ok()`. This matches the behavior of GNU `dd` + // when given the command-line argument `of=/dev/null`. let i = seek.unwrap_or(0).try_into().unwrap(); if !cflags.notrunc { - dst.set_len(i) - .map_err_context(|| "failed to truncate output file".to_string())?; + dst.set_len(i).ok(); } dst.seek(io::SeekFrom::Start(i)) .map_err_context(|| "failed to seek in output file".to_string())?; diff --git a/tests/by-util/test_dd.rs b/tests/by-util/test_dd.rs index 7a52488eb..04f5490ec 100644 --- a/tests/by-util/test_dd.rs +++ b/tests/by-util/test_dd.rs @@ -1095,3 +1095,10 @@ fn test_truncated_record() { .stdout_is("ac") .stderr_is("0+1 records in\n0+1 records out\n2 truncated records\n"); } + +/// Test that the output file can be `/dev/null`. +#[cfg(unix)] +#[test] +fn test_outfile_dev_null() { + new_ucmd!().arg("of=/dev/null").succeeds().no_stdout(); +} From b09bae2acfebad4f21b5827b4f5ca3d44d0dbf93 Mon Sep 17 00:00:00 2001 From: Jeffrey Finkelstein Date: Wed, 2 Feb 2022 22:36:44 -0500 Subject: [PATCH 35/78] dd: collect progress reporting into its own module Collect structs, implementations, and functions that have to do with reporting number of blocks read and written into their own new module, `progress.rs`. This commit also adds docstrings for everything and unit tests for the significant methods. This commit does not change the behavior of `dd`, just the organization of the code to make it more maintainable and testable. --- src/uu/dd/src/datastructures.rs | 70 ----- src/uu/dd/src/dd.rs | 125 +------- src/uu/dd/src/parseargs.rs | 2 +- src/uu/dd/src/progress.rs | 517 ++++++++++++++++++++++++++++++++ 4 files changed, 524 insertions(+), 190 deletions(-) create mode 100644 src/uu/dd/src/progress.rs diff --git a/src/uu/dd/src/datastructures.rs b/src/uu/dd/src/datastructures.rs index 8380965a9..c9c89e858 100644 --- a/src/uu/dd/src/datastructures.rs +++ b/src/uu/dd/src/datastructures.rs @@ -7,72 +7,11 @@ // spell-checker:ignore ctable, outfile use std::error::Error; -use std::time; use uucore::error::UError; use crate::conversion_tables::*; -pub struct ProgUpdate { - pub read_stat: ReadStat, - pub write_stat: WriteStat, - pub duration: time::Duration, -} - -impl ProgUpdate { - pub(crate) fn new( - read_stat: ReadStat, - write_stat: WriteStat, - duration: time::Duration, - ) -> Self { - Self { - read_stat, - write_stat, - duration, - } - } -} - -#[derive(Clone, Copy, Default)] -pub struct ReadStat { - pub reads_complete: u64, - pub reads_partial: u64, - pub records_truncated: u32, -} - -impl ReadStat { - /// Whether this counter has zero complete reads and zero partial reads. - pub(crate) fn is_empty(&self) -> bool { - self.reads_complete == 0 && self.reads_partial == 0 - } -} - -impl std::ops::AddAssign for ReadStat { - fn add_assign(&mut self, other: Self) { - *self = Self { - reads_complete: self.reads_complete + other.reads_complete, - reads_partial: self.reads_partial + other.reads_partial, - records_truncated: self.records_truncated + other.records_truncated, - } - } -} - -#[derive(Clone, Copy, Default)] -pub struct WriteStat { - pub writes_complete: u64, - pub writes_partial: u64, - pub bytes_total: u128, -} -impl std::ops::AddAssign for WriteStat { - fn add_assign(&mut self, other: Self) { - *self = Self { - writes_complete: self.writes_complete + other.writes_complete, - writes_partial: self.writes_partial + other.writes_partial, - bytes_total: self.bytes_total + other.bytes_total, - } - } -} - type Cbs = usize; /// Stores all Conv Flags that apply to the input @@ -138,15 +77,6 @@ pub struct OFlags { pub seek_bytes: bool, } -/// The value of the status cl-option. -/// Controls printing of transfer stats -#[derive(Copy, Clone, Debug, PartialEq)] -pub enum StatusLevel { - Progress, - Noxfer, - None, -} - /// The value of count=N /// Defaults to Reads(N) /// if iflag=count_bytes diff --git a/src/uu/dd/src/dd.rs b/src/uu/dd/src/dd.rs index b38671a9a..c8004b893 100644 --- a/src/uu/dd/src/dd.rs +++ b/src/uu/dd/src/dd.rs @@ -5,7 +5,7 @@ // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore fname, tname, fpath, specfile, testfile, unspec, ifile, ofile, outfile, fullblock, urand, fileio, atoe, atoibm, behaviour, bmax, bremain, btotal, cflags, creat, ctable, ctty, datastructures, doesnt, etoa, fileout, fname, gnudd, iconvflags, nocache, noctty, noerror, nofollow, nolinks, nonblock, oconvflags, outfile, parseargs, rlen, rmax, rposition, rremain, rsofar, rstat, sigusr, sigval, wlen, wstat seekable +// spell-checker:ignore fname, tname, fpath, specfile, testfile, unspec, ifile, ofile, outfile, fullblock, urand, fileio, atoe, atoibm, behaviour, bmax, bremain, cflags, creat, ctable, ctty, datastructures, doesnt, etoa, fileout, fname, gnudd, iconvflags, nocache, noctty, noerror, nofollow, nolinks, nonblock, oconvflags, outfile, parseargs, rlen, rmax, rposition, rremain, rsofar, rstat, sigusr, wlen, wstat seekable mod datastructures; use datastructures::*; @@ -16,27 +16,23 @@ use parseargs::Matches; mod conversion_tables; use conversion_tables::*; +mod progress; +use progress::{gen_prog_updater, ProgUpdate, ReadStat, StatusLevel, WriteStat}; + use std::cmp; use std::convert::TryInto; use std::env; -#[cfg(target_os = "linux")] -use std::error::Error; use std::fs::{File, OpenOptions}; use std::io::{self, Read, Seek, Write}; #[cfg(target_os = "linux")] use std::os::unix::fs::OpenOptionsExt; use std::path::Path; use std::sync::mpsc; -#[cfg(target_os = "linux")] -use std::sync::{atomic::AtomicUsize, atomic::Ordering, Arc}; use std::thread; use std::time; -use byte_unit::Byte; use clap::{crate_version, App, AppSettings, Arg, ArgMatches}; use gcd::Gcd; -#[cfg(target_os = "linux")] -use signal_hook::consts::signal; use uucore::display::Quotable; use uucore::error::{FromIo, UResult, USimpleError}; use uucore::show_error; @@ -351,8 +347,8 @@ where fn print_stats(&self, i: &Input, prog_update: &ProgUpdate) { match i.print_level { Some(StatusLevel::None) => {} - Some(StatusLevel::Noxfer) => print_io_lines(prog_update), - Some(StatusLevel::Progress) | None => print_transfer_stats(prog_update), + Some(StatusLevel::Noxfer) => prog_update.print_io_lines(), + Some(StatusLevel::Progress) | None => prog_update.print_transfer_stats(), } } @@ -771,115 +767,6 @@ fn read_helper(i: &mut Input, bsize: usize) -> std::io::Result<(Read } } -// Print io lines of a status update: -// + records in -// + records out -fn print_io_lines(update: &ProgUpdate) { - eprintln!( - "{}+{} records in", - update.read_stat.reads_complete, update.read_stat.reads_partial - ); - eprintln!( - "{}+{} records out", - update.write_stat.writes_complete, update.write_stat.writes_partial - ); - match update.read_stat.records_truncated { - 0 => {} - 1 => eprintln!("1 truncated record"), - n => eprintln!("{} truncated records", n), - } -} -// Print the progress line of a status update: -// bytes (, ) copied,