1
Fork 0
mirror of https://github.com/RGBCube/uutils-coreutils synced 2025-07-30 20:47:46 +00:00

Fix tests

This commit is contained in:
electricboogie 2021-04-25 12:38:43 -05:00
parent 094d9a9e47
commit f0a473f40e
3 changed files with 49 additions and 23 deletions

View file

@ -1,4 +1,4 @@
use std::{clone::Clone}; use std::clone::Clone;
use std::cmp::Ordering::Less; use std::cmp::Ordering::Less;
use std::collections::VecDeque; use std::collections::VecDeque;
use std::error::Error; use std::error::Error;
@ -92,10 +92,11 @@ where
let mut idx = 0; let mut idx = 0;
for chunk_num in 0..self.chunks as usize { for chunk_num in 0..self.chunks as usize {
if !self.buffers[chunk_num].is_empty() { if !self.buffers[chunk_num].is_empty() {
if self.buffers[idx].is_empty() || (super::compare_by)( if self.buffers[idx].is_empty()
|| (super::compare_by)(
self.buffers[chunk_num].front().unwrap(), self.buffers[chunk_num].front().unwrap(),
self.buffers[idx].front().unwrap(), self.buffers[idx].front().unwrap(),
&self.settings &self.settings,
) == Less ) == Less
{ {
idx = chunk_num; idx = chunk_num;
@ -106,7 +107,7 @@ where
// unwrap due to checks above // unwrap due to checks above
let r = self.buffers[idx].pop_front().unwrap(); let r = self.buffers[idx].pop_front().unwrap();
Some(Ok(r)) Some(Ok(r))
} }
} }
/// Perform an external sort on an unsorted stream of incoming data /// Perform an external sort on an unsorted stream of incoming data
@ -126,7 +127,11 @@ where
{ {
/// Create a new `ExternalSorter` with a specified memory buffer and /// Create a new `ExternalSorter` with a specified memory buffer and
/// temporary directory /// temporary directory
pub fn new(buffer_bytes: u64, tmp_dir: Option<PathBuf>, settings: GlobalSettings) -> ExternalSorter<Line> { pub fn new(
buffer_bytes: u64,
tmp_dir: Option<PathBuf>,
settings: GlobalSettings,
) -> ExternalSorter<Line> {
ExternalSorter { ExternalSorter {
buffer_bytes, buffer_bytes,
tmp_dir, tmp_dir,
@ -142,7 +147,11 @@ where
/// ///
/// This method can fail due to issues writing intermediate sorted chunks /// This method can fail due to issues writing intermediate sorted chunks
/// to disk, or due to serde serialization issues /// to disk, or due to serde serialization issues
pub fn sort_by<I>(&self, unsorted: I, settings: GlobalSettings) -> Result<ExtSortedIterator<Line>, Box<dyn Error>> pub fn sort_by<I>(
&self,
unsorted: I,
settings: GlobalSettings,
) -> Result<ExtSortedIterator<Line>, Box<dyn Error>>
where where
I: Iterator<Item = Line>, I: Iterator<Item = Line>,
{ {
@ -193,7 +202,10 @@ where
} }
// initialize buffers for each chunk // initialize buffers for each chunk
iter.max_per_chunk = self.buffer_bytes.checked_div(iter.chunks).unwrap_or(self.buffer_bytes); iter.max_per_chunk = self
.buffer_bytes
.checked_div(iter.chunks)
.unwrap_or(self.buffer_bytes);
iter.buffers = vec![VecDeque::new(); iter.chunks as usize]; iter.buffers = vec![VecDeque::new(); iter.chunks as usize];
iter.chunk_offsets = vec![0 as u64; iter.chunks as usize]; iter.chunk_offsets = vec![0 as u64; iter.chunks as usize];
for chunk_num in 0..iter.chunks { for chunk_num in 0..iter.chunks {
@ -221,7 +233,11 @@ where
} }
} }
fn fill_buff<Line>(vec: &mut VecDeque<Line>, file: File, max_bytes: u64) -> Result<u64, Box<dyn Error>> fn fill_buff<Line>(
vec: &mut VecDeque<Line>,
file: File,
max_bytes: u64,
) -> Result<u64, Box<dyn Error>>
where where
Line: ExternallySortable, Line: ExternallySortable,
{ {

View file

@ -15,11 +15,11 @@
#[macro_use] #[macro_use]
extern crate uucore; extern crate uucore;
mod numeric_str_cmp;
mod external_sort; mod external_sort;
mod numeric_str_cmp;
use external_sort::{ExternalSorter, ExternallySortable};
use clap::{App, Arg}; use clap::{App, Arg};
use external_sort::{ExternalSorter, ExternallySortable};
use fnv::FnvHasher; use fnv::FnvHasher;
use itertools::Itertools; use itertools::Itertools;
use numeric_str_cmp::{numeric_str_cmp, NumInfo, NumInfoParseSettings}; use numeric_str_cmp::{numeric_str_cmp, NumInfo, NumInfoParseSettings};
@ -39,9 +39,9 @@ use std::io::{stdin, stdout, BufRead, BufReader, BufWriter, Lines, Read, Write};
use std::mem::replace; use std::mem::replace;
use std::ops::Range; use std::ops::Range;
use std::path::Path; use std::path::Path;
use std::path::PathBuf;
use unicode_width::UnicodeWidthStr; use unicode_width::UnicodeWidthStr;
use uucore::fs::is_stdin_interactive; // for Iterator::dedup() use uucore::fs::is_stdin_interactive; // for Iterator::dedup()
use std::path::PathBuf;
static NAME: &str = "sort"; static NAME: &str = "sort";
static ABOUT: &str = "Display sorted concatenation of all FILE(s)."; static ABOUT: &str = "Display sorted concatenation of all FILE(s).";
@ -136,7 +136,9 @@ impl GlobalSettings {
fn human_numeric_convert(a: &str) -> usize { fn human_numeric_convert(a: &str) -> usize {
let num_str = &a[get_leading_gen(a)]; let num_str = &a[get_leading_gen(a)];
let (_, suf_str) = a.split_at(num_str.len()); let (_, suf_str) = a.split_at(num_str.len());
let num_usize = num_str.parse::<usize>().expect("Error parsing buffer size: "); let num_usize = num_str
.parse::<usize>()
.expect("Error parsing buffer size: ");
let suf_usize: usize = match suf_str.to_uppercase().as_str() { let suf_usize: usize = match suf_str.to_uppercase().as_str() {
// SI Units // SI Units
"K" => 1000usize, "K" => 1000usize,
@ -323,7 +325,9 @@ impl Line {
); );
range.shorten(num_range); range.shorten(num_range);
NumCache::WithInfo(info) NumCache::WithInfo(info)
} else if selector.settings.mode == SortMode::GeneralNumeric && settings.buffer_size == DEFAULT_BUF_SIZE { } else if selector.settings.mode == SortMode::GeneralNumeric
&& settings.buffer_size == DEFAULT_BUF_SIZE
{
let str = range.get_str(&line); let str = range.get_str(&line);
NumCache::AsF64(general_f64_parse(&str[get_leading_gen(str)])) NumCache::AsF64(general_f64_parse(&str[get_leading_gen(str)]))
} else { } else {
@ -1050,7 +1054,7 @@ pub fn uumain(args: impl uucore::Args) -> i32 {
.value_of(OPT_BUF_SIZE) .value_of(OPT_BUF_SIZE)
.map(String::from) .map(String::from)
.unwrap_or(format!("{}", DEFAULT_BUF_SIZE)); .unwrap_or(format!("{}", DEFAULT_BUF_SIZE));
GlobalSettings::human_numeric_convert(&input) GlobalSettings::human_numeric_convert(&input)
} }
} }
@ -1261,13 +1265,17 @@ fn exec_check_file(unwrapped_lines: &[Line], settings: &GlobalSettings) -> i32 {
} }
fn ext_sort_by(unsorted: Vec<Line>, settings: GlobalSettings) -> Vec<Line> { fn ext_sort_by(unsorted: Vec<Line>, settings: GlobalSettings) -> Vec<Line> {
let external_sorter = ExternalSorter::new(settings.buffer_size as u64, Some(settings.tmp_dir.clone()), settings.clone()); let external_sorter = ExternalSorter::new(
settings.buffer_size as u64,
Some(settings.tmp_dir.clone()),
settings.clone(),
);
let iter = external_sorter let iter = external_sorter
.sort_by(unsorted.into_iter(), settings.clone()) .sort_by(unsorted.into_iter(), settings.clone())
.unwrap() .unwrap()
.map(|x| x.unwrap()) .map(|x| x.unwrap())
.collect::<Vec<Line>>(); .collect::<Vec<Line>>();
iter iter
} }
fn sort_by(lines: &mut Vec<Line>, settings: &GlobalSettings) { fn sort_by(lines: &mut Vec<Line>, settings: &GlobalSettings) {
@ -1291,18 +1299,19 @@ fn compare_by(a: &Line, b: &Line, global_settings: &GlobalSettings) -> Ordering
(b_str, b_selection.num_cache.as_num_info()), (b_str, b_selection.num_cache.as_num_info()),
), ),
// serde JSON has issues with f64 null values, so caching them won't work for us with ext sort // serde JSON has issues with f64 null values, so caching them won't work for us with ext sort
SortMode::GeneralNumeric => SortMode::GeneralNumeric => {
if global_settings.buffer_size == DEFAULT_BUF_SIZE { if global_settings.buffer_size == DEFAULT_BUF_SIZE {
general_numeric_compare( general_numeric_compare(
a_selection.num_cache.as_f64(), a_selection.num_cache.as_f64(),
b_selection.num_cache.as_f64() b_selection.num_cache.as_f64(),
) )
} else { } else {
general_numeric_compare( general_numeric_compare(
general_f64_parse(&a_str[get_leading_gen(a_str)]), general_f64_parse(&a_str[get_leading_gen(a_str)]),
general_f64_parse(&b_str[get_leading_gen(b_str)]) general_f64_parse(&b_str[get_leading_gen(b_str)]),
) )
}, }
}
SortMode::Month => month_compare(a_str, b_str), SortMode::Month => month_compare(a_str, b_str),
SortMode::Version => version_compare(a_str, b_str), SortMode::Version => version_compare(a_str, b_str),
SortMode::Default => default_compare(a_str, b_str), SortMode::Default => default_compare(a_str, b_str),

View file

@ -33,7 +33,8 @@ fn test_helper(file_name: &str, args: &str) {
fn test_larger_than_specified_segment() { fn test_larger_than_specified_segment() {
new_ucmd!() new_ucmd!()
.arg("-n") .arg("-n")
.arg("-S 50M") .arg("-S")
.arg("50K")
.arg("ext_sort.txt") .arg("ext_sort.txt")
.succeeds() .succeeds()
.stdout_is_fixture(format!("{}", "ext_sort.expected")); .stdout_is_fixture(format!("{}", "ext_sort.expected"));