mirror of
https://github.com/RGBCube/uutils-coreutils
synced 2025-07-30 20:47:46 +00:00
Fix tests
This commit is contained in:
parent
094d9a9e47
commit
f0a473f40e
3 changed files with 49 additions and 23 deletions
|
@ -1,4 +1,4 @@
|
||||||
use std::{clone::Clone};
|
use std::clone::Clone;
|
||||||
use std::cmp::Ordering::Less;
|
use std::cmp::Ordering::Less;
|
||||||
use std::collections::VecDeque;
|
use std::collections::VecDeque;
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
|
@ -92,10 +92,11 @@ where
|
||||||
let mut idx = 0;
|
let mut idx = 0;
|
||||||
for chunk_num in 0..self.chunks as usize {
|
for chunk_num in 0..self.chunks as usize {
|
||||||
if !self.buffers[chunk_num].is_empty() {
|
if !self.buffers[chunk_num].is_empty() {
|
||||||
if self.buffers[idx].is_empty() || (super::compare_by)(
|
if self.buffers[idx].is_empty()
|
||||||
|
|| (super::compare_by)(
|
||||||
self.buffers[chunk_num].front().unwrap(),
|
self.buffers[chunk_num].front().unwrap(),
|
||||||
self.buffers[idx].front().unwrap(),
|
self.buffers[idx].front().unwrap(),
|
||||||
&self.settings
|
&self.settings,
|
||||||
) == Less
|
) == Less
|
||||||
{
|
{
|
||||||
idx = chunk_num;
|
idx = chunk_num;
|
||||||
|
@ -106,7 +107,7 @@ where
|
||||||
// unwrap due to checks above
|
// unwrap due to checks above
|
||||||
let r = self.buffers[idx].pop_front().unwrap();
|
let r = self.buffers[idx].pop_front().unwrap();
|
||||||
Some(Ok(r))
|
Some(Ok(r))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Perform an external sort on an unsorted stream of incoming data
|
/// Perform an external sort on an unsorted stream of incoming data
|
||||||
|
@ -126,7 +127,11 @@ where
|
||||||
{
|
{
|
||||||
/// Create a new `ExternalSorter` with a specified memory buffer and
|
/// Create a new `ExternalSorter` with a specified memory buffer and
|
||||||
/// temporary directory
|
/// temporary directory
|
||||||
pub fn new(buffer_bytes: u64, tmp_dir: Option<PathBuf>, settings: GlobalSettings) -> ExternalSorter<Line> {
|
pub fn new(
|
||||||
|
buffer_bytes: u64,
|
||||||
|
tmp_dir: Option<PathBuf>,
|
||||||
|
settings: GlobalSettings,
|
||||||
|
) -> ExternalSorter<Line> {
|
||||||
ExternalSorter {
|
ExternalSorter {
|
||||||
buffer_bytes,
|
buffer_bytes,
|
||||||
tmp_dir,
|
tmp_dir,
|
||||||
|
@ -142,7 +147,11 @@ where
|
||||||
///
|
///
|
||||||
/// This method can fail due to issues writing intermediate sorted chunks
|
/// This method can fail due to issues writing intermediate sorted chunks
|
||||||
/// to disk, or due to serde serialization issues
|
/// to disk, or due to serde serialization issues
|
||||||
pub fn sort_by<I>(&self, unsorted: I, settings: GlobalSettings) -> Result<ExtSortedIterator<Line>, Box<dyn Error>>
|
pub fn sort_by<I>(
|
||||||
|
&self,
|
||||||
|
unsorted: I,
|
||||||
|
settings: GlobalSettings,
|
||||||
|
) -> Result<ExtSortedIterator<Line>, Box<dyn Error>>
|
||||||
where
|
where
|
||||||
I: Iterator<Item = Line>,
|
I: Iterator<Item = Line>,
|
||||||
{
|
{
|
||||||
|
@ -193,7 +202,10 @@ where
|
||||||
}
|
}
|
||||||
|
|
||||||
// initialize buffers for each chunk
|
// initialize buffers for each chunk
|
||||||
iter.max_per_chunk = self.buffer_bytes.checked_div(iter.chunks).unwrap_or(self.buffer_bytes);
|
iter.max_per_chunk = self
|
||||||
|
.buffer_bytes
|
||||||
|
.checked_div(iter.chunks)
|
||||||
|
.unwrap_or(self.buffer_bytes);
|
||||||
iter.buffers = vec![VecDeque::new(); iter.chunks as usize];
|
iter.buffers = vec![VecDeque::new(); iter.chunks as usize];
|
||||||
iter.chunk_offsets = vec![0 as u64; iter.chunks as usize];
|
iter.chunk_offsets = vec![0 as u64; iter.chunks as usize];
|
||||||
for chunk_num in 0..iter.chunks {
|
for chunk_num in 0..iter.chunks {
|
||||||
|
@ -221,7 +233,11 @@ where
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn fill_buff<Line>(vec: &mut VecDeque<Line>, file: File, max_bytes: u64) -> Result<u64, Box<dyn Error>>
|
fn fill_buff<Line>(
|
||||||
|
vec: &mut VecDeque<Line>,
|
||||||
|
file: File,
|
||||||
|
max_bytes: u64,
|
||||||
|
) -> Result<u64, Box<dyn Error>>
|
||||||
where
|
where
|
||||||
Line: ExternallySortable,
|
Line: ExternallySortable,
|
||||||
{
|
{
|
||||||
|
|
|
@ -15,11 +15,11 @@
|
||||||
#[macro_use]
|
#[macro_use]
|
||||||
extern crate uucore;
|
extern crate uucore;
|
||||||
|
|
||||||
mod numeric_str_cmp;
|
|
||||||
mod external_sort;
|
mod external_sort;
|
||||||
|
mod numeric_str_cmp;
|
||||||
|
|
||||||
use external_sort::{ExternalSorter, ExternallySortable};
|
|
||||||
use clap::{App, Arg};
|
use clap::{App, Arg};
|
||||||
|
use external_sort::{ExternalSorter, ExternallySortable};
|
||||||
use fnv::FnvHasher;
|
use fnv::FnvHasher;
|
||||||
use itertools::Itertools;
|
use itertools::Itertools;
|
||||||
use numeric_str_cmp::{numeric_str_cmp, NumInfo, NumInfoParseSettings};
|
use numeric_str_cmp::{numeric_str_cmp, NumInfo, NumInfoParseSettings};
|
||||||
|
@ -39,9 +39,9 @@ use std::io::{stdin, stdout, BufRead, BufReader, BufWriter, Lines, Read, Write};
|
||||||
use std::mem::replace;
|
use std::mem::replace;
|
||||||
use std::ops::Range;
|
use std::ops::Range;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
|
use std::path::PathBuf;
|
||||||
use unicode_width::UnicodeWidthStr;
|
use unicode_width::UnicodeWidthStr;
|
||||||
use uucore::fs::is_stdin_interactive; // for Iterator::dedup()
|
use uucore::fs::is_stdin_interactive; // for Iterator::dedup()
|
||||||
use std::path::PathBuf;
|
|
||||||
|
|
||||||
static NAME: &str = "sort";
|
static NAME: &str = "sort";
|
||||||
static ABOUT: &str = "Display sorted concatenation of all FILE(s).";
|
static ABOUT: &str = "Display sorted concatenation of all FILE(s).";
|
||||||
|
@ -136,7 +136,9 @@ impl GlobalSettings {
|
||||||
fn human_numeric_convert(a: &str) -> usize {
|
fn human_numeric_convert(a: &str) -> usize {
|
||||||
let num_str = &a[get_leading_gen(a)];
|
let num_str = &a[get_leading_gen(a)];
|
||||||
let (_, suf_str) = a.split_at(num_str.len());
|
let (_, suf_str) = a.split_at(num_str.len());
|
||||||
let num_usize = num_str.parse::<usize>().expect("Error parsing buffer size: ");
|
let num_usize = num_str
|
||||||
|
.parse::<usize>()
|
||||||
|
.expect("Error parsing buffer size: ");
|
||||||
let suf_usize: usize = match suf_str.to_uppercase().as_str() {
|
let suf_usize: usize = match suf_str.to_uppercase().as_str() {
|
||||||
// SI Units
|
// SI Units
|
||||||
"K" => 1000usize,
|
"K" => 1000usize,
|
||||||
|
@ -323,7 +325,9 @@ impl Line {
|
||||||
);
|
);
|
||||||
range.shorten(num_range);
|
range.shorten(num_range);
|
||||||
NumCache::WithInfo(info)
|
NumCache::WithInfo(info)
|
||||||
} else if selector.settings.mode == SortMode::GeneralNumeric && settings.buffer_size == DEFAULT_BUF_SIZE {
|
} else if selector.settings.mode == SortMode::GeneralNumeric
|
||||||
|
&& settings.buffer_size == DEFAULT_BUF_SIZE
|
||||||
|
{
|
||||||
let str = range.get_str(&line);
|
let str = range.get_str(&line);
|
||||||
NumCache::AsF64(general_f64_parse(&str[get_leading_gen(str)]))
|
NumCache::AsF64(general_f64_parse(&str[get_leading_gen(str)]))
|
||||||
} else {
|
} else {
|
||||||
|
@ -1050,7 +1054,7 @@ pub fn uumain(args: impl uucore::Args) -> i32 {
|
||||||
.value_of(OPT_BUF_SIZE)
|
.value_of(OPT_BUF_SIZE)
|
||||||
.map(String::from)
|
.map(String::from)
|
||||||
.unwrap_or(format!("{}", DEFAULT_BUF_SIZE));
|
.unwrap_or(format!("{}", DEFAULT_BUF_SIZE));
|
||||||
|
|
||||||
GlobalSettings::human_numeric_convert(&input)
|
GlobalSettings::human_numeric_convert(&input)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -1261,13 +1265,17 @@ fn exec_check_file(unwrapped_lines: &[Line], settings: &GlobalSettings) -> i32 {
|
||||||
}
|
}
|
||||||
|
|
||||||
fn ext_sort_by(unsorted: Vec<Line>, settings: GlobalSettings) -> Vec<Line> {
|
fn ext_sort_by(unsorted: Vec<Line>, settings: GlobalSettings) -> Vec<Line> {
|
||||||
let external_sorter = ExternalSorter::new(settings.buffer_size as u64, Some(settings.tmp_dir.clone()), settings.clone());
|
let external_sorter = ExternalSorter::new(
|
||||||
|
settings.buffer_size as u64,
|
||||||
|
Some(settings.tmp_dir.clone()),
|
||||||
|
settings.clone(),
|
||||||
|
);
|
||||||
let iter = external_sorter
|
let iter = external_sorter
|
||||||
.sort_by(unsorted.into_iter(), settings.clone())
|
.sort_by(unsorted.into_iter(), settings.clone())
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.map(|x| x.unwrap())
|
.map(|x| x.unwrap())
|
||||||
.collect::<Vec<Line>>();
|
.collect::<Vec<Line>>();
|
||||||
iter
|
iter
|
||||||
}
|
}
|
||||||
|
|
||||||
fn sort_by(lines: &mut Vec<Line>, settings: &GlobalSettings) {
|
fn sort_by(lines: &mut Vec<Line>, settings: &GlobalSettings) {
|
||||||
|
@ -1291,18 +1299,19 @@ fn compare_by(a: &Line, b: &Line, global_settings: &GlobalSettings) -> Ordering
|
||||||
(b_str, b_selection.num_cache.as_num_info()),
|
(b_str, b_selection.num_cache.as_num_info()),
|
||||||
),
|
),
|
||||||
// serde JSON has issues with f64 null values, so caching them won't work for us with ext sort
|
// serde JSON has issues with f64 null values, so caching them won't work for us with ext sort
|
||||||
SortMode::GeneralNumeric =>
|
SortMode::GeneralNumeric => {
|
||||||
if global_settings.buffer_size == DEFAULT_BUF_SIZE {
|
if global_settings.buffer_size == DEFAULT_BUF_SIZE {
|
||||||
general_numeric_compare(
|
general_numeric_compare(
|
||||||
a_selection.num_cache.as_f64(),
|
a_selection.num_cache.as_f64(),
|
||||||
b_selection.num_cache.as_f64()
|
b_selection.num_cache.as_f64(),
|
||||||
)
|
)
|
||||||
} else {
|
} else {
|
||||||
general_numeric_compare(
|
general_numeric_compare(
|
||||||
general_f64_parse(&a_str[get_leading_gen(a_str)]),
|
general_f64_parse(&a_str[get_leading_gen(a_str)]),
|
||||||
general_f64_parse(&b_str[get_leading_gen(b_str)])
|
general_f64_parse(&b_str[get_leading_gen(b_str)]),
|
||||||
)
|
)
|
||||||
},
|
}
|
||||||
|
}
|
||||||
SortMode::Month => month_compare(a_str, b_str),
|
SortMode::Month => month_compare(a_str, b_str),
|
||||||
SortMode::Version => version_compare(a_str, b_str),
|
SortMode::Version => version_compare(a_str, b_str),
|
||||||
SortMode::Default => default_compare(a_str, b_str),
|
SortMode::Default => default_compare(a_str, b_str),
|
||||||
|
|
|
@ -33,7 +33,8 @@ fn test_helper(file_name: &str, args: &str) {
|
||||||
fn test_larger_than_specified_segment() {
|
fn test_larger_than_specified_segment() {
|
||||||
new_ucmd!()
|
new_ucmd!()
|
||||||
.arg("-n")
|
.arg("-n")
|
||||||
.arg("-S 50M")
|
.arg("-S")
|
||||||
|
.arg("50K")
|
||||||
.arg("ext_sort.txt")
|
.arg("ext_sort.txt")
|
||||||
.succeeds()
|
.succeeds()
|
||||||
.stdout_is_fixture(format!("{}", "ext_sort.expected"));
|
.stdout_is_fixture(format!("{}", "ext_sort.expected"));
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue