RGBCube/uutils-coreutils
1
Fork 0
mirror of https://github.com/RGBCube/uutils-coreutils synced 2025-07-30 20:47:46 +00:00
Code Issues Activity

Add dynamic buffer adjustment, fix test comment

Browse source
This commit is contained in:
electricboogie 2021-04-25 15:13:27 -05:00
parent ab594b7b4c
commit 733949b2e7
3 changed files with 22 additions and 16 deletions
Download patch file Download diff file Expand all files Collapse all files

19
src/uu/sort/src/external_sort/mod.rs
View file

@ -174,13 +174,23 @@ where
{ {
let mut total_read = 0; let mut total_read = 0;
let mut chunk = Vec::new(); let mut chunk = Vec::new();
// Initial buffer is specified by user
let mut adjusted_buffer_size = self.buffer_bytes;
// make the initial chunks on disk // make the initial chunks on disk
for seq in unsorted { for seq in unsorted {
total_read += seq.get_size(); let seq_size = seq.get_size();
total_read += seq_size;
// Grow buffer size for a Line larger than buffer
adjusted_buffer_size =
if adjusted_buffer_size < seq_size {
seq_size
} else {
adjusted_buffer_size
};
chunk.push(seq); chunk.push(seq);
if total_read >= self.buffer_bytes { if total_read >= adjusted_buffer_size {
super::sort_by(&mut chunk, &self.settings); super::sort_by(&mut chunk, &self.settings);
self.write_chunk( self.write_chunk(
&iter.tmp_dir.path().join(iter.chunks.to_string()), &iter.tmp_dir.path().join(iter.chunks.to_string()),
@ -247,10 +257,7 @@ where
let line_s = line?; let line_s = line?;
bytes_read += line_s.len() + 1; bytes_read += line_s.len() + 1;
// This is where the bad stuff happens usually // This is where the bad stuff happens usually
let deserialized: Line = match serde_json::from_str(&line_s) { let deserialized: Line = serde_json::from_str(&line_s).expect("JSON read error: ");
Ok(x) => x,
Err(err) => panic!("JSON read error: {}", err),
};
total_read += deserialized.get_size(); total_read += deserialized.get_size();
vec.push_back(deserialized); vec.push_back(deserialized);
if total_read > max_bytes { if total_read > max_bytes {

13
src/uu/sort/src/sort.rs
View file

@ -141,13 +141,12 @@ impl GlobalSettings {
.expect("Error parsing buffer size: "); .expect("Error parsing buffer size: ");
let suf_usize: usize = match suf_str.to_uppercase().as_str() { let suf_usize: usize = match suf_str.to_uppercase().as_str() {
// SI Units // SI Units
"K" => 1000usize, "K" => 1024usize,
"M" => 1000000usize, "M" => 1024000usize,
"G" => 1000000000usize, "G" => 1024000000usize,
"T" => 1000000000000usize, "T" => 1024000000000usize,
"P" => 1000000000000000usize, // GNU regards empty human numeric value as 1024 bytes
"E" => 1000000000000000000usize, _ => 1024usize,
_ => 1usize,
}; };
num_usize * suf_usize num_usize * suf_usize
} }

6
tests/by-util/test_sort.rs
View file

@ -59,13 +59,13 @@ fn test_human_numeric_whitespace() {
test_helper("human-numeric-whitespace", "-h"); test_helper("human-numeric-whitespace", "-h");
} }
// This tests the ext sort feature, but it also tests where // This tests where serde often fails when reading back JSON
// serde might fail when reading back JSON if it finds a null value // if it finds a null value
#[test] #[test]
fn test_extsort_as64_bailout() { fn test_extsort_as64_bailout() {
new_ucmd!() new_ucmd!()
.arg("-g") .arg("-g")
.arg("-S 10K") .arg("-S 5K")
.arg("multiple_decimals_general.txt") .arg("multiple_decimals_general.txt")
.succeeds() .succeeds()
.stdout_is_fixture("multiple_decimals_general.expected"); .stdout_is_fixture("multiple_decimals_general.expected");