From da94e350448239ca6ddd6442d76e4cd610dd6d98 Mon Sep 17 00:00:00 2001 From: electricboogie <32370782+electricboogie@users.noreply.github.com> Date: Sun, 18 Apr 2021 13:02:50 -0500 Subject: [PATCH] Cleanup, removed unused code, add copyright --- src/uu/sort/src/ext_sorter.rs | 94 +---------------------------------- src/uu/sort/src/sort.rs | 20 +++++--- 2 files changed, 14 insertions(+), 100 deletions(-) diff --git a/src/uu/sort/src/ext_sorter.rs b/src/uu/sort/src/ext_sorter.rs index c19f1262b..d607cbd3e 100644 --- a/src/uu/sort/src/ext_sorter.rs +++ b/src/uu/sort/src/ext_sorter.rs @@ -1,4 +1,5 @@ // Copyright 2018 Andre-Philippe Paquet +// Copyright 2021 Robert Swinford // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -38,7 +39,7 @@ pub struct ExternalSorter { impl ExternalSorter { pub fn new() -> ExternalSorter { ExternalSorter { - segment_size: 10000000, + segment_size: 16000000000, sort_dir: None, parallel: false, } @@ -73,18 +74,6 @@ impl ExternalSorter { self } - /// Sorts a given iterator, returning a new iterator with items - pub fn sort( - &self, - iterator: I, - ) -> Result Ordering + Send + Sync>, Error> - where - T: Sortable + Ord, - I: Iterator, - { - self.sort_by(iterator, |a, b| a.cmp(b)) - } - /// Sorts a given iterator with a comparator function, returning a new iterator with items pub fn sort_by(&self, iterator: I, cmp: F) -> Result, Error> where @@ -122,21 +111,6 @@ impl ExternalSorter { SortedIterator::new(tempdir, pass_through_queue, segments_file, count, cmp) } - /// Sorts a given iterator with a key extraction function, returning a new iterator with items - pub fn sort_by_key( - &self, - iterator: I, - f: F, - ) -> Result Ordering + Send + Sync>, Error> - where - T: Sortable, - I: Iterator, - F: Fn(&T) -> K + Send + Sync, - K: Ord, - { - self.sort_by(iterator, move |a, b| f(a).cmp(&f(b))) - } - /// We only want to create directory if it's needed (i.e. if the dataset /// doesn't fit in memory) to prevent filesystem latency fn lazy_create_dir<'a>( @@ -243,10 +217,6 @@ impl Ordering + Send + Sync> SortedIterator cmp, }) } - - pub fn sorted_count(&self) -> u64 { - self.count - } } impl Ordering> Iterator for SortedIterator { @@ -285,63 +255,3 @@ impl Ordering> Iterator for SortedIterator { }) } } - -#[cfg(test)] -pub mod test { - use super::*; - - use byteorder::{ReadBytesExt, WriteBytesExt}; - - #[test] - fn test_smaller_than_segment() { - let sorter = ExternalSorter::new(); - let data: Vec = (0..100u32).collect(); - let data_rev: Vec = data.iter().rev().cloned().collect(); - - let sorted_iter = sorter.sort(data_rev.into_iter()).unwrap(); - - // should not have used any segments (all in memory) - assert_eq!(sorted_iter.segments_file.len(), 0); - let sorted_data: Vec = sorted_iter.collect(); - - assert_eq!(data, sorted_data); - } - - #[test] - fn test_multiple_segments() { - let sorter = ExternalSorter::new().with_segment_size(100); - let data: Vec = (0..1000u32).collect(); - - let data_rev: Vec = data.iter().rev().cloned().collect(); - let sorted_iter = sorter.sort(data_rev.into_iter()).unwrap(); - assert_eq!(sorted_iter.segments_file.len(), 10); - - let sorted_data: Vec = sorted_iter.collect(); - assert_eq!(data, sorted_data); - } - - #[test] - fn test_parallel() { - let sorter = ExternalSorter::new() - .with_segment_size(100) - .with_parallel_sort(); - let data: Vec = (0..1000u32).collect(); - - let data_rev: Vec = data.iter().rev().cloned().collect(); - let sorted_iter = sorter.sort(data_rev.into_iter()).unwrap(); - assert_eq!(sorted_iter.segments_file.len(), 10); - - let sorted_data: Vec = sorted_iter.collect(); - assert_eq!(data, sorted_data); - } - - impl Sortable for u32 { - fn encode(&self, writer: &mut W) { - writer.write_u32::(*self).unwrap(); - } - - fn decode(reader: &mut R) -> Option { - reader.read_u32::().ok() - } - } -} diff --git a/src/uu/sort/src/sort.rs b/src/uu/sort/src/sort.rs index 07a8879b7..fab712978 100644 --- a/src/uu/sort/src/sort.rs +++ b/src/uu/sort/src/sort.rs @@ -15,14 +15,14 @@ #[macro_use] extern crate uucore; +mod ext_sorter; mod numeric_str_cmp; -pub mod ext_sorter; -pub use ext_sorter::{ExternalSorter, Sortable, SortedIterator}; use clap::{App, Arg}; use fnv::FnvHasher; use itertools::Itertools; use numeric_str_cmp::{numeric_str_cmp, NumInfo, NumInfoParseSettings}; +use ext_sorter::{ExternalSorter, Sortable}; use rand::distributions::Alphanumeric; use rand::{thread_rng, Rng}; use semver::Version; @@ -39,7 +39,7 @@ use std::io::{stdin, stdout, BufRead, BufReader, BufWriter, Lines, Read, Write}; use std::mem::replace; use std::ops::{Range, RangeInclusive}; use std::path::{Path, PathBuf}; -use uucore::fs::is_stdin_interactive; // for Iterator::dedup() +use uucore::fs::is_stdin_interactive; // for Iterator::dedup(); static NAME: &str = "sort"; static ABOUT: &str = "Display sorted concatenation of all FILE(s)."; @@ -90,7 +90,8 @@ static NEGATIVE: char = '-'; static POSITIVE: char = '+'; static DEFAULT_TMPDIR: &str = r"/tmp"; -static DEFAULT_BUF_SIZE: usize = 10000000usize; +// 16GB buffer for Vec before we dump to disk +static DEFAULT_BUF_SIZE: usize = 16000000000; #[derive(Eq, Ord, PartialEq, PartialOrd, Clone)] enum SortMode { @@ -281,7 +282,7 @@ impl Sortable for Line { selections_joined.append(&mut deserialized_line.selections); } Some(Line { - line: line_joined.strip_suffix("\n").unwrap().to_owned(), + line: line_joined.strip_suffix("\n").unwrap_or("").to_owned(), selections: selections_joined, }) }; @@ -881,7 +882,7 @@ pub fn uumain(args: impl uucore::Args) -> i32 { } if matches.is_present(OPT_BUF_SIZE) { - // 10K is the default extsort buffer, but that's too small, so we set at 10M + // 10K is the default extsort buffer, but that's too small, so we set at 100M // Although the "default" is never used unless extsort options are given settings.buffer_size = { let input = matches @@ -889,7 +890,11 @@ pub fn uumain(args: impl uucore::Args) -> i32 { .map(String::from) .unwrap_or(format!("{}", DEFAULT_BUF_SIZE)); - human_numeric_convert(&input) + if human_numeric_convert(&input) < 128000 { + panic!("sort will not operate with less than 128K of memory."); + } else { + human_numeric_convert(&input) + } } } @@ -1030,7 +1035,6 @@ fn exec(files: Vec, settings: &GlobalSettings) -> i32 { return exec_check_file(&lines, &settings); } - lines = sort_by(lines, &settings); if settings.merge {