Cleanup, removed unused code, add copyright

src/uu/sort/src/ext_sorter.rs

@@ -1,4 +1,5 @@
 // Copyright 2018 Andre-Philippe Paquet
+// Copyright 2021 Robert Swinford
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -38,7 +39,7 @@ pub struct ExternalSorter {
 impl ExternalSorter {
     pub fn new() -> ExternalSorter {
         ExternalSorter {
-            segment_size: 10000000,
+            segment_size: 16000000000,
             sort_dir: None,
             parallel: false,
         }
@@ -73,18 +74,6 @@ impl ExternalSorter {
         self
     }
 
-    /// Sorts a given iterator, returning a new iterator with items
-    pub fn sort<T, I>(
-        &self,
-        iterator: I,
-    ) -> Result<SortedIterator<T, impl Fn(&T, &T) -> Ordering + Send + Sync>, Error>
-    where
-        T: Sortable + Ord,
-        I: Iterator<Item = T>,
-    {
-        self.sort_by(iterator, |a, b| a.cmp(b))
-    }
-
     /// Sorts a given iterator with a comparator function, returning a new iterator with items
     pub fn sort_by<T, I, F>(&self, iterator: I, cmp: F) -> Result<SortedIterator<T, F>, Error>
     where
@@ -122,21 +111,6 @@ impl ExternalSorter {
         SortedIterator::new(tempdir, pass_through_queue, segments_file, count, cmp)
     }
 
-    /// Sorts a given iterator with a key extraction function, returning a new iterator with items
-    pub fn sort_by_key<T, I, F, K>(
-        &self,
-        iterator: I,
-        f: F,
-    ) -> Result<SortedIterator<T, impl Fn(&T, &T) -> Ordering + Send + Sync>, Error>
-    where
-        T: Sortable,
-        I: Iterator<Item = T>,
-        F: Fn(&T) -> K + Send + Sync,
-        K: Ord,
-    {
-        self.sort_by(iterator, move |a, b| f(a).cmp(&f(b)))
-    }
-
     /// We only want to create directory if it's needed (i.e. if the dataset
     /// doesn't fit in memory) to prevent filesystem latency
     fn lazy_create_dir<'a>(
@@ -243,10 +217,6 @@ impl<T: Sortable, F: Fn(&T, &T) -> Ordering + Send + Sync> SortedIterator<T, F>
             cmp,
         })
     }
-
-    pub fn sorted_count(&self) -> u64 {
-        self.count
-    }
 }
 
 impl<T: Sortable, F: Fn(&T, &T) -> Ordering> Iterator for SortedIterator<T, F> {
@@ -285,63 +255,3 @@ impl<T: Sortable, F: Fn(&T, &T) -> Ordering> Iterator for SortedIterator<T, F> {
         })
     }
 }
-
-#[cfg(test)]
-pub mod test {
-    use super::*;
-
-    use byteorder::{ReadBytesExt, WriteBytesExt};
-
-    #[test]
-    fn test_smaller_than_segment() {
-        let sorter = ExternalSorter::new();
-        let data: Vec<u32> = (0..100u32).collect();
-        let data_rev: Vec<u32> = data.iter().rev().cloned().collect();
-
-        let sorted_iter = sorter.sort(data_rev.into_iter()).unwrap();
-
-        // should not have used any segments (all in memory)
-        assert_eq!(sorted_iter.segments_file.len(), 0);
-        let sorted_data: Vec<u32> = sorted_iter.collect();
-
-        assert_eq!(data, sorted_data);
-    }
-
-    #[test]
-    fn test_multiple_segments() {
-        let sorter = ExternalSorter::new().with_segment_size(100);
-        let data: Vec<u32> = (0..1000u32).collect();
-
-        let data_rev: Vec<u32> = data.iter().rev().cloned().collect();
-        let sorted_iter = sorter.sort(data_rev.into_iter()).unwrap();
-        assert_eq!(sorted_iter.segments_file.len(), 10);
-
-        let sorted_data: Vec<u32> = sorted_iter.collect();
-        assert_eq!(data, sorted_data);
-    }
-
-    #[test]
-    fn test_parallel() {
-        let sorter = ExternalSorter::new()
-            .with_segment_size(100)
-            .with_parallel_sort();
-        let data: Vec<u32> = (0..1000u32).collect();
-
-        let data_rev: Vec<u32> = data.iter().rev().cloned().collect();
-        let sorted_iter = sorter.sort(data_rev.into_iter()).unwrap();
-        assert_eq!(sorted_iter.segments_file.len(), 10);
-
-        let sorted_data: Vec<u32> = sorted_iter.collect();
-        assert_eq!(data, sorted_data);
-    }
-
-    impl Sortable for u32 {
-        fn encode<W: Write>(&self, writer: &mut W) {
-            writer.write_u32::<byteorder::LittleEndian>(*self).unwrap();
-        }
-
-        fn decode<R: Read>(reader: &mut R) -> Option<u32> {
-            reader.read_u32::<byteorder::LittleEndian>().ok()
-        }
-    }
-}

src/uu/sort/src/sort.rs

@@ -15,14 +15,14 @@
 #[macro_use]
 extern crate uucore;
 
+mod ext_sorter;
 mod numeric_str_cmp;
-pub mod ext_sorter;
-pub use ext_sorter::{ExternalSorter, Sortable, SortedIterator};
 
 use clap::{App, Arg};
 use fnv::FnvHasher;
 use itertools::Itertools;
 use numeric_str_cmp::{numeric_str_cmp, NumInfo, NumInfoParseSettings};
+use ext_sorter::{ExternalSorter, Sortable};
 use rand::distributions::Alphanumeric;
 use rand::{thread_rng, Rng};
 use semver::Version;
@@ -39,7 +39,7 @@ use std::io::{stdin, stdout, BufRead, BufReader, BufWriter, Lines, Read, Write};
 use std::mem::replace;
 use std::ops::{Range, RangeInclusive};
 use std::path::{Path, PathBuf};
-use uucore::fs::is_stdin_interactive; // for Iterator::dedup()
+use uucore::fs::is_stdin_interactive; // for Iterator::dedup();
 
 static NAME: &str = "sort";
 static ABOUT: &str = "Display sorted concatenation of all FILE(s).";
@@ -90,7 +90,8 @@ static NEGATIVE: char = '-';
 static POSITIVE: char = '+';
 
 static DEFAULT_TMPDIR: &str = r"/tmp";
-static DEFAULT_BUF_SIZE: usize = 10000000usize;
+// 16GB buffer for Vec<Lines> before we dump to disk
+static DEFAULT_BUF_SIZE: usize = 16000000000;
 
 #[derive(Eq, Ord, PartialEq, PartialOrd, Clone)]
 enum SortMode {
@@ -281,7 +282,7 @@ impl Sortable for Line {
                 selections_joined.append(&mut deserialized_line.selections);
             }
             Some(Line {
-                line: line_joined.strip_suffix("\n").unwrap().to_owned(),
+                line: line_joined.strip_suffix("\n").unwrap_or("").to_owned(),
                 selections: selections_joined,
             })
         };
@@ -881,7 +882,7 @@ pub fn uumain(args: impl uucore::Args) -> i32 {
     }
 
     if matches.is_present(OPT_BUF_SIZE) {
-        // 10K is the default extsort buffer, but that's too small, so we set at 10M
+        // 10K is the default extsort buffer, but that's too small, so we set at 100M
         // Although the "default" is never used unless extsort options are given
         settings.buffer_size = {
             let input = matches
@@ -889,7 +890,11 @@ pub fn uumain(args: impl uucore::Args) -> i32 {
                 .map(String::from)
                 .unwrap_or(format!("{}", DEFAULT_BUF_SIZE));
 
-            human_numeric_convert(&input)
+            if human_numeric_convert(&input) < 128000 {
+                panic!("sort will not operate with less than 128K of memory.");
+            } else {
+                human_numeric_convert(&input)
+            }
         }
     }
 
@@ -1030,7 +1035,6 @@ fn exec(files: Vec<String>, settings: &GlobalSettings) -> i32 {
         return exec_check_file(&lines, &settings);
     }
 
-
     lines = sort_by(lines, &settings);
 
     if settings.merge {