1
Fork 0
mirror of https://github.com/RGBCube/uutils-coreutils synced 2025-08-01 13:37:48 +00:00

Set a dynamic minimum buffer size

This commit is contained in:
electricboogie 2021-04-25 21:17:56 -05:00
parent 8e258075f6
commit 1a407c2328

View file

@ -176,15 +176,28 @@ where
let mut chunk = Vec::new(); let mut chunk = Vec::new();
// Initial buffer is specified by user // Initial buffer is specified by user
let mut adjusted_buffer_size = self.buffer_bytes; let mut adjusted_buffer_size = self.buffer_bytes;
let (iter_size, _) = unsorted.size_hint();
// make the initial chunks on disk // make the initial chunks on disk
for seq in unsorted { for seq in unsorted {
let seq_size = seq.get_size(); let seq_size = seq.get_size();
total_read += seq_size; total_read += seq_size;
// Grow buffer size for a struct/Line larger than buffer
// GNU minimum is 16 * (sizeof struct + 2), but GNU uses about
// 1/10 the memory that we do. And GNU even says in the code it may
// not work on small buffer sizes.
//
// The following seems to work pretty well, and has about the same max
// RSS as lower minimum values.
//
let minimum_buffer_size: u64 = iter_size as u64 * seq_size / 8;
adjusted_buffer_size = adjusted_buffer_size =
// Grow buffer size for a struct/Line larger than buffer
if adjusted_buffer_size < seq_size { if adjusted_buffer_size < seq_size {
seq_size seq_size
} else if adjusted_buffer_size < minimum_buffer_size {
minimum_buffer_size
} else { } else {
adjusted_buffer_size adjusted_buffer_size
}; };