From f1788d9e7058c3cb80d54277b3260755a6275965 Mon Sep 17 00:00:00 2001 From: nicoo Date: Tue, 16 Jun 2020 01:17:16 +0200 Subject: [PATCH] fixup! factor::numeric::Montgomery: Fix overflow bug --- src/uu/factor/src/numeric.rs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/uu/factor/src/numeric.rs b/src/uu/factor/src/numeric.rs index 0525e70b6..2d5d1ec21 100644 --- a/src/uu/factor/src/numeric.rs +++ b/src/uu/factor/src/numeric.rs @@ -80,16 +80,16 @@ impl Montgomery { let (xnm, overflow) = (x as u128).overflowing_add(nm); // x + n*m debug_assert_eq!(xnm % (1 << 64), 0); - if !overflow { - let y = (xnm >> 64) as u64 + overflow as u64; // (x + n*m) / R - if y >= *n { - y - n - } else { - y - } + // (x + n*m) / R + // in case of overflow, this is (2¹²⁸ + xnm)/2⁶⁴ - n = xnm/2⁶⁴ + (2⁶⁴ - n) + let y = + (xnm >> 64) as u64 + + if !overflow { 0 } else { n.wrapping_neg() }; + + if y >= *n { + y - n } else { - // y = (2¹²⁸ + xnm)/2⁶⁴ - n = xnm/2⁶⁴ + (2⁶⁴ - n) - (xnm >> 64) as u64 + n.wrapping_neg() + y } } }