RGBCube/serenity
1
Fork 0
mirror of https://github.com/RGBCube/serenity synced 2025-07-28 06:17:34 +00:00
Code Issues Activity Actions

LibCrypto: Remove all uses of VLAs

Browse source 
This removes all uses of VLAs with either Vectors with inline capacity
for the expected soft upper bound, or the occasional heap allocation.
This commit is contained in:
Ali Mohammad Pur 2021-05-13 12:13:11 +04:30 • committed by Andreas Kling
parent 0d50d3ed1e
commit b05beb79d4
7 changed files with 41 additions and 36 deletions
Download patch file Download diff file Expand all files Collapse all files

26
Userland/Libraries/LibCrypto/PK/Code/EMSA_PSS.h
View file

@ -33,7 +33,7 @@ public:
auto& hash_fn = this->hasher();
hash_fn.update(in);
auto message_hash = hash_fn.digest();
auto hash_length = hash_fn.DigestSize;
constexpr auto hash_length = hash_fn.DigestSize;
auto em_length = (em_bits + 7) / 8;
u8 salt[SaltLength];
@ -51,8 +51,9 @@ public:
hash_fn.update(m_buffer);
auto hash = hash_fn.digest();
u8 DB_data[em_length - HashFunction::DigestSize - 1];
auto DB = Bytes { DB_data, em_length - HashFunction::DigestSize - 1 };
Vector<u8, 256> DB_data;
DB_data.resize(em_length - HashFunction::DigestSize - 1);
Bytes DB = DB_data;
auto DB_offset = 0;
for (size_t i = 0; i < em_length - SaltLength - HashFunction::DigestSize - 2; ++i)
@ -64,8 +65,9 @@ public:
auto mask_length = em_length - HashFunction::DigestSize - 1;
u8 DB_mask[mask_length];
auto DB_mask_buffer = Bytes { DB_mask, mask_length };
Vector<u8, 256> DB_mask;
DB_mask.resize(mask_length);
Bytes DB_mask_buffer { DB_mask };
// FIXME: we should probably allow reading from u8*
MGF1(ReadonlyBytes { hash.data, HashFunction::DigestSize }, mask_length, DB_mask_buffer);
@ -102,11 +104,13 @@ public:
if ((octet >> (8 - i)) & 0x01)
return VerificationConsistency::Inconsistent;
u8 DB_mask[mask_length];
auto DB_mask_buffer = Bytes { DB_mask, mask_length };
Vector<u8, 256> DB_mask;
DB_mask.resize(mask_length);
Bytes DB_mask_buffer { DB_mask };
MGF1(H, mask_length, DB_mask_buffer);
u8 DB[mask_length];
Vector<u8, 256> DB;
DB.resize(mask_length);
for (size_t i = 0; i < mask_length; ++i)
DB[i] = masked_DB[i] ^ DB_mask[i];
@ -122,8 +126,8 @@ public:
if (DB[check_octets + 1] != 0x01)
return VerificationConsistency::Inconsistent;
auto* salt = DB + mask_length - SaltLength;
u8 m_prime[8 + HashFunction::DigestSize + SaltLength] { 0, 0, 0, 0, 0, 0, 0, 0 };
auto* salt = DB.span().offset(mask_length - SaltLength);
u8 m_prime[8 + HashFunction::DigestSize + SaltLength] { 0 };
auto m_prime_buffer = Bytes { m_prime, sizeof(m_prime) };
@ -133,7 +137,7 @@ public:
hash_fn.update(m_prime_buffer);
auto H_prime = hash_fn.digest();
if (__builtin_memcmp(message_hash.data, H_prime.data, HashFunction::DigestSize))
if (__builtin_memcmp(message_hash.data, H_prime.data, HashFunction::DigestSize) != 0)
return VerificationConsistency::Inconsistent;
return VerificationConsistency::Consistent;

22
Userland/Libraries/LibCrypto/PK/RSA.cpp
View file

@ -293,8 +293,9 @@ void RSA_EMSA_PSS<HashFunction>::sign(ReadonlyBytes in, Bytes& out)
// -- encode via EMSA_PSS
auto mod_bits = m_rsa.private_key().modulus().trimmed_length() * sizeof(u32) * 8;
u8 EM[mod_bits];
auto EM_buf = Bytes { EM, mod_bits };
Vector<u8, 2048> EM;
EM.resize(mod_bits);
auto EM_buf = Bytes { EM };
m_emsa_pss.encode(in, EM_buf, mod_bits - 1);
// -- sign via RSA
@ -308,8 +309,9 @@ VerificationConsistency RSA_EMSA_PSS<HashFunction>::verify(ReadonlyBytes in)
if (in.size() != mod_bytes)
return VerificationConsistency::Inconsistent;
u8 EM[mod_bytes];
auto EM_buf = Bytes { EM, mod_bytes };
Vector<u8, 256> EM;
EM.resize(mod_bytes);
auto EM_buf = Bytes { EM };
// -- verify via RSA
m_rsa.verify(in, EM_buf);
@ -333,22 +335,20 @@ void RSA_PKCS1_EME::encrypt(ReadonlyBytes in, Bytes& out)
}
auto ps_length = mod_len - in.size() - 3;
u8 ps[ps_length];
Vector<u8, 8096> ps;
ps.resize(ps_length);
// FIXME: Without this assertion, GCC refuses to compile due to a memcpy overflow(!?)
VERIFY(ps_length < 16384);
fill_with_random(ps, ps_length);
fill_with_random(ps.data(), ps_length);
// since arc4random can create zeros (shocking!)
// we have to go through and un-zero the zeros
for (size_t i = 0; i < ps_length; ++i)
while (!ps[i])
fill_with_random(ps + i, 1);
fill_with_random(ps.span().offset(i), 1);
u8 paddings[] { 0x00, 0x02 };
out.overwrite(0, paddings, 2);
out.overwrite(2, ps, ps_length);
out.overwrite(2, ps.data(), ps_length);
out.overwrite(2 + ps_length, paddings, 1);
out.overwrite(3 + ps_length, in.data(), in.size());
out = out.trim(3 + ps_length + in.size()); // should be a single block