fix(cksum): handle memory exhaustion gracefully for shake algorithms

Author: killuazoldyck680

src/uu/cksum/src/cksum.rs

@@ -52,7 +52,7 @@ fn maybe_sanitize_length(
     match (algo_cli, input_length) {
         // No provided length is not a problem so far.
         (_, None) => Ok(None),
-        
+
         // For SHA2 and SHA3, if a length is provided, ensure it is correct.
         (Some(algo @ (AlgoKind::Sha2 | AlgoKind::Sha3)), Some(s_len)) => {
             sanitize_sha2_sha3_length_str(algo, s_len).map(Some)
@@ -65,11 +65,16 @@ fn maybe_sanitize_length(
         (Some(AlgoKind::Shake128 | AlgoKind::Shake256), Some(len)) => match len.parse::<usize>() {
             Ok(0) => Ok(None),
             Ok(l) => {
-                if l > u32::MAX as usize {
-                    Err(ChecksumError::InvalidLength(len.into()).into()) 
-                } else {
-                    Ok(Some(HashLength::from_bits(l)))
+                let bytes_needed = (l + 7) / 8;
+
+                let mut test_buffer: Vec<u8> = Vec::new();
+                let safety_cushion = bytes_needed.saturating_add(65536);
+
+                if test_buffer.try_reserve(safety_cushion).is_err() {
+                    return Err(uucore::error::USimpleError::new(1, "memory exhausted"));
                 }
+
+                Ok(Some(HashLength::from_bits(l)))
             }
             Err(_) => Err(ChecksumError::InvalidLength(len.into()).into()),
         },
@@ -121,7 +126,6 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> {
     if matches.get_flag(options::DEBUG) {
         print_cpu_debug_info();
     }
-
     checksum_main(algo_cli, length, matches, output_format)
 }