Merge pull request #5517 from sylvestre/fuzzing-improv

fuzzing: Refactor the error management

fuzz/fuzz_targets/fuzz_common.rs

@@ -3,13 +3,28 @@
 // For the full copyright and license information, please view the LICENSE
 // file that was distributed with this source code.
 
-use libc::{dup, dup2, STDOUT_FILENO};
+use libc::{close, dup, dup2, pipe, STDERR_FILENO, STDOUT_FILENO};
 use std::ffi::OsString;
 use std::io;
+use std::io::Write;
+use std::os::fd::RawFd;
 use std::process::Command;
 use std::sync::atomic::Ordering;
 use std::sync::{atomic::AtomicBool, Once};
 
+/// Represents the result of running a command, including its standard output,
+/// standard error, and exit code.
+pub struct CommandResult {
+    /// The standard output (stdout) of the command as a string.
+    pub stdout: String,
+
+    /// The standard error (stderr) of the command as a string.
+    pub stderr: String,
+
+    /// The exit code of the command.
+    pub exit_code: i32,
+}
+
 static CHECK_GNU: Once = Once::new();
 static IS_GNU: AtomicBool = AtomicBool::new(false);
 
@@ -32,58 +47,135 @@ pub fn is_gnu_cmd(cmd_path: &str) -> Result<(), std::io::Error> {
     }
 }
 
-pub fn generate_and_run_uumain<F>(args: &[OsString], uumain_function: F) -> (String, i32)
+pub fn generate_and_run_uumain<F>(args: &[OsString], uumain_function: F) -> CommandResult
 where
     F: FnOnce(std::vec::IntoIter<OsString>) -> i32,
 {
-    let uumain_exit_status;
-
+    // Duplicate the stdout and stderr file descriptors
     let original_stdout_fd = unsafe { dup(STDOUT_FILENO) };
-    println!("Running test {:?}", &args[1..]);
-    let mut pipe_fds = [-1; 2];
-    unsafe { libc::pipe(pipe_fds.as_mut_ptr()) };
-
-    {
-        unsafe { dup2(pipe_fds[1], STDOUT_FILENO) };
-        uumain_exit_status = uumain_function(args.to_owned().into_iter());
-        unsafe { dup2(original_stdout_fd, STDOUT_FILENO) };
-        unsafe { libc::close(original_stdout_fd) };
+    let original_stderr_fd = unsafe { dup(STDERR_FILENO) };
+    if original_stdout_fd == -1 || original_stderr_fd == -1 {
+        return CommandResult {
+            stdout: "Failed to duplicate STDOUT_FILENO or STDERR_FILENO".to_string(),
+            stderr: "".to_string(),
+            exit_code: -1,
+        };
     }
-    unsafe { libc::close(pipe_fds[1]) };
+    println!("Running test {:?}", &args[0..]);
+    let mut pipe_stdout_fds = [-1; 2];
+    let mut pipe_stderr_fds = [-1; 2];
 
+    // Create pipes for stdout and stderr
+    if unsafe { pipe(pipe_stdout_fds.as_mut_ptr()) } == -1
+        || unsafe { pipe(pipe_stderr_fds.as_mut_ptr()) } == -1
+    {
+        return CommandResult {
+            stdout: "Failed to create pipes".to_string(),
+            stderr: "".to_string(),
+            exit_code: -1,
+        };
+    }
+
+    // Redirect stdout and stderr to their respective pipes
+    if unsafe { dup2(pipe_stdout_fds[1], STDOUT_FILENO) } == -1
+        || unsafe { dup2(pipe_stderr_fds[1], STDERR_FILENO) } == -1
+    {
+        unsafe {
+            close(pipe_stdout_fds[0]);
+            close(pipe_stdout_fds[1]);
+            close(pipe_stderr_fds[0]);
+            close(pipe_stderr_fds[1]);
+        }
+        return CommandResult {
+            stdout: "Failed to redirect STDOUT_FILENO or STDERR_FILENO".to_string(),
+            stderr: "".to_string(),
+            exit_code: -1,
+        };
+    }
+
+    let uumain_exit_status = uumain_function(args.to_owned().into_iter());
+
+    io::stdout().flush().unwrap();
+    io::stderr().flush().unwrap();
+
+    // Restore the original stdout and stderr
+    if unsafe { dup2(original_stdout_fd, STDOUT_FILENO) } == -1
+        || unsafe { dup2(original_stderr_fd, STDERR_FILENO) } == -1
+    {
+        return CommandResult {
+            stdout: "Failed to restore the original STDOUT_FILENO or STDERR_FILENO".to_string(),
+            stderr: "".to_string(),
+            exit_code: -1,
+        };
+    }
+    unsafe {
+        close(original_stdout_fd);
+        close(original_stderr_fd);
+
+        close(pipe_stdout_fds[1]);
+        close(pipe_stderr_fds[1]);
+    }
+
+    let captured_stdout = read_from_fd(pipe_stdout_fds[0]).trim().to_string();
+    let captured_stderr = read_from_fd(pipe_stderr_fds[0]).to_string();
+    let captured_stderr = captured_stderr
+        .split_once(':')
+        .map(|x| x.1)
+        .unwrap_or("")
+        .trim()
+        .to_string();
+
+    CommandResult {
+        stdout: captured_stdout,
+        stderr: captured_stderr,
+        exit_code: uumain_exit_status,
+    }
+}
+
+fn read_from_fd(fd: RawFd) -> String {
     let mut captured_output = Vec::new();
     let mut read_buffer = [0; 1024];
     loop {
         let bytes_read = unsafe {
             libc::read(
-                pipe_fds[0],
+                fd,
                 read_buffer.as_mut_ptr() as *mut libc::c_void,
                 read_buffer.len(),
             )
         };
-        if bytes_read <= 0 {
+
+        if bytes_read == -1 {
+            eprintln!("Failed to read from the pipe");
+            break;
+        }
+        if bytes_read == 0 {
             break;
         }
         captured_output.extend_from_slice(&read_buffer[..bytes_read as usize]);
     }
 
-    unsafe { libc::close(pipe_fds[0]) };
+    unsafe { libc::close(fd) };
 
-    let my_output = String::from_utf8_lossy(&captured_output)
-        .to_string()
-        .trim()
-        .to_owned();
-
-    (my_output, uumain_exit_status)
+    String::from_utf8_lossy(&captured_output).into_owned()
 }
 
 pub fn run_gnu_cmd(
     cmd_path: &str,
     args: &[OsString],
     check_gnu: bool,
-) -> Result<(String, i32), io::Error> {
+) -> Result<CommandResult, CommandResult> {
     if check_gnu {
-        is_gnu_cmd(cmd_path)?; // Check if it's a GNU implementation
+        match is_gnu_cmd(cmd_path) {
+            Ok(_) => {} // if the check passes, do nothing
+            Err(e) => {
+                // Convert the io::Error into the function's error type
+                return Err(CommandResult {
+                    stdout: String::new(),
+                    stderr: e.to_string(),
+                    exit_code: -1,
+                });
+            }
+        }
     }
 
     let mut command = Command::new(cmd_path);
@@ -91,17 +183,92 @@ pub fn run_gnu_cmd(
         command.arg(arg);
     }
 
-    let output = command.output()?;
+    let output = match command.output() {
+        Ok(output) => output,
+        Err(e) => {
+            return Err(CommandResult {
+                stdout: String::new(),
+                stderr: e.to_string(),
+                exit_code: -1,
+            });
+        }
+    };
     let exit_code = output.status.code().unwrap_or(-1);
+
+    // Here we get stdout and stderr as Strings
+    let stdout = String::from_utf8_lossy(&output.stdout).to_string();
+    let stderr = String::from_utf8_lossy(&output.stderr).to_string();
+    let stderr = stderr
+        .split_once(':')
+        .map(|x| x.1)
+        .unwrap_or("")
+        .trim()
+        .to_string();
+
     if output.status.success() || !check_gnu {
-        Ok((
-            String::from_utf8_lossy(&output.stdout).to_string(),
+        Ok(CommandResult {
+            stdout,
+            stderr,
             exit_code,
-        ))
+        })
     } else {
-        Err(io::Error::new(
-            io::ErrorKind::Other,
-            format!("GNU command execution failed with exit code {}", exit_code),
-        ))
+        Err(CommandResult {
+            stdout,
+            stderr,
+            exit_code,
+        })
+    }
+}
+
+pub fn compare_result(
+    test_type: &str,
+    input: &str,
+    rust_stdout: &str,
+    gnu_stdout: &str,
+    rust_stderr: &str,
+    gnu_stderr: &str,
+    rust_exit_code: i32,
+    gnu_exit_code: i32,
+    fail_on_stderr_diff: bool,
+) {
+    println!("Test Type: {}", test_type);
+    println!("Input: {}", input);
+
+    let mut discrepancies = Vec::new();
+    let mut should_panic = false;
+
+    if rust_stdout.trim() != gnu_stdout.trim() {
+        discrepancies.push("stdout differs");
+        println!("Rust stdout: {}", rust_stdout);
+        println!("GNU stdout: {}", gnu_stdout);
+        should_panic = true;
+    }
+    if rust_stderr.trim() != gnu_stderr.trim() {
+        discrepancies.push("stderr differs");
+        println!("Rust stderr: {}", rust_stderr);
+        println!("GNU stderr: {}", gnu_stderr);
+        if fail_on_stderr_diff {
+            should_panic = true;
+        }
+    }
+    if rust_exit_code != gnu_exit_code {
+        discrepancies.push("exit code differs");
+        println!("Rust exit code: {}", rust_exit_code);
+        println!("GNU exit code: {}", gnu_exit_code);
+        should_panic = true;
+    }
+
+    if discrepancies.is_empty() {
+        println!("All outputs and exit codes matched.");
+    } else {
+        println!("Discrepancy detected: {}", discrepancies.join(", "));
+        if should_panic {
+            panic!("Test failed for {}: {}", test_type, input);
+        } else {
+            println!(
+                "Test completed with discrepancies for {}: {}",
+                test_type, input
+            );
+        }
     }
 }

fuzz/fuzz_targets/fuzz_expr.rs

@@ -13,8 +13,8 @@ use rand::Rng;
 use std::{env, ffi::OsString};
 
 mod fuzz_common;
-use crate::fuzz_common::{generate_and_run_uumain, run_gnu_cmd};
-
+use crate::fuzz_common::CommandResult;
+use crate::fuzz_common::{compare_result, generate_and_run_uumain, run_gnu_cmd};
 static CMD_PATH: &str = "expr";
 
 fn generate_random_string(max_length: usize) -> String {
@@ -84,37 +84,35 @@ fuzz_target!(|_data: &[u8]| {
     let mut args = vec![OsString::from("expr")];
     args.extend(expr.split_whitespace().map(OsString::from));
 
-    let (rust_output, uumain_exit_code) = generate_and_run_uumain(&args, uumain);
-
     // Use C locale to avoid false positives, like in https://github.com/uutils/coreutils/issues/5378,
     // because uutils expr doesn't support localization yet
     // TODO remove once uutils expr supports localization
     env::set_var("LC_COLLATE", "C");
+    let rust_result = generate_and_run_uumain(&args, uumain);
 
-    // Run GNU expr with the provided arguments and compare the output
-    match run_gnu_cmd(CMD_PATH, &args[1..], true) {
-        Ok((gnu_output, gnu_exit_code)) => {
-            let gnu_output = gnu_output.trim().to_owned();
-            if uumain_exit_code != gnu_exit_code {
-                println!("Expression: {}", expr);
-                println!("Rust code: {}", uumain_exit_code);
-                println!("GNU code: {}", gnu_exit_code);
-                panic!("Different error codes");
+    let gnu_result = match run_gnu_cmd(CMD_PATH, &args[1..], false) {
+        Ok(result) => result,
+        Err(error_result) => {
+            eprintln!("Failed to run GNU command:");
+            eprintln!("Stderr: {}", error_result.stderr);
+            eprintln!("Exit Code: {}", error_result.exit_code);
+            CommandResult {
+                stdout: String::new(),
+                stderr: error_result.stderr,
+                exit_code: error_result.exit_code,
             }
-            if rust_output == gnu_output {
-                println!(
-                    "Outputs matched for expression: {} => Result: {}",
-                    expr, rust_output
+        }
+    };
+
+    compare_result(
+        "expr",
+        &format!("{:?}", &args[1..]),
+        &rust_result.stdout,
+        &gnu_result.stdout,
+        &rust_result.stderr,
+        &gnu_result.stderr,
+        rust_result.exit_code,
+        gnu_result.exit_code,
+        false, // Set to true if you want to fail on stderr diff
     );
-            } else {
-                println!("Expression: {}", expr);
-                println!("Rust output: {}", rust_output);
-                println!("GNU output: {}", gnu_output);
-                panic!("Different output between Rust & GNU");
-            }
-        }
-        Err(_) => {
-            println!("GNU expr execution failed for expression: {}", expr);
-        }
-    }
 });

fuzz/fuzz_targets/fuzz_test.rs

@@ -13,7 +13,8 @@ use rand::Rng;
 use std::ffi::OsString;
 
 mod fuzz_common;
-use crate::fuzz_common::{generate_and_run_uumain, run_gnu_cmd};
+use crate::fuzz_common::CommandResult;
+use crate::fuzz_common::{compare_result, generate_and_run_uumain, run_gnu_cmd};
 
 #[derive(PartialEq, Debug, Clone)]
 enum ArgType {
@@ -204,31 +205,31 @@ fuzz_target!(|_data: &[u8]| {
         args.push(OsString::from(generate_test_arg()));
     }
 
-    let (rust_output, uumain_exit_status) = generate_and_run_uumain(&args, uumain);
+    let rust_result = generate_and_run_uumain(&args, uumain);
 
-    // Run GNU test with the provided arguments and compare the output
-    match run_gnu_cmd(CMD_PATH, &args[1..], false) {
-        Ok((gnu_output, gnu_exit_status)) => {
-            let gnu_output = gnu_output.trim().to_owned();
-            println!("gnu_exit_status {}", gnu_exit_status);
-            println!("uumain_exit_status {}", uumain_exit_status);
-            if rust_output != gnu_output || uumain_exit_status != gnu_exit_status {
-                println!("Discrepancy detected!");
-                println!("Test: {:?}", &args[1..]);
-                println!("My output: {}", rust_output);
-                println!("GNU output: {}", gnu_output);
-                println!("My exit status: {}", uumain_exit_status);
-                println!("GNU exit status: {}", gnu_exit_status);
-                panic!();
-            } else {
-                println!(
-                    "Outputs and exit statuses matched for expression {:?}",
-                    &args[1..]
+    let gnu_result = match run_gnu_cmd(CMD_PATH, &args[1..], false) {
+        Ok(result) => result,
+        Err(error_result) => {
+            eprintln!("Failed to run GNU command:");
+            eprintln!("Stderr: {}", error_result.stderr);
+            eprintln!("Exit Code: {}", error_result.exit_code);
+            CommandResult {
+                stdout: String::new(),
+                stderr: error_result.stderr,
+                exit_code: error_result.exit_code,
+            }
+        }
+    };
+
+    compare_result(
+        "test",
+        &format!("{:?}", &args[1..]),
+        &rust_result.stdout,
+        &gnu_result.stdout,
+        &rust_result.stderr,
+        &gnu_result.stderr,
+        rust_result.exit_code,
+        gnu_result.exit_code,
+        false, // Set to true if you want to fail on stderr diff
     );
-            }
-        }
-        Err(_) => {
-            println!("GNU test execution failed for expression {:?}", &args[1..]);
-        }
-    }
 });