RGBCube/uutils-coreutils
1
Fork 0
mirror of https://github.com/RGBCube/uutils-coreutils synced 2025-07-27 11:07:44 +00:00
Code Issues Activity

od: fix handling of non-ascii chars

Browse source
This commit is contained in:
Tomasz Guz 2025-06-25 17:14:56 +02:00
parent fb2399f56b
commit cad92245f7
2 changed files with 20 additions and 45 deletions
Download patch file Download diff file Expand all files Collapse all files

44
src/uu/od/src/prn_char.rs
View file

@ -2,7 +2,6 @@
// //
// For the full copyright and license information, please view the LICENSE // For the full copyright and license information, please view the LICENSE
// file that was distributed with this source code. // file that was distributed with this source code.
use std::str::from_utf8;
use crate::formatteriteminfo::{FormatWriter, FormatterItemInfo}; use crate::formatteriteminfo::{FormatWriter, FormatterItemInfo};
@ -51,33 +50,13 @@ fn format_item_c(bytes: &[u8]) -> String {
let b = bytes[0]; let b = bytes[0];
if b & 0x80 == 0x00 { if b & 0x80 == 0x00 {
// ASCII byte (0xxxxxxx)
match C_CHARS.get(b as usize) { match C_CHARS.get(b as usize) {
Some(s) => format!("{s:>4}"), Some(s) => format!("{s:>4}"),
None => format!("{b:>4}"), None => format!("{b:>4}"),
} }
} else if (b & 0xc0) == 0x80 {
// second or subsequent octet of an utf-8 sequence
String::from(" **")
} else if ((b & 0xe0) == 0xc0) && (bytes.len() >= 2) {
// start of a 2 octet utf-8 sequence
match from_utf8(&bytes[0..2]) {
Ok(s) => format!("{s:>4}"),
Err(_) => format!(" {b:03o}"),
}
} else if ((b & 0xf0) == 0xe0) && (bytes.len() >= 3) {
// start of a 3 octet utf-8 sequence
match from_utf8(&bytes[0..3]) {
Ok(s) => format!("{s:>4}"),
Err(_) => format!(" {b:03o}"),
}
} else if ((b & 0xf8) == 0xf0) && (bytes.len() >= 4) {
// start of a 4 octet utf-8 sequence
match from_utf8(&bytes[0..4]) {
Ok(s) => format!("{s:>4}"),
Err(_) => format!(" {b:03o}"),
}
} else { } else {
// invalid utf-8 // Continuation or leading byte of a multibyte UTF-8 sequence — treat as raw byte
format!(" {b:03o}") format!(" {b:03o}")
} }
} }
@ -125,27 +104,22 @@ fn test_format_item_c() {
assert_eq!(" 177", format_item_c(&[0x7f])); assert_eq!(" 177", format_item_c(&[0x7f]));
assert_eq!(" A", format_item_c(&[0x41, 0x21])); assert_eq!(" A", format_item_c(&[0x41, 0x21]));
assert_eq!(" **", format_item_c(&[0x80])); assert_eq!(" 200", format_item_c(&[0x80]));
assert_eq!(" **", format_item_c(&[0x9f])); assert_eq!(" 237", format_item_c(&[0x9f]));
assert_eq!(" ß", format_item_c(&[0xc3, 0x9f])); assert_eq!(" 303", format_item_c(&[0xc3, 0x9f]));
assert_eq!(" ß", format_item_c(&[0xc3, 0x9f, 0x21])); assert_eq!(" 303", format_item_c(&[0xc3, 0x9f, 0x21]));
assert_eq!(" \u{1000}", format_item_c(&[0xe1, 0x80, 0x80])); assert_eq!(" 341", format_item_c(&[0xe1, 0x80, 0x80]));
assert_eq!(" \u{1000}", format_item_c(&[0xe1, 0x80, 0x80, 0x21]));
assert_eq!(" \u{1f496}", format_item_c(&[0xf0, 0x9f, 0x92, 0x96])); assert_eq!(" 360", format_item_c(&[0xf0, 0x9f, 0x92, 0x96]));
assert_eq!(
" \u{1f496}",
format_item_c(&[0xf0, 0x9f, 0x92, 0x96, 0x21])
);
assert_eq!(" 300", format_item_c(&[0xc0, 0x80])); // invalid utf-8 (UTF-8 null) assert_eq!(" 300", format_item_c(&[0xc0, 0x80])); // invalid utf-8 (UTF-8 null)
assert_eq!(" 301", format_item_c(&[0xc1, 0xa1])); // invalid utf-8 assert_eq!(" 301", format_item_c(&[0xc1, 0xa1])); // invalid utf-8
assert_eq!(" 303", format_item_c(&[0xc3, 0xc3])); // invalid utf-8 assert_eq!(" 303", format_item_c(&[0xc3, 0xc3])); // invalid utf-8
assert_eq!(" 360", format_item_c(&[0xf0, 0x82, 0x82, 0xac])); // invalid utf-8 (overlong) assert_eq!(" 360", format_item_c(&[0xf0, 0x82, 0x82, 0xac])); // invalid utf-8 (overlong)
assert_eq!(" 360", format_item_c(&[0xf0, 0x9f, 0x92])); // invalid utf-8 (missing octet) assert_eq!(" 360", format_item_c(&[0xf0, 0x9f, 0x92])); // invalid utf-8 (missing octet)
assert_eq!(" \u{10FFFD}", format_item_c(&[0xf4, 0x8f, 0xbf, 0xbd])); // largest valid utf-8 // spell-checker:ignore 10FFFD FFFD assert_eq!(" 364", format_item_c(&[0xf4, 0x8f, 0xbf, 0xbd])); // largest valid utf-8 // spell-checker:ignore 10FFFD FFFD
assert_eq!(" 364", format_item_c(&[0xf4, 0x90, 0x00, 0x00])); // invalid utf-8 assert_eq!(" 364", format_item_c(&[0xf4, 0x90, 0x00, 0x00])); // invalid utf-8
assert_eq!(" 365", format_item_c(&[0xf5, 0x80, 0x80, 0x80])); // invalid utf-8 assert_eq!(" 365", format_item_c(&[0xf5, 0x80, 0x80, 0x80])); // invalid utf-8
assert_eq!(" 377", format_item_c(&[0xff])); // invalid utf-8 assert_eq!(" 377", format_item_c(&[0xff])); // invalid utf-8

21
tests/by-util/test_od.rs
View file

@ -279,18 +279,19 @@ fn test_f64() {
#[test] #[test]
fn test_multibyte() { fn test_multibyte() {
let input = "’‐ˆ‘˜語🙂✅🐶𝛑Universität Tübingen \u{1B000}"; // spell-checker:disable-line
new_ucmd!() new_ucmd!()
.arg("-c") .args(&["-t", "c"])
.arg("-w12") .run_piped_stdin(input.as_bytes())
.run_piped_stdin("Universität Tübingen \u{1B000}".as_bytes()) // spell-checker:disable-line
.success() .success()
.no_stderr() .no_stderr()
.stdout_is(unindent( .stdout_is(unindent(
" r"
0000000 U n i v e r s i t ä ** t 0000000 342 200 231 342 200 220 313 206 342 200 230 313 234 350 252 236
0000014 T ü ** b i n g e n \u{1B000} 0000020 360 237 231 202 342 234 205 360 237 220 266 360 235 233 221 U
0000030 ** ** ** 0000040 n i v e r s i t 303 244 t T 303 274 b
0000033 0000060 i n g e n 360 233 200 200
0000072
", ",
)); ));
} }
@ -714,10 +715,10 @@ fn test_ascii_dump() {
r" r"
0000000 00 01 0a 0d 10 1f 20 61 62 63 7d 7e 7f 80 90 a0 >...... abc}~....< 0000000 00 01 0a 0d 10 1f 20 61 62 63 7d 7e 7f 80 90 a0 >...... abc}~....<
nul soh nl cr dle us sp a b c } ~ del nul dle sp nul soh nl cr dle us sp a b c } ~ del nul dle sp
\0 001 \n \r 020 037 a b c } ~ 177 ** ** ** >...... abc}~....< \0 001 \n \r 020 037 a b c } ~ 177 200 220 240 >...... abc}~....<
0000020 b0 c0 d0 e0 f0 ff >......< 0000020 b0 c0 d0 e0 f0 ff >......<
0 @ P ` p del 0 @ P ` p del
** 300 320 340 360 377 >......< 260 300 320 340 360 377 >......<
0000026 0000026
", ",
)); ));