Merge pull request #1825 from cbjadwani/uniq-utf8-issues

uniq: utf-8 issues

tests/by-util/test_uniq.rs

@@ -138,3 +138,12 @@ fn test_stdin_zero_terminated() {
         .run()
         .stdout_is_fixture("sorted-zero-terminated.expected");
 }
+
+#[test]
+fn test_invalid_utf8() {
+    new_ucmd!()
+        .arg("not-utf8-sequence.txt")
+        .run()
+        .failure()
+        .stderr_only("uniq: error: invalid utf-8 sequence of 1 bytes from index 0");
+}

tests/fixtures/uniq/not-utf8-sequence.txt

@@ -0,0 +1,2 @@
+Next line contains two bytes - 0xCC and 0xCD - which are not a valid utf-8 sequence
+ÌÍ

tests/fixtures/uniq/skip-2-fields.expected

@@ -1,2 +1,2 @@
-	  aaa  aa  a
+	  aaa  ⟪⟫  a
 aa  a

tests/fixtures/uniq/skip-fields.txt

@@ -1,4 +1,4 @@
-	  aaa  aa  a
+	  aaa  ⟪⟫  a
 	  ZZZ     aa  a
 ZZZ     aa  a
 	  ZZZ     bb  a