feat(ts): translate lossy and chunked UTF-8 decoding

ts/README.md

@@ -217,6 +217,12 @@ are std's `flat_map` of each char's, sized as it is, and write what they have le
 their lossy forms over bytes, decode as `char::decode_utf16` does: an unpaired surrogate
 fails with `rt.FromUtf16Error`, `{ kind }`, its private `FromUtf16ErrorKind`'s name, or
 becomes U+FFFD, and so does an odd byte at the end.
+`String::from_utf8_lossy` is a `TextDecoder`'s, which writes one U+FFFD for each maximal
+invalid sequence, as std's does; its `Cow<str>` is the text either way. `utf8_chunks` splits
+where `str::from_utf8` fails, so each `Utf8Chunk`'s `invalid` bytes are std's, and `Debug`
+of it writes what is left as std's does, each invalid byte as `\xNN`. `from_utf8_unchecked`,
+of a `str` or a `String`, decodes as `from_utf8_lossy` does the UTF-8 its contract promises.
+`String::into_bytes` and `Vec::<u8>::from` a string encode its UTF-8 anew.
 
 An `f64` is a JS number, so its operators are JS's, which round as Rust's do. Where
 JS differs from std, the runtime has std's: `round` halves away from zero, `max` and

ts/fixtures/lang/utf8/caller.ts

@@ -5,11 +5,11 @@ import { op } from "./lang.ts";
 export { ops, run } from "./lang.ts";
 
 /**
- * `utf8::decodes` of raw bytes: a byte order mark, which std keeps, and the sequences at
- * each edge of what it takes: surrogates, overlongs, past U+10FFFF, and cut short at the
- * end or by another byte.
+ * Raw bytes for `utf8::decodes` and `utf8::lossy`: a byte order mark, which std keeps, and
+ * the sequences at each edge of what it takes: surrogates, overlongs, past U+10FFFF, and
+ * cut short at the end or by another byte.
  */
-const utf8 = [
+const bytes = [
 	[0xef, 0xbb, 0xbf, 0x61],
 	[0xed, 0x9f, 0xbf, 0xed, 0xa0, 0x80, 0xed, 0xbf, 0xbf],
 	[0xc0, 0x80, 0xc1, 0xbf, 0xc2, 0x80],
@@ -20,6 +20,10 @@ const utf8 = [
 	[0xe2, 0x82],
 	[0xc2],
 	[0x80, 0xf8, 0xff],
-].map((bytes) => Uint8Array.of(...op("utf8::decodes"), 0, ...bytes));
+	[],
+	[0x61, 0xc3, 0xa9, 0xe2, 0x82, 0xac, 0xf0, 0x9f, 0x98, 0x80],
+];
 
-export const cases: Uint8Array[] = [...utf8];
+export const cases: Uint8Array[] = ["utf8::decodes", "utf8::lossy"].flatMap((name) =>
+	bytes.map((b) => Uint8Array.of(...op(name), 0, ...b)),
+);

ts/fixtures/lang/utf8/expected/alloc/string.ts

@@ -0,0 +1,13 @@
+// Generated by rust2-ts. Do not edit.
+
+import * as rt from "../rt.ts";
+import { eq$1v93ipo as str_error_eq$1v93ipo } from "../core/str/error.ts";
+
+export function eq$09knsz6(self: rt.FromUtf8Error, other: rt.FromUtf8Error): boolean {
+	if (!rt.eqBytes(self.bytes, other.bytes)) {
+		return false;
+	}
+	return str_error_eq$1v93ipo(self.error, other.error);
+}
+
+//# sourceMappingURL=string.ts.map

ts/fixtures/lang/utf8/expected/core/str/error.ts

@@ -0,0 +1,12 @@
+// Generated by rust2-ts. Do not edit.
+
+import * as rt from "../../rt.ts";
+
+export function eq$1v93ipo(self: rt.Utf8Error, other: rt.Utf8Error): boolean {
+	if (!(self.validUpTo === other.validUpTo)) {
+		return false;
+	}
+	return self.errorLen === other.errorLen;
+}
+
+//# sourceMappingURL=error.ts.map

ts/fixtures/lang/utf8/expected/core/str/lossy.ts

@@ -0,0 +1,12 @@
+// Generated by rust2-ts. Do not edit.
+
+import * as rt from "../../rt.ts";
+
+export function eq$00vlkdw(self: rt.Utf8Chunk, other: rt.Utf8Chunk): boolean {
+	if (!(self.valid === other.valid)) {
+		return false;
+	}
+	return rt.byteSliceEq(self.invalid, other.invalid);
+}
+
+//# sourceMappingURL=lossy.ts.map

ts/fixtures/lang/utf8/expected/lib.ts

@@ -1,14 +1,14 @@
 // Generated by rust2-ts. Do not edit.
 
 import * as rt from "./rt.ts";
-import { decodes } from "./utf8.ts";
+import { decodes, lossy } from "./utf8.ts";
 
 /**
  * Each case's name and what it runs. The first two bytes, big-endian, pick one modulo
  * their count, so a case added at the end moves no other.
  */
 export function cases(): [string, (a0: rt.u8, a1: rt.ByteSlice, a2: rt.ByteVec) => undefined][] {
-	return [["utf8::decodes", decodes as (a0: rt.u8, a1: rt.ByteSlice, a2: rt.ByteVec) => undefined] as [string, (a0: rt.u8, a1: rt.ByteSlice, a2: rt.ByteVec) => undefined]];
+	return [["utf8::decodes", decodes as (a0: rt.u8, a1: rt.ByteSlice, a2: rt.ByteVec) => undefined] as [string, (a0: rt.u8, a1: rt.ByteSlice, a2: rt.ByteVec) => undefined], ["utf8::lossy", lossy as (a0: rt.u8, a1: rt.ByteSlice, a2: rt.ByteVec) => undefined] as [string, (a0: rt.u8, a1: rt.ByteSlice, a2: rt.ByteVec) => undefined]];
 }
 
 //# sourceMappingURL=lib.ts.map

ts/fixtures/lang/utf8/expected/utf8.ts

@@ -1,36 +1,41 @@
 // Generated by rust2-ts. Do not edit.
 
 import * as rt from "./rt.ts";
+import { eq$09knsz6 as string_eq$09knsz6 } from "./alloc/string.ts";
 import { mapOr$02k4ito as option_mapOr$02k4ito } from "./core/option.ts";
+import { eq$00vlkdw as lossy_eq$00vlkdw } from "./core/str/lossy.ts";
 import { put } from "./harness.ts";
 
-export function decodes(kind: rt.u8, data: rt.ByteSlice, out: rt.ByteVec): void {
-	let bytes!: rt.ByteVec;
+/** The raw input, or each byte's from `BYTES`. */
+export function input(kind: rt.u8, data: rt.ByteSlice): rt.ByteVec {
 	if (rt.u8Rem(kind, 2) === 0) {
-		bytes = rt.byteSliceToByteVec(data);
-	} else {
-		bytes = rt.iterToByteVec(rt.iterMap(rt.iterSlice(rt.byteSliceView(data)), (_2: rt.u8): rt.u8 => {
-			const _54: rt.usize = rt.u8Rem(_2, 32) as rt.usize;
-			if (!(_54 < 32)) {
-				rt.panic("index out of bounds: the len is " + 32 + " but the index is " + _54);
-			}
-			return Uint8Array.of(0, 65, 127, 192, 193, 194, 223, 224, 225, 237, 238, 239, 240, 241, 244, 245, 128, 143, 144, 159, 160, 191, 128, 143, 144, 159, 160, 191, 128, 191, 152, 187)[_54];
-		}), undefined, true);
+		return rt.byteSliceToByteVec(data);
 	}
+	return rt.iterToByteVec(rt.iterMap(rt.iterSlice(rt.byteSliceView(data)), (_2: rt.u8): rt.u8 => {
+		const _52: rt.usize = rt.u8Rem(_2, 32) as rt.usize;
+		if (!(_52 < 32)) {
+			rt.panic("index out of bounds: the len is " + 32 + " but the index is " + _52);
+		}
+		return Uint8Array.of(0, 65, 127, 192, 193, 194, 223, 224, 225, 237, 238, 239, 240, 241, 244, 245, 128, 143, 144, 159, 160, 191, 128, 143, 144, 159, 160, 191, 128, 191, 152, 187)[_52];
+	}), undefined, true);
+}
+
+export function decodes(kind: rt.u8, data: rt.ByteSlice, out: rt.ByteVec): void {
+	const bytes: rt.ByteVec = input(kind, data);
 	const iter: rt.RangeInclusive<rt.usize> = new rt.RangeInclusive(0, bytes.length, rt.usizeStep);
 	while (true) {
-		const _14: rt.usize | rt.Done = iter.next();
-		if (_14 === rt.DONE) {
+		const _10: rt.usize | rt.Done = iter.next();
+		if (_10 === rt.DONE) {
 			break;
 		}
-		const _18: rt.Result<string, rt.Utf8Error> = rt.strFromUtf8(rt.byteSliceRange(rt.byteVecSlice(bytes), _14));
-		if (_18.ok) {
-			const s: string = _18.value;
+		const _14: rt.Result<string, rt.Utf8Error> = rt.strFromUtf8(rt.byteSliceRange(rt.byteVecSlice(bytes), _10));
+		if (_14.ok) {
+			const s: string = _14.value;
 			const args2: rt.Fmt[] = [(f: rt.Formatter) => rt.fmtStr(s, f)];
 			put(out, rt.fmtFormat([args2[0]]));
 			rt.byteVecPush(out, rt.stringLen(s) & 0xff);
 		} else {
-			const e: rt.Utf8Error = _18.error;
+			const e: rt.Utf8Error = _14.error;
 			rt.byteVecPush(out, e.validUpTo & 0xff);
 			rt.byteVecPush(out, option_mapOr$02k4ito(e.errorLen, 255, (n: rt.usize): rt.u8 => {
 				return n & 0xff;
@@ -39,11 +44,77 @@ export function decodes(kind: rt.u8, data: rt.ByteSlice, out: rt.ByteVec): void
 			put(out, rt.fmtFormat([args4[0], "|", rt.fmtSpec({ align: ">", width: 64 }, args4[0]), "|", args4[1], "|", rt.fmtSpec({ alternate: true }, args4[1])]));
 		}
 	}
-	const _61: rt.Result<string, rt.Utf8Error> = rt.strFromUtf8(rt.byteVecSlice(bytes));
-	if (_61.ok) {
-		rt.byteVecPush(out, rt.stringLen(_61.value) & 0xff);
+	const _57: rt.Result<string, rt.Utf8Error> = rt.strFromUtf8(rt.byteVecSlice(bytes));
+	if (_57.ok) {
+		rt.byteVecPush(out, rt.stringLen(_57.value) & 0xff);
+	} else {
+		rt.byteVecPush(out, _57.error.validUpTo & 0xff);
+	}
+}
+
+export function lossy(kind: rt.u8, data: rt.ByteSlice, out: rt.ByteVec): void {
+	const bytes: rt.ByteVec = input(kind, data);
+	const iter: rt.RangeInclusive<rt.usize> = new rt.RangeInclusive(0, bytes.length, rt.usizeStep);
+	while (true) {
+		const _10: rt.usize | rt.Done = iter.next();
+		if (_10 === rt.DONE) {
+			break;
+		}
+		const rest: rt.ByteSlice = rt.byteSliceRange(rt.byteVecSlice(bytes), _10);
+		const text: string = rt.stringFromUtf8Lossy(rest);
+		const args2: rt.Fmt[] = [(f: rt.Formatter) => rt.fmtPad(f, text), (f: rt.Formatter) => rt.fmtStr(text, f)];
+		put(out, rt.fmtFormat([args2[0], "|", args2[1]]));
+		rt.byteVecPush(out, rt.stringLen(text) & 0xff);
+		const owned: string = text;
+		put(out, owned);
+		put(out, rt.stringFromUtf8Lossy(rt.stringAsBytes(owned)));
+		rt.byteVecExtendFromSlice(out, rt.byteVecSlice(rt.stringIntoBytes(owned)));
+		rt.byteVecExtendFromSlice(out, rt.byteVecSlice(rt.stringIntoBytes(owned)));
+		put(out, rt.stringFromUtf8Lossy(rt.byteVecSlice(rt.stringIntoBytes(owned))));
+		const chunks: rt.Iter<rt.Utf8Chunk> = rt.byteSliceUtf8Chunks(rest);
+		const args4: rt.Fmt[] = [(f: rt.Formatter) => rt.fmtUtf8Chunks(chunks, f)];
+		put(out, rt.fmtFormat([args4[0], "|", rt.fmtSpec({ alternate: true }, args4[0])]));
+		let last: rt.Utf8Chunk | undefined = undefined;
+		while (true) {
+			const _75: rt.Utf8Chunk | rt.Done = chunks.next();
+			if (_75 !== rt.DONE) {
+				const chunk: rt.Utf8Chunk = _75;
+				put(out, chunk.valid);
+				rt.byteVecExtendFromSlice(out, chunk.invalid);
+				rt.byteVecPush(out, chunk.invalid.length & 0xff);
+				const args6: rt.Fmt[] = [(f: rt.Formatter) => rt.fmtUtf8ChunkDebug(chunk, f)];
+				put(out, rt.fmtFormat([args6[0], "|", rt.fmtSpec({ alternate: true }, args6[0])]));
+				const args8: rt.Fmt[] = [(f: rt.Formatter) => rt.fmtUtf8Chunks(chunks, f)];
+				put(out, rt.fmtFormat([args8[0]]));
+				rt.byteVecPush(out, rt.iterCount(chunks.clone()) & 0xff);
+				rt.byteVecPush(out, lossy_eq$00vlkdw(chunk, chunk) ? 1 : 0);
+				const _130: rt.Utf8Chunk | undefined = last;
+				const _133: rt.Utf8Chunk | undefined = chunk;
+				rt.byteVecPush(out, rt.eqOption(lossy_eq$00vlkdw)(_130, _133) ? 1 : 0);
+				last = chunk;
+			} else {
+				break;
+			}
+		}
+		const args10: rt.Fmt[] = [(f: rt.Formatter) => rt.fmtUtf8Chunks(chunks, f)];
+		put(out, rt.fmtFormat([args10[0]]));
+	}
+	const _148: rt.Result<string, rt.FromUtf8Error> = rt.stringFromUtf8(bytes);
+	if (_148.ok) {
+		put(out, _148.value);
 	} else {
-		rt.byteVecPush(out, _61.error.validUpTo & 0xff);
+		const e: rt.FromUtf8Error = _148.error;
+		rt.byteVecExtendFromSlice(out, rt.byteVecSlice(e.bytes));
+		const u: rt.Utf8Error = e.error;
+		rt.byteVecPush(out, u.validUpTo & 0xff);
+		rt.byteVecPush(out, option_mapOr$02k4ito(u.errorLen, 255, (n: rt.usize): rt.u8 => {
+			return n & 0xff;
+		}));
+		const args12: rt.Fmt[] = [(f: rt.Formatter) => rt.fmtUtf8ErrorDisplay(u, f)];
+		put(out, rt.fmtFormat([args12[0]]));
+		put(out, rt.stringFromUtf8Lossy(rt.byteVecSlice(e.bytes)));
+		rt.byteVecPush(out, string_eq$09knsz6(rt.fromUtf8ErrorClone(e), e) ? 1 : 0);
+		rt.byteVecExtendFromSlice(out, rt.byteVecSlice(e.bytes));
 	}
 }
 

ts/fixtures/lang/utf8/src/lib.rs

@@ -15,5 +15,5 @@ use harness::Case;
 /// Each case's name and what it runs. The first two bytes, big-endian, pick one modulo
 /// their count, so a case added at the end moves no other.
 fn cases() -> Vec<(&'static str, Case)> {
-	vec![("utf8::decodes", utf8::decodes)]
+	vec![("utf8::decodes", utf8::decodes), ("utf8::lossy", utf8::lossy)]
 }

ts/fixtures/lang/utf8/src/utf8.rs

@@ -1,5 +1,6 @@
 //! `str::from_utf8` from every offset of its input, and the `Utf8Error` it fails with:
-//! where it stops, how many bytes are invalid there, and how it prints. Input is either
+//! where it stops, how many bytes are invalid there, and how it prints; and the same input
+//! decoded lossily, split into chunks, and kept by a `FromUtf8Error`. Input is either
 //! raw, or bytes from an alphabet of UTF-8's boundaries, so random input makes the
 //! truncated, overlong, surrogate, and past-the-last-char sequences std refuses as often
 //! as the ones it takes.
@@ -13,11 +14,16 @@ const BYTES: [u8; 32] = [
 	0x9f, 0xa0, 0xbf, 0x80, 0x8f, 0x90, 0x9f, 0xa0, 0xbf, 0x80, 0xbf, 0x98, 0xbb,
 ];
 
-pub fn decodes(kind: u8, data: &[u8], out: &mut Vec<u8>) {
-	let bytes: Vec<u8> = match kind % 2 {
+/// The raw input, or each byte's from `BYTES`.
+fn input(kind: u8, data: &[u8]) -> Vec<u8> {
+	match kind % 2 {
 		0 => data.to_vec(),
 		_ => data.iter().map(|&b| BYTES[(b % 32) as usize]).collect(),
-	};
+	}
+}
+
+pub fn decodes(kind: u8, data: &[u8], out: &mut Vec<u8>) {
+	let bytes = input(kind, data);
 	for start in 0..=bytes.len() {
 		match std::str::from_utf8(&bytes[start..]) {
 			Ok(s) => {
@@ -37,3 +43,55 @@ pub fn decodes(kind: u8, data: &[u8], out: &mut Vec<u8>) {
 		Err(e) => out.push(e.valid_up_to() as u8),
 	}
 }
+
+pub fn lossy(kind: u8, data: &[u8], out: &mut Vec<u8>) {
+	let bytes = input(kind, data);
+	for start in 0..=bytes.len() {
+		let rest = &bytes[start..];
+		// A `Cow` of the bytes if they are valid, else of a `String` with each invalid
+		// sequence replaced.
+		let text = String::from_utf8_lossy(rest);
+		put(out, &format!("{text}|{text:?}"));
+		out.push(text.len() as u8);
+		let owned: String = text.into_owned();
+		put(out, &owned);
+		// Its UTF-8 again, taken unchecked.
+		put(out, unsafe { std::str::from_utf8_unchecked(owned.as_bytes()) });
+		let from: Vec<u8> = owned.clone().into();
+		out.extend_from_slice(&from);
+		out.extend_from_slice(&Vec::from(owned.as_str()));
+		put(out, &unsafe { String::from_utf8_unchecked(owned.into_bytes()) });
+
+		let mut chunks = rest.utf8_chunks();
+		put(out, &format!("{chunks:?}|{chunks:#?}"));
+		let mut last = None;
+		while let Some(chunk) = chunks.next() {
+			put(out, chunk.valid());
+			out.extend_from_slice(chunk.invalid());
+			out.push(chunk.invalid().len() as u8);
+			put(out, &format!("{chunk:?}|{chunk:#?}"));
+			// What it has left, which a clone reads on from independently.
+			put(out, &format!("{chunks:?}"));
+			out.push(chunks.clone().count() as u8);
+			out.push((chunk.clone() == chunk) as u8);
+			out.push((last.as_ref() == Some(&chunk)) as u8);
+			last = Some(chunk);
+		}
+		put(out, &format!("{chunks:?}"));
+	}
+
+	match String::from_utf8(bytes) {
+		Ok(s) => put(out, &s),
+		Err(e) => {
+			out.extend_from_slice(e.as_bytes());
+			let u = e.utf8_error();
+			out.push(u.valid_up_to() as u8);
+			out.push(u.error_len().map_or(0xff, |n| n as u8));
+			put(out, &format!("{u}"));
+			put(out, &String::from_utf8_lossy(e.as_bytes()));
+			out.push((e.clone() == e) as u8);
+			let back: Vec<u8> = e.into_bytes();
+			out.extend_from_slice(&back);
+		}
+	}
+}

ts/runtime/rt.ts

@@ -2035,7 +2035,8 @@ export class FromUtf8Error {
 
 	constructor(bytes: ByteVec) {
 		this.bytes = bytes;
-		this.error = utf8Error(byteVecView(bytes));
+		// Made only of bytes that are not UTF-8.
+		this.error = utf8Check(byteVecView(bytes)) as Utf8Error;
 	}
 }
 
@@ -2044,8 +2045,8 @@ export function fromUtf8ErrorClone(e: FromUtf8Error): FromUtf8Error {
 	return new FromUtf8Error(byteVecFrom(byteVecView(e.bytes)));
 }
 
-/** Where `v`, which is not UTF-8, stops being so, and how many bytes are invalid there, as std reports it. */
-function utf8Error(v: Uint8Array): Utf8Error {
+/** Where `v` stops being UTF-8, and how many bytes are invalid there, as std reports it: `undefined` if it never does. */
+export function utf8Check(v: Uint8Array): Utf8Error | undefined {
 	const tail = (b: number | undefined) => b !== undefined && b >= 0x80 && b <= 0xbf;
 	for (let i = 0; i < v.length; ) {
 		const first = v[i];
@@ -2066,7 +2067,7 @@ function utf8Error(v: Uint8Array): Utf8Error {
 		}
 		i += width;
 	}
-	throw new Error("utf8Error: the bytes are valid UTF-8");
+	return undefined;
 }
 
 /** `str::from_utf8`. */
@@ -2075,7 +2076,7 @@ export function strFromUtf8(bytes: ByteSlice): Result<string, Utf8Error> {
 	try {
 		return Ok(decoder.decode(v));
 	} catch {
-		return Err(utf8Error(v));
+		return Err(utf8Check(v) as Utf8Error);
 	}
 }
 
@@ -2093,6 +2094,11 @@ export function stringAsBytes(s: string): ByteSlice {
 	return byteSliceFrom(encoder.encode(s));
 }
 
+/** `String::into_bytes`: its UTF-8 bytes. */
+export function stringIntoBytes(s: string): ByteVec {
+	return byteVecFrom(encoder.encode(s));
+}
+
 /** `str::len`: its length in UTF-8 bytes. */
 export function stringLen(s: string): number {
 	return encoder.encode(s).length;