diff --git a/ts/README.md b/ts/README.md index 0d2472593..a7c2ce7b6 100644 --- a/ts/README.md +++ b/ts/README.md @@ -217,6 +217,12 @@ are std's `flat_map` of each char's, sized as it is, and write what they have le their lossy forms over bytes, decode as `char::decode_utf16` does: an unpaired surrogate fails with `rt.FromUtf16Error`, `{ kind }`, its private `FromUtf16ErrorKind`'s name, or becomes U+FFFD, and so does an odd byte at the end. +`String::from_utf8_lossy` is a `TextDecoder`'s, which writes one U+FFFD for each maximal +invalid sequence, as std's does; its `Cow` is the text either way. `utf8_chunks` splits +where `str::from_utf8` fails, so each `Utf8Chunk`'s `invalid` bytes are std's, and `Debug` +of it writes what is left as std's does, each invalid byte as `\xNN`. `from_utf8_unchecked`, +of a `str` or a `String`, decodes as `from_utf8_lossy` does the UTF-8 its contract promises. +`String::into_bytes` and `Vec::::from` a string encode its UTF-8 anew. An `f64` is a JS number, so its operators are JS's, which round as Rust's do. Where JS differs from std, the runtime has std's: `round` halves away from zero, `max` and diff --git a/ts/fixtures/lang/utf8/caller.ts b/ts/fixtures/lang/utf8/caller.ts index afcc1bdb1..372956453 100644 --- a/ts/fixtures/lang/utf8/caller.ts +++ b/ts/fixtures/lang/utf8/caller.ts @@ -5,11 +5,11 @@ import { op } from "./lang.ts"; export { ops, run } from "./lang.ts"; /** - * `utf8::decodes` of raw bytes: a byte order mark, which std keeps, and the sequences at - * each edge of what it takes: surrogates, overlongs, past U+10FFFF, and cut short at the - * end or by another byte. + * Raw bytes for `utf8::decodes` and `utf8::lossy`: a byte order mark, which std keeps, and + * the sequences at each edge of what it takes: surrogates, overlongs, past U+10FFFF, and + * cut short at the end or by another byte. */ -const utf8 = [ +const bytes = [ [0xef, 0xbb, 0xbf, 0x61], [0xed, 0x9f, 0xbf, 0xed, 0xa0, 0x80, 0xed, 0xbf, 0xbf], [0xc0, 0x80, 0xc1, 0xbf, 0xc2, 0x80], @@ -20,6 +20,10 @@ const utf8 = [ [0xe2, 0x82], [0xc2], [0x80, 0xf8, 0xff], -].map((bytes) => Uint8Array.of(...op("utf8::decodes"), 0, ...bytes)); + [], + [0x61, 0xc3, 0xa9, 0xe2, 0x82, 0xac, 0xf0, 0x9f, 0x98, 0x80], +]; -export const cases: Uint8Array[] = [...utf8]; +export const cases: Uint8Array[] = ["utf8::decodes", "utf8::lossy"].flatMap((name) => + bytes.map((b) => Uint8Array.of(...op(name), 0, ...b)), +); diff --git a/ts/fixtures/lang/utf8/expected/alloc/string.ts b/ts/fixtures/lang/utf8/expected/alloc/string.ts new file mode 100644 index 000000000..b2cfd6bd2 --- /dev/null +++ b/ts/fixtures/lang/utf8/expected/alloc/string.ts @@ -0,0 +1,13 @@ +// Generated by rust2-ts. Do not edit. + +import * as rt from "../rt.ts"; +import { eq$1v93ipo as str_error_eq$1v93ipo } from "../core/str/error.ts"; + +export function eq$09knsz6(self: rt.FromUtf8Error, other: rt.FromUtf8Error): boolean { + if (!rt.eqBytes(self.bytes, other.bytes)) { + return false; + } + return str_error_eq$1v93ipo(self.error, other.error); +} + +//# sourceMappingURL=string.ts.map diff --git a/ts/fixtures/lang/utf8/expected/core/str/error.ts b/ts/fixtures/lang/utf8/expected/core/str/error.ts new file mode 100644 index 000000000..1c9975cc3 --- /dev/null +++ b/ts/fixtures/lang/utf8/expected/core/str/error.ts @@ -0,0 +1,12 @@ +// Generated by rust2-ts. Do not edit. + +import * as rt from "../../rt.ts"; + +export function eq$1v93ipo(self: rt.Utf8Error, other: rt.Utf8Error): boolean { + if (!(self.validUpTo === other.validUpTo)) { + return false; + } + return self.errorLen === other.errorLen; +} + +//# sourceMappingURL=error.ts.map diff --git a/ts/fixtures/lang/utf8/expected/core/str/lossy.ts b/ts/fixtures/lang/utf8/expected/core/str/lossy.ts new file mode 100644 index 000000000..666f08a7b --- /dev/null +++ b/ts/fixtures/lang/utf8/expected/core/str/lossy.ts @@ -0,0 +1,12 @@ +// Generated by rust2-ts. Do not edit. + +import * as rt from "../../rt.ts"; + +export function eq$00vlkdw(self: rt.Utf8Chunk, other: rt.Utf8Chunk): boolean { + if (!(self.valid === other.valid)) { + return false; + } + return rt.byteSliceEq(self.invalid, other.invalid); +} + +//# sourceMappingURL=lossy.ts.map diff --git a/ts/fixtures/lang/utf8/expected/lib.ts b/ts/fixtures/lang/utf8/expected/lib.ts index d15066979..b503294bf 100644 --- a/ts/fixtures/lang/utf8/expected/lib.ts +++ b/ts/fixtures/lang/utf8/expected/lib.ts @@ -1,14 +1,14 @@ // Generated by rust2-ts. Do not edit. import * as rt from "./rt.ts"; -import { decodes } from "./utf8.ts"; +import { decodes, lossy } from "./utf8.ts"; /** * Each case's name and what it runs. The first two bytes, big-endian, pick one modulo * their count, so a case added at the end moves no other. */ export function cases(): [string, (a0: rt.u8, a1: rt.ByteSlice, a2: rt.ByteVec) => undefined][] { - return [["utf8::decodes", decodes as (a0: rt.u8, a1: rt.ByteSlice, a2: rt.ByteVec) => undefined] as [string, (a0: rt.u8, a1: rt.ByteSlice, a2: rt.ByteVec) => undefined]]; + return [["utf8::decodes", decodes as (a0: rt.u8, a1: rt.ByteSlice, a2: rt.ByteVec) => undefined] as [string, (a0: rt.u8, a1: rt.ByteSlice, a2: rt.ByteVec) => undefined], ["utf8::lossy", lossy as (a0: rt.u8, a1: rt.ByteSlice, a2: rt.ByteVec) => undefined] as [string, (a0: rt.u8, a1: rt.ByteSlice, a2: rt.ByteVec) => undefined]]; } //# sourceMappingURL=lib.ts.map diff --git a/ts/fixtures/lang/utf8/expected/utf8.ts b/ts/fixtures/lang/utf8/expected/utf8.ts index 80b53b512..790fb23b5 100644 --- a/ts/fixtures/lang/utf8/expected/utf8.ts +++ b/ts/fixtures/lang/utf8/expected/utf8.ts @@ -1,36 +1,41 @@ // Generated by rust2-ts. Do not edit. import * as rt from "./rt.ts"; +import { eq$09knsz6 as string_eq$09knsz6 } from "./alloc/string.ts"; import { mapOr$02k4ito as option_mapOr$02k4ito } from "./core/option.ts"; +import { eq$00vlkdw as lossy_eq$00vlkdw } from "./core/str/lossy.ts"; import { put } from "./harness.ts"; -export function decodes(kind: rt.u8, data: rt.ByteSlice, out: rt.ByteVec): void { - let bytes!: rt.ByteVec; +/** The raw input, or each byte's from `BYTES`. */ +export function input(kind: rt.u8, data: rt.ByteSlice): rt.ByteVec { if (rt.u8Rem(kind, 2) === 0) { - bytes = rt.byteSliceToByteVec(data); - } else { - bytes = rt.iterToByteVec(rt.iterMap(rt.iterSlice(rt.byteSliceView(data)), (_2: rt.u8): rt.u8 => { - const _54: rt.usize = rt.u8Rem(_2, 32) as rt.usize; - if (!(_54 < 32)) { - rt.panic("index out of bounds: the len is " + 32 + " but the index is " + _54); - } - return Uint8Array.of(0, 65, 127, 192, 193, 194, 223, 224, 225, 237, 238, 239, 240, 241, 244, 245, 128, 143, 144, 159, 160, 191, 128, 143, 144, 159, 160, 191, 128, 191, 152, 187)[_54]; - }), undefined, true); + return rt.byteSliceToByteVec(data); } + return rt.iterToByteVec(rt.iterMap(rt.iterSlice(rt.byteSliceView(data)), (_2: rt.u8): rt.u8 => { + const _52: rt.usize = rt.u8Rem(_2, 32) as rt.usize; + if (!(_52 < 32)) { + rt.panic("index out of bounds: the len is " + 32 + " but the index is " + _52); + } + return Uint8Array.of(0, 65, 127, 192, 193, 194, 223, 224, 225, 237, 238, 239, 240, 241, 244, 245, 128, 143, 144, 159, 160, 191, 128, 143, 144, 159, 160, 191, 128, 191, 152, 187)[_52]; + }), undefined, true); +} + +export function decodes(kind: rt.u8, data: rt.ByteSlice, out: rt.ByteVec): void { + const bytes: rt.ByteVec = input(kind, data); const iter: rt.RangeInclusive = new rt.RangeInclusive(0, bytes.length, rt.usizeStep); while (true) { - const _14: rt.usize | rt.Done = iter.next(); - if (_14 === rt.DONE) { + const _10: rt.usize | rt.Done = iter.next(); + if (_10 === rt.DONE) { break; } - const _18: rt.Result = rt.strFromUtf8(rt.byteSliceRange(rt.byteVecSlice(bytes), _14)); - if (_18.ok) { - const s: string = _18.value; + const _14: rt.Result = rt.strFromUtf8(rt.byteSliceRange(rt.byteVecSlice(bytes), _10)); + if (_14.ok) { + const s: string = _14.value; const args2: rt.Fmt[] = [(f: rt.Formatter) => rt.fmtStr(s, f)]; put(out, rt.fmtFormat([args2[0]])); rt.byteVecPush(out, rt.stringLen(s) & 0xff); } else { - const e: rt.Utf8Error = _18.error; + const e: rt.Utf8Error = _14.error; rt.byteVecPush(out, e.validUpTo & 0xff); rt.byteVecPush(out, option_mapOr$02k4ito(e.errorLen, 255, (n: rt.usize): rt.u8 => { return n & 0xff; @@ -39,11 +44,77 @@ export function decodes(kind: rt.u8, data: rt.ByteSlice, out: rt.ByteVec): void put(out, rt.fmtFormat([args4[0], "|", rt.fmtSpec({ align: ">", width: 64 }, args4[0]), "|", args4[1], "|", rt.fmtSpec({ alternate: true }, args4[1])])); } } - const _61: rt.Result = rt.strFromUtf8(rt.byteVecSlice(bytes)); - if (_61.ok) { - rt.byteVecPush(out, rt.stringLen(_61.value) & 0xff); + const _57: rt.Result = rt.strFromUtf8(rt.byteVecSlice(bytes)); + if (_57.ok) { + rt.byteVecPush(out, rt.stringLen(_57.value) & 0xff); + } else { + rt.byteVecPush(out, _57.error.validUpTo & 0xff); + } +} + +export function lossy(kind: rt.u8, data: rt.ByteSlice, out: rt.ByteVec): void { + const bytes: rt.ByteVec = input(kind, data); + const iter: rt.RangeInclusive = new rt.RangeInclusive(0, bytes.length, rt.usizeStep); + while (true) { + const _10: rt.usize | rt.Done = iter.next(); + if (_10 === rt.DONE) { + break; + } + const rest: rt.ByteSlice = rt.byteSliceRange(rt.byteVecSlice(bytes), _10); + const text: string = rt.stringFromUtf8Lossy(rest); + const args2: rt.Fmt[] = [(f: rt.Formatter) => rt.fmtPad(f, text), (f: rt.Formatter) => rt.fmtStr(text, f)]; + put(out, rt.fmtFormat([args2[0], "|", args2[1]])); + rt.byteVecPush(out, rt.stringLen(text) & 0xff); + const owned: string = text; + put(out, owned); + put(out, rt.stringFromUtf8Lossy(rt.stringAsBytes(owned))); + rt.byteVecExtendFromSlice(out, rt.byteVecSlice(rt.stringIntoBytes(owned))); + rt.byteVecExtendFromSlice(out, rt.byteVecSlice(rt.stringIntoBytes(owned))); + put(out, rt.stringFromUtf8Lossy(rt.byteVecSlice(rt.stringIntoBytes(owned)))); + const chunks: rt.Iter = rt.byteSliceUtf8Chunks(rest); + const args4: rt.Fmt[] = [(f: rt.Formatter) => rt.fmtUtf8Chunks(chunks, f)]; + put(out, rt.fmtFormat([args4[0], "|", rt.fmtSpec({ alternate: true }, args4[0])])); + let last: rt.Utf8Chunk | undefined = undefined; + while (true) { + const _75: rt.Utf8Chunk | rt.Done = chunks.next(); + if (_75 !== rt.DONE) { + const chunk: rt.Utf8Chunk = _75; + put(out, chunk.valid); + rt.byteVecExtendFromSlice(out, chunk.invalid); + rt.byteVecPush(out, chunk.invalid.length & 0xff); + const args6: rt.Fmt[] = [(f: rt.Formatter) => rt.fmtUtf8ChunkDebug(chunk, f)]; + put(out, rt.fmtFormat([args6[0], "|", rt.fmtSpec({ alternate: true }, args6[0])])); + const args8: rt.Fmt[] = [(f: rt.Formatter) => rt.fmtUtf8Chunks(chunks, f)]; + put(out, rt.fmtFormat([args8[0]])); + rt.byteVecPush(out, rt.iterCount(chunks.clone()) & 0xff); + rt.byteVecPush(out, lossy_eq$00vlkdw(chunk, chunk) ? 1 : 0); + const _130: rt.Utf8Chunk | undefined = last; + const _133: rt.Utf8Chunk | undefined = chunk; + rt.byteVecPush(out, rt.eqOption(lossy_eq$00vlkdw)(_130, _133) ? 1 : 0); + last = chunk; + } else { + break; + } + } + const args10: rt.Fmt[] = [(f: rt.Formatter) => rt.fmtUtf8Chunks(chunks, f)]; + put(out, rt.fmtFormat([args10[0]])); + } + const _148: rt.Result = rt.stringFromUtf8(bytes); + if (_148.ok) { + put(out, _148.value); } else { - rt.byteVecPush(out, _61.error.validUpTo & 0xff); + const e: rt.FromUtf8Error = _148.error; + rt.byteVecExtendFromSlice(out, rt.byteVecSlice(e.bytes)); + const u: rt.Utf8Error = e.error; + rt.byteVecPush(out, u.validUpTo & 0xff); + rt.byteVecPush(out, option_mapOr$02k4ito(u.errorLen, 255, (n: rt.usize): rt.u8 => { + return n & 0xff; + })); + const args12: rt.Fmt[] = [(f: rt.Formatter) => rt.fmtUtf8ErrorDisplay(u, f)]; + put(out, rt.fmtFormat([args12[0]])); + put(out, rt.stringFromUtf8Lossy(rt.byteVecSlice(e.bytes))); + rt.byteVecPush(out, string_eq$09knsz6(rt.fromUtf8ErrorClone(e), e) ? 1 : 0); + rt.byteVecExtendFromSlice(out, rt.byteVecSlice(e.bytes)); } } diff --git a/ts/fixtures/lang/utf8/src/lib.rs b/ts/fixtures/lang/utf8/src/lib.rs index 3efca4a31..352cde484 100644 --- a/ts/fixtures/lang/utf8/src/lib.rs +++ b/ts/fixtures/lang/utf8/src/lib.rs @@ -15,5 +15,5 @@ use harness::Case; /// Each case's name and what it runs. The first two bytes, big-endian, pick one modulo /// their count, so a case added at the end moves no other. fn cases() -> Vec<(&'static str, Case)> { - vec![("utf8::decodes", utf8::decodes)] + vec![("utf8::decodes", utf8::decodes), ("utf8::lossy", utf8::lossy)] } diff --git a/ts/fixtures/lang/utf8/src/utf8.rs b/ts/fixtures/lang/utf8/src/utf8.rs index 10149c633..c646ac87a 100644 --- a/ts/fixtures/lang/utf8/src/utf8.rs +++ b/ts/fixtures/lang/utf8/src/utf8.rs @@ -1,5 +1,6 @@ //! `str::from_utf8` from every offset of its input, and the `Utf8Error` it fails with: -//! where it stops, how many bytes are invalid there, and how it prints. Input is either +//! where it stops, how many bytes are invalid there, and how it prints; and the same input +//! decoded lossily, split into chunks, and kept by a `FromUtf8Error`. Input is either //! raw, or bytes from an alphabet of UTF-8's boundaries, so random input makes the //! truncated, overlong, surrogate, and past-the-last-char sequences std refuses as often //! as the ones it takes. @@ -13,11 +14,16 @@ const BYTES: [u8; 32] = [ 0x9f, 0xa0, 0xbf, 0x80, 0x8f, 0x90, 0x9f, 0xa0, 0xbf, 0x80, 0xbf, 0x98, 0xbb, ]; -pub fn decodes(kind: u8, data: &[u8], out: &mut Vec) { - let bytes: Vec = match kind % 2 { +/// The raw input, or each byte's from `BYTES`. +fn input(kind: u8, data: &[u8]) -> Vec { + match kind % 2 { 0 => data.to_vec(), _ => data.iter().map(|&b| BYTES[(b % 32) as usize]).collect(), - }; + } +} + +pub fn decodes(kind: u8, data: &[u8], out: &mut Vec) { + let bytes = input(kind, data); for start in 0..=bytes.len() { match std::str::from_utf8(&bytes[start..]) { Ok(s) => { @@ -37,3 +43,55 @@ pub fn decodes(kind: u8, data: &[u8], out: &mut Vec) { Err(e) => out.push(e.valid_up_to() as u8), } } + +pub fn lossy(kind: u8, data: &[u8], out: &mut Vec) { + let bytes = input(kind, data); + for start in 0..=bytes.len() { + let rest = &bytes[start..]; + // A `Cow` of the bytes if they are valid, else of a `String` with each invalid + // sequence replaced. + let text = String::from_utf8_lossy(rest); + put(out, &format!("{text}|{text:?}")); + out.push(text.len() as u8); + let owned: String = text.into_owned(); + put(out, &owned); + // Its UTF-8 again, taken unchecked. + put(out, unsafe { std::str::from_utf8_unchecked(owned.as_bytes()) }); + let from: Vec = owned.clone().into(); + out.extend_from_slice(&from); + out.extend_from_slice(&Vec::from(owned.as_str())); + put(out, &unsafe { String::from_utf8_unchecked(owned.into_bytes()) }); + + let mut chunks = rest.utf8_chunks(); + put(out, &format!("{chunks:?}|{chunks:#?}")); + let mut last = None; + while let Some(chunk) = chunks.next() { + put(out, chunk.valid()); + out.extend_from_slice(chunk.invalid()); + out.push(chunk.invalid().len() as u8); + put(out, &format!("{chunk:?}|{chunk:#?}")); + // What it has left, which a clone reads on from independently. + put(out, &format!("{chunks:?}")); + out.push(chunks.clone().count() as u8); + out.push((chunk.clone() == chunk) as u8); + out.push((last.as_ref() == Some(&chunk)) as u8); + last = Some(chunk); + } + put(out, &format!("{chunks:?}")); + } + + match String::from_utf8(bytes) { + Ok(s) => put(out, &s), + Err(e) => { + out.extend_from_slice(e.as_bytes()); + let u = e.utf8_error(); + out.push(u.valid_up_to() as u8); + out.push(u.error_len().map_or(0xff, |n| n as u8)); + put(out, &format!("{u}")); + put(out, &String::from_utf8_lossy(e.as_bytes())); + out.push((e.clone() == e) as u8); + let back: Vec = e.into_bytes(); + out.extend_from_slice(&back); + } + } +} diff --git a/ts/runtime/rt.ts b/ts/runtime/rt.ts index 2864e9f00..6c28bca92 100644 --- a/ts/runtime/rt.ts +++ b/ts/runtime/rt.ts @@ -2035,7 +2035,8 @@ export class FromUtf8Error { constructor(bytes: ByteVec) { this.bytes = bytes; - this.error = utf8Error(byteVecView(bytes)); + // Made only of bytes that are not UTF-8. + this.error = utf8Check(byteVecView(bytes)) as Utf8Error; } } @@ -2044,8 +2045,8 @@ export function fromUtf8ErrorClone(e: FromUtf8Error): FromUtf8Error { return new FromUtf8Error(byteVecFrom(byteVecView(e.bytes))); } -/** Where `v`, which is not UTF-8, stops being so, and how many bytes are invalid there, as std reports it. */ -function utf8Error(v: Uint8Array): Utf8Error { +/** Where `v` stops being UTF-8, and how many bytes are invalid there, as std reports it: `undefined` if it never does. */ +export function utf8Check(v: Uint8Array): Utf8Error | undefined { const tail = (b: number | undefined) => b !== undefined && b >= 0x80 && b <= 0xbf; for (let i = 0; i < v.length; ) { const first = v[i]; @@ -2066,7 +2067,7 @@ function utf8Error(v: Uint8Array): Utf8Error { } i += width; } - throw new Error("utf8Error: the bytes are valid UTF-8"); + return undefined; } /** `str::from_utf8`. */ @@ -2075,7 +2076,7 @@ export function strFromUtf8(bytes: ByteSlice): Result { try { return Ok(decoder.decode(v)); } catch { - return Err(utf8Error(v)); + return Err(utf8Check(v) as Utf8Error); } } @@ -2093,6 +2094,11 @@ export function stringAsBytes(s: string): ByteSlice { return byteSliceFrom(encoder.encode(s)); } +/** `String::into_bytes`: its UTF-8 bytes. */ +export function stringIntoBytes(s: string): ByteVec { + return byteVecFrom(encoder.encode(s)); +} + /** `str::len`: its length in UTF-8 bytes. */ export function stringLen(s: string): number { return encoder.encode(s).length; diff --git a/ts/runtime/rt/str.ts b/ts/runtime/rt/str.ts index 801675824..646e9ea39 100644 --- a/ts/runtime/rt/str.ts +++ b/ts/runtime/rt/str.ts @@ -6,9 +6,28 @@ // well formed, so a pattern's text matches where its UTF-8 would, never across a // surrogate pair. -import { type ByteSlice, Err, type FmtResult, type Formatter, fmtCharDebug, fmtDebugStructFields, fmtPad, Ok, type Place, panic, panicFmt, type Result, stringLen } from "../rt.ts"; +import { + ByteSlice, + byteSliceView, + Err, + type FmtResult, + type Formatter, + fmtCharDebug, + fmtDebugStructFields, + fmtList, + fmtPad, + fmtStr, + Ok, + type Place, + panic, + panicFmt, + type Result, + stringLen, + utf8Check, +} from "../rt.ts"; +import { escapeDebug } from "./char.ts"; import { type Bounds, boundsRange, type Part } from "./collections.ts"; -import type { char, u8, u16, usize } from "./int.ts"; +import { type char, type u8, type u16, type usize, u8Debug } from "./int.ts"; import { type Clones, DONE, type Done, Iter, iterFilter, iterForEach, iterMap, iterRev, iterSlice, type RangeInclusive, type SizeHint } from "./iter.ts"; import { drop as dispose } from "./task.ts"; @@ -785,6 +804,82 @@ export function strBytes(s: string): Iter { return iterSlice(encoder.encode(s)); } +// Lossy and chunked UTF-8. A non-fatal `TextDecoder` writes one U+FFFD for each maximal +// invalid sequence, as WHATWG's decoder and std's both do. + +// `ignoreBOM` keeps a leading U+FEFF, as Rust does. +const lossy = /* @__PURE__ */ new TextDecoder("utf-8", { ignoreBOM: true }); + +/** `String::from_utf8_lossy`: the text, each invalid sequence replaced by U+FFFD. */ +export function stringFromUtf8Lossy(bytes: ByteSlice): string { + return lossy.decode(byteSliceView(bytes)); +} + +/** `core::str::Utf8Chunk`: valid text, and the invalid bytes after it. */ +export type Utf8Chunk = { + /** `Utf8Chunk::valid`. */ + readonly valid: string; + /** `Utf8Chunk::invalid`: empty only in the last chunk. */ + readonly invalid: ByteSlice; +}; + +/** `<[u8]>::utf8_chunks`. */ +export function byteSliceUtf8Chunks(bytes: ByteSlice): Iter { + return new Utf8Chunks(bytes); +} + +class Utf8Chunks extends Iter { + #source: ByteSlice; + + constructor(source: ByteSlice) { + super(); + this.#source = source; + } + + next(): Utf8Chunk | Done { + const s = this.#source; + if (s.length === 0) return DONE; + const error = utf8Check(byteSliceView(s)); + const valid = error?.validUpTo ?? s.length; + // A sequence the input ends in the middle of is invalid up to the end. + const end = error === undefined ? s.length : valid + (error.errorLen ?? s.length - valid); + this.#source = new ByteSlice(s.data, s.start + end, s.length - end); + const text = lossy.decode(s.data.subarray(s.start, s.start + valid)); + return { valid: text, invalid: new ByteSlice(s.data, s.start + valid, end - valid) }; + } + + /** What it has left, which std's `Debug` writes. */ + source(): ByteSlice { + return this.#source; + } + + override clone(): Iter { + return new Utf8Chunks(this.#source); + } +} + +/** `Debug` of a `Utf8Chunks`: what it has left as a string, each invalid byte as `\xNN`. */ +export function fmtUtf8Chunks(it: Iter, f: Formatter): FmtResult { + return fmtDebugStructFields(f, "Utf8Chunks", ["source"], [ + (f) => { + let out = '"'; + for (const chunk of new Utf8Chunks((it as Utf8Chunks).source())) { + for (const c of chunk.valid) out += escapeDebug(c, '"'); + for (const b of byteSliceView(chunk.invalid)) out += `\\x${b.toString(16).toUpperCase().padStart(2, "0")}`; + } + return f.out.writeStr(`${out}"`); + }, + ]); +} + +/** `Debug` of a `Utf8Chunk`, as std derives it. */ +export function fmtUtf8ChunkDebug(chunk: Utf8Chunk, f: Formatter): FmtResult { + return fmtDebugStructFields(f, "Utf8Chunk", ["valid", "invalid"], [ + (f) => fmtStr(chunk.valid, f), + (f) => fmtList(byteSliceView(chunk.invalid), f, u8Debug), + ]); +} + // `String`'s edits, each giving the edited string, which the caller writes where the // `&mut String` points, and what the edit returns beside it. diff --git a/ts/src/lower/collections.rs b/ts/src/lower/collections.rs index 65ca57671..9a113a915 100644 --- a/ts/src/lower/collections.rs +++ b/ts/src/lower/collections.rs @@ -2500,7 +2500,7 @@ fn std_key<'tcx>(b: &Body<'_, '_, 'tcx>, ty: Ty<'tcx>) -> bool { ty::Adt(..) if let Some(bound) = range_of(b, ty) => held(bound), ty::Adt(..) if let Some((inner, _)) = cell_of(b, ty) => held(inner), ty::Adt(_, args) if derived_std(b, ty) => args.types().all(held), - ty::Adt(..) if b.t.std_of(ty).is_some_and(Std::error) => true, + ty::Adt(..) if b.t.std_of(ty).is_some_and(Std::record) => true, ty::Adt(..) => library::string(b.t.tcx, ty) || matches!(b.t.std_of(ty), Some(Std::Ordering | Std::Duration)), _ => false, } @@ -2833,7 +2833,7 @@ pub fn eq_fn<'tcx>(b: &mut Body<'_, '_, 'tcx>, ty: Ty<'tcx>, span: Span) -> Expr // Whose `ne` is `!eq`, which compares what it holds by `eq`. ty::Adt(..) if let Some((inner, by)) = cell_of(b, ty) => Expr::rt(by, vec![eq_fn(b, inner, span)]), ty::Ref(..) if let Some(inner) = b.t.place_pointee(ty) => Expr::rt("cellBy", vec![eq_fn(b, inner, span)]), - _ if by_own(b, ty) || derived_std(b, ty) || b.t.std_of(ty).is_some_and(Std::error) => { + _ if by_own(b, ty) || derived_std(b, ty) || b.t.std_of(ty).is_some_and(Std::record) => { let eq = trait_fn(tcx, "PartialEq", "eq"); b.fn_value(eq, tcx.mk_args(&[ty.into(), ty.into()]), span) } diff --git a/ts/src/lower/fmt.rs b/ts/src/lower/fmt.rs index 26bac5dda..a89d82ae1 100644 --- a/ts/src/lower/fmt.rs +++ b/ts/src/lower/fmt.rs @@ -535,6 +535,7 @@ pub fn std_impl<'tcx>( (Std::FromUtf16Error, Trait::Display) => rt("fromUtf16ErrorDisplay", vec![value, f]), (Std::Utf8Error, Trait::Debug) => rt("utf8ErrorDebug", vec![value, f]), (Std::Utf8Error, Trait::Display) => rt("utf8ErrorDisplay", vec![value, f]), + (Std::Utf8Chunk, Trait::Debug) => rt("utf8ChunkDebug", vec![value, f]), (Std::Duration, Trait::Debug) => Expr::rt("durationDebug", vec![value, f]), (Std::Layout, Trait::Debug) => rt("layoutDebug", vec![value, f]), (Std::LayoutError, Trait::Debug) => rt("layoutErrorDebug", vec![value, f]), @@ -577,6 +578,7 @@ fn iter_debug<'tcx>( let path: Vec<&str> = path.iter().map(String::as_str).collect(); let (item, wraps): (Item, &[Wrap]) = match path.as_slice() { ["str", "iter", "Chars"] => return Some(Expr::rt("fmtChars", vec![value.clone(), f.clone()])), + ["str", "lossy", "Utf8Chunks"] => return Some(Expr::rt("fmtUtf8Chunks", vec![value.clone(), f.clone()])), // Without its fields, which neither `#` nor a width changes. ["str", "iter", "EncodeUtf16"] => { return Some(Expr::rt( diff --git a/ts/src/lower/iter.rs b/ts/src/lower/iter.rs index 9538ff48c..d45a5d510 100644 --- a/ts/src/lower/iter.rs +++ b/ts/src/lower/iter.rs @@ -1639,6 +1639,7 @@ fn clones<'tcx>(b: &mut Body<'_, '_, 'tcx>, ty: Ty<'tcx>, span: Span) -> Option< Some(match path.as_slice() { // Those that borrow what they read, as their clones do too, or hold nothing. ["iter", "sources", "empty", "Empty"] + | ["str", "lossy", "Utf8Chunks"] | [ "slice", "iter", diff --git a/ts/src/lower/library.rs b/ts/src/lower/library.rs index 9979bac44..f21498de2 100644 --- a/ts/src/lower/library.rs +++ b/ts/src/lower/library.rs @@ -52,6 +52,8 @@ pub enum Std { TryFromCharError, /// `rt.DecodeUtf16Error`: `{ code }`, the unpaired surrogate. DecodeUtf16Error, + /// `rt.Utf8Chunk`: `{ valid, invalid }`, never changed once made. + Utf8Chunk, ParseFloatError, TryFromFloatSecsError, /// `rt.Formatter`. @@ -192,8 +194,9 @@ pub enum Std { } impl Std { - /// Whether it is one of std's errors, whose derived `PartialEq` lowers from its MIR. - pub fn error(self) -> bool { + /// Whether it is one of std's errors, or a `Utf8Chunk`, whose runtime fields have std's + /// names and whose derived `PartialEq` lowers from its MIR. + pub fn record(self) -> bool { matches!( self, Self::FromUtf8Error @@ -209,6 +212,7 @@ impl Std { | Self::TryFromFloatSecsError | Self::FmtError | Self::LayoutError + | Self::Utf8Chunk ) } @@ -321,6 +325,7 @@ impl Std { ["char", "convert", "CharTryFromError"] => Some(Self::CharTryFromError), ["char", "TryFromCharError"] => Some(Self::TryFromCharError), ["char", "decode", "DecodeUtf16Error"] => Some(Self::DecodeUtf16Error), + ["str", "lossy", "Utf8Chunk"] => Some(Self::Utf8Chunk), ["fmt", "Formatter"] => Some(Self::Formatter), ["fmt", "Arguments"] => Some(Self::Arguments), ["fmt", "Error"] => Some(Self::FmtError), @@ -839,9 +844,9 @@ pub fn may_from_mir(tcx: TyCtxt<'_>, did: DefId) -> bool { if parses { return true; } - // A std error's derived `PartialEq`, which compares its fields. + // A std error's or a `Utf8Chunk`'s derived `PartialEq`, which compares its fields. if let ty::Adt(adt, _) = self_ty.kind() - && Std::of(tcx, adt.did()).is_some_and(Std::error) + && Std::of(tcx, adt.did()).is_some_and(Std::record) && tcx.is_automatically_derived(imp) { return name == rustc_span::sym::eq; @@ -1461,6 +1466,13 @@ fn mapping<'tcx>( if string(tcx, to) && string(tcx, from.peel_refs()) { return Some(b.t.pointee_value(targ(gargs, 1), value)); } + // A `Vec` of a `String`'s or a `&str`'s UTF-8. + if b.t.std_of(to) == Some(Std::Vec) && string(tcx, from.peel_refs()) { + return Some(Expr::rt( + "stringIntoBytes", + vec![b.t.pointee_value(targ(gargs, 1), value)], + )); + } // A `Vec` is owned, and a reference to a slice, an array, or a `Vec` borrows its slice. if let Some(elem) = cow_slice(tcx, to) { let owned = b.t.std_of(targ(gargs, 1)) == Some(Std::Vec); @@ -1743,7 +1755,9 @@ fn mapping<'tcx>( Expr::rt("fromUtf8ErrorClone", vec![value]) } // Never changed once made. - ty::Adt(adt, _) if Std::of(tcx, adt.did()) == Some(Std::DecodeUtf16Error) => value, + ty::Adt(adt, _) if matches!(Std::of(tcx, adt.did()), Some(Std::DecodeUtf16Error | Std::Utf8Chunk)) => { + value + } // Another that holds nothing. ty::Adt(adt, _) if empty_iter(tcx, adt.did()) => b.zero_sized_std(ty, Vec::new(), span).ok()?, // Through the reference, a slice of the bytes. @@ -2456,6 +2470,19 @@ fn mapping<'tcx>( // A `Box`'s bytes are their own, as `as_bytes` gives them. ("String" | "str", "as_bytes") | ("str", "into_boxed_bytes") => Expr::rt("stringAsBytes", args), ("String", "from_utf8") => Expr::rt("stringFromUtf8", args), + ("String", "into_bytes") => Expr::rt("stringIntoBytes", args), + // A `Cow` is its text, whether std borrows the bytes or replaces any. + ("String", "from_utf8_lossy") => Expr::rt("stringFromUtf8Lossy", args), + // UTF-8 by their contract, which the lossy decoder reads as the text. + ("converts" | "str", "from_utf8_unchecked") => Expr::rt("stringFromUtf8Lossy", args), + ("String", "from_utf8_unchecked") => { + Expr::rt("stringFromUtf8Lossy", vec![Expr::rt("byteVecSlice", vec![args.pop()?])]) + } + ("FromUtf8Error", "as_bytes") => Expr::rt("byteVecSlice", vec![args.pop()?.member("bytes")]), + ("FromUtf8Error", "into_bytes") => args.pop()?.member("bytes"), + ("FromUtf8Error", "utf8_error") => args.pop()?.member("error"), + ("slice", "utf8_chunks") => Expr::rt("byteSliceUtf8Chunks", args), + ("Utf8Chunk", "valid" | "invalid") => args.pop()?.member(&name), ( "String", "from_utf16" | "from_utf16_lossy" | "from_utf16le" | "from_utf16le_lossy" | "from_utf16be" diff --git a/ts/src/lower/mir/place.rs b/ts/src/lower/mir/place.rs index 5d7d99d46..ff3da13ec 100644 --- a/ts/src/lower/mir/place.rs +++ b/ts/src/lower/mir/place.rs @@ -622,7 +622,7 @@ impl<'m, 'b, 'a, 'tcx> Fx<'m, 'b, 'a, 'tcx> { // `TryFromIntError`'s `IntErrorKind`. Some(Std::TryFromIntError) => base.member("kind"), // The runtime's field of each of std's errors has its name. - Some(std) if std.error() => { + Some(std) if std.record() => { base.member(crate::lower::camel(adt.non_enum_variant().fields[field].name.as_str())) } // Cast to its variant's class: the slice an `rt.Borrowed` wraps, or the `Vec`. diff --git a/ts/src/lower/mod.rs b/ts/src/lower/mod.rs index 5e22cdf29..831d0b7b2 100644 --- a/ts/src/lower/mod.rs +++ b/ts/src/lower/mod.rs @@ -1661,6 +1661,7 @@ impl<'a, 'tcx> Translator<'a, 'tcx> { library::Std::CharTryFromError => Type::Rt("CharTryFromError", vec![]), library::Std::TryFromCharError => Type::Rt("TryFromCharError", vec![]), library::Std::DecodeUtf16Error => Type::Rt("DecodeUtf16Error", vec![]), + library::Std::Utf8Chunk => Type::Rt("Utf8Chunk", vec![]), library::Std::ParseFloatError => Type::Rt("ParseFloatError", vec![]), library::Std::TryFromFloatSecsError => Type::Rt("TryFromFloatSecsError", vec![]), library::Std::Formatter => Type::Rt("Formatter", vec![]),