Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions app/components/EmailPanel.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ import { useShortcuts } from "~/hooks/useShortcuts";
import { useUIStore } from "~/hooks/useUIStore";
import { plural, t } from "~/i18n";
import { unsubscribeOf } from "~/lib/unsubscribe";
import { splitEmailList, toEmailListValue } from "~/lib/utils";
import { htmlToPlainText, splitEmailList, toEmailListValue } from "~/lib/utils";
import { useDeleteEmail, useEmail, useThreadReplies } from "~/queries/emails";
import { useMailbox } from "~/queries/mailboxes";
import api from "~/services/api";
Expand Down Expand Up @@ -322,7 +322,7 @@ export default function EmailPanel({ emailId, siblings = [], inPane = false }: E
from,
subject: target.subject || t("(no subject)"),
html: target.body || "",
text: target.body ? target.body.replace(/<[^>]*>/g, "").trim() : "",
text: target.body ? htmlToPlainText(target.body) : "",
};
// The mailbox takes it; the notes that follow, Undo and what became of it, are the
// hook's. Nothing here says "sent" (ADR-0016).
Expand Down
28 changes: 7 additions & 21 deletions app/lib/utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
*/
import DOMPurify from "dompurify";
import { formatQuotedDate } from "shared/dates";
import { htmlToText } from "shared/html-text";
import { oneLineTextOf, plainTextOf } from "shared/html-text";
import { quoteLanguage, t } from "../i18n";

export {
Expand Down Expand Up @@ -57,37 +57,23 @@ export function toEmailListValue(addresses: string[]): string | string[] | undef
* Convert HTML content to plain text, keeping line breaks between blocks.
*/
export function htmlToPlainText(html: string): string {
// No DOM and no HTML injection: script and style content is never text (shared/html-text).
return decodeHtmlEntities(htmlToText(html, { keepLineBreaks: true }));
// No DOM and no HTML injection: script and style content is never text, and each character
// reference is decoded once (shared/html-text).
return plainTextOf(html);
}

/**
* Strip all HTML tags from a string.
*/
export function stripHtml(html: string): string {
return htmlToText(html);
}

function decodeHtmlEntities(text: string): string {
return text
.replace(/&#(\d+);/g, (_match: string, code: string) => String.fromCharCode(Number(code)))
.replace(/&#x([0-9a-f]+);/gi, (_match: string, hex: string) =>
String.fromCharCode(Number.parseInt(hex, 16)),
)
.replace(/&amp;/g, "&")
.replace(/&lt;/g, "<")
.replace(/&gt;/g, ">")
.replace(/&quot;/g, '"')
.replace(/&#39;/g, "'")
.replace(/&apos;/g, "'")
.replace(/&nbsp;/g, " ");
return oneLineTextOf(html);
}

export function getSnippetText(snippet?: string | null, maxLength = 100): string {
if (!snippet) return "";

// htmlToText never shows script or style content, even when the snippet is cut mid-element.
const clean = decodeHtmlEntities(htmlToText(snippet)).replace(/\s+/g, " ").trim();
// Never script or style content, even when the snippet is cut mid-element.
const clean = oneLineTextOf(snippet);

if (!clean) return "";
return clean.length > maxLength ? `${clean.slice(0, maxLength)}...` : clean;
Expand Down
16 changes: 11 additions & 5 deletions scripts/dev-setup.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,18 +9,24 @@
*/
import { execFileSync } from "node:child_process";
import { randomBytes } from "node:crypto";
import { existsSync, readFileSync, writeFileSync } from "node:fs";
import { readFileSync, writeFileSync } from "node:fs";

const DEV_VARS = ".dev.vars";

function ensureDevVars(): string {
if (!existsSync(DEV_VARS)) {
const secret = randomBytes(32).toString("hex");
const setup = randomBytes(24).toString("hex");
writeFileSync(DEV_VARS, `AUTH_SECRET=${secret}\nSETUP_TOKEN=${setup}\n`, { mode: 0o600 });
const secret = randomBytes(32).toString("hex");
const setup = randomBytes(24).toString("hex");
try {
// Created only if it is not there: the developer's own file is never written over.
writeFileSync(DEV_VARS, `AUTH_SECRET=${secret}\nSETUP_TOKEN=${setup}\n`, {
mode: 0o600,
flag: "wx",
});
process.stdout.write(
`dev-setup: wrote ${DEV_VARS} with a new session secret and setup token\n`,
);
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error;
}
const match = /^SETUP_TOKEN=(.+)$/m.exec(readFileSync(DEV_VARS, "utf8"));
return match?.[1]?.trim() ?? "";
Expand Down
12 changes: 11 additions & 1 deletion scripts/e2e-server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,16 @@ const QUIET = { ...process.env, CI: "1", WRANGLER_SEND_METRICS: "false" };
/** What the build put next to the Worker, so it can be put back when the tests are done. */
let builtVarsBefore: string | null = null;

/** A file's text, or null when it is not there: read without asking first, so nothing can change between. */
function readIfThere(path: string): string | null {
try {
return readFileSync(path, "utf8");
} catch (error) {
if ((error as NodeJS.ErrnoException).code === "ENOENT") return null;
throw error;
}
}

function restore(): void {
if (builtVarsBefore !== null) writeFileSync(BUILT_VARS, builtVarsBefore, { mode: 0o600 });
else rmSync(BUILT_VARS, { force: true });
Expand All @@ -45,7 +55,7 @@ function prepare(): void {
};
// The build copied the developer's local variables next to the Worker. They are replaced
// for the run and put back afterwards.
builtVarsBefore = existsSync(BUILT_VARS) ? readFileSync(BUILT_VARS, "utf8") : null;
builtVarsBefore = readIfThere(BUILT_VARS);
writeFileSync(
BUILT_VARS,
`AUTH_SECRET=${secrets.authSecret}\nSETUP_TOKEN=${secrets.setupToken}\n`,
Expand Down
2 changes: 1 addition & 1 deletion scripts/pins.ts
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ export function newestSettled(
}

async function settled(name: string, now: number): Promise<string | undefined> {
const res = await fetch(`${REGISTRY}/${name.replace("/", "%2F")}`, {
const res = await fetch(`${REGISTRY}/${name.replaceAll("/", "%2F")}`, {
headers: { accept: "application/json" },
});
if (!res.ok) throw new Error(`${name}: registry answered ${res.status}`);
Expand Down
11 changes: 10 additions & 1 deletion shared/html-text.ts
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ const NAMED: Record<string, string> = {
};

/** Character references back to characters, each once: what one makes is not read again. */
function decodeEntities(text: string): string {
export function decodeEntities(text: string): string {
return text.replace(
/&(#\d{1,7}|#x[0-9a-f]{1,6}|lt|gt|quot|apos|nbsp|amp);/gi,
(match: string, name: string) => {
Expand All @@ -159,3 +159,12 @@ function decodeEntities(text: string): string {
export function plainTextOf(html: string): string {
return decodeEntities(htmlToText(html, { keepLineBreaks: true }));
}

/**
* The text of an HTML fragment on one line, its character references decoded once: for what a
* model, a tool or a quoted reply is given. Text, never markup: whatever uses it as HTML
* escapes it first.
*/
export function oneLineTextOf(html: string): string {
return decodeEntities(htmlToText(html));
}
4 changes: 3 additions & 1 deletion test/unit/content-policy.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,9 @@ describe("the frame's sizing script", () => {
allowRemote: false,
nonce,
});
const scripts = [...html.matchAll(/<script[^>]*>([\s\S]*?)<\/script>/g)].map((m) => m[1]);
const scripts = [...html.matchAll(/<script\b[^>]*>([\s\S]*?)<\/script\b[^>]*>/gi)].map(
(m) => m[1],
);
expect(scripts).toEqual([MAIL_FRAME_SCRIPT]);
const without = mailDocument({
sanitizedBody: "<p>Hi</p>",
Expand Down
51 changes: 51 additions & 0 deletions test/unit/html-text.test.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
import { describe, expect, it } from "vitest";
import { getSnippetText, htmlToPlainText, stripHtml } from "../../app/lib/utils.ts";
import { htmlToText } from "../../shared/html-text.ts";
import { buildQuotedReplyBlock, stripHtmlToText } from "../../workers/lib/email-helpers.ts";

describe("htmlToText", () => {
it("keeps the text and drops the tags", () => {
Expand Down Expand Up @@ -77,3 +79,52 @@ describe("htmlToText with line breaks kept", () => {
expect(text.endsWith("b")).toBe(true);
});
});

describe("the text Podbay makes of a stranger's HTML", () => {
// The forms a regular expression over markup misses: an end tag with a space or a line
// break in it, capitals, a tag inside a tag, and a character reference written twice.
const HOSTILE = [
"<p>before</p><script >steal()</script ><p>after</p>",
"<p>before</p><SCRIPT>steal()</SCRIPT><p>after</p>",
"<p>before</p><script>steal()</script\n><p>after</p>",
"<p>before</p><style >p{}</style ><p>after</p>",
];
// A tag inside a tag is one tag named "scr<script", as a browser reads it: what follows is
// text, and no tag is left to put back together.
const NESTED = "<p>before</p><scr<script>ipt>steal()</script><p>after</p>";
const makers = {
"the Worker's text for a model, a tool and a quote": stripHtmlToText,
"the app's one-line text": stripHtml,
"the app's list snippet": (html: string) => getSnippetText(html, 500),
"the app's text part of a message": htmlToPlainText,
};

for (const [who, make] of Object.entries(makers)) {
it(`${who}: never keeps script or style content, and never a tag`, () => {
for (const html of HOSTILE) {
const text = make(html);
expect(text, html).toContain("before");
expect(text, html).not.toContain("steal()");
expect(text, html).not.toContain("p{}");
expect(text, html).not.toMatch(/<\/?[a-z]/i);
}
expect(make(NESTED)).not.toMatch(/<\/?[a-z]/i);
});

it(`${who}: decodes each character reference once`, () => {
expect(make("<p>a &amp;lt;b&amp;gt; c</p>")).toBe("a &lt;b&gt; c");
expect(make("<p>caf&#233; &amp; co</p>")).toBe("café & co");
});
}

it("a quoted reply shows what was written as text, and the reply holds no element of it", () => {
const quoted = buildQuotedReplyBlock({
sender: "ada@sender.test",
date: "2026-11-18T10:00:00Z",
body: "<p>&lt;script&gt;steal()&lt;/script&gt;</p><script >steal()</script >",
});
expect(quoted).toContain("&lt;script&gt;steal()&lt;/script&gt;");
expect(quoted).not.toContain("<script");
expect(quoted.match(/steal\(\)/g)).toHaveLength(1);
});
});
1 change: 1 addition & 0 deletions tsconfig.test.json
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
"app/lib/user-agent.ts",
"app/lib/session.ts",
"app/lib/live.ts",
"app/lib/utils.ts",
"app/hooks/useDisplay.ts",
"app/services/api.ts",
"app/types/**/*.ts",
Expand Down
15 changes: 6 additions & 9 deletions workers/lib/email-helpers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

import { formatQuotedDate } from "../../shared/dates";
import { Folders } from "../../shared/folders";
import { oneLineTextOf } from "../../shared/html-text";
/**
* Shared email helpers to eliminate duplication across API routes, MCP, and agent.
*
Expand Down Expand Up @@ -156,18 +157,14 @@ export function textToHtml(text: string): string {
}

/**
* Strip HTML tags and normalize whitespace to produce plain text.
* Removes <style> and <script> blocks first to avoid injecting their
* content into the output.
* The text of an HTML body on one line, for a model, a tool and a quoted reply. One scan of the
* markup (shared/html-text), not regular expressions over it: the content of a script or a
* style is never text, however its tags are written, and each character reference is decoded
* once. The result is text; a quoted reply escapes it before it becomes HTML.
*/
export function stripHtmlToText(html: string): string {
if (!html) return "";
return html
.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "")
.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "")
.replace(/<[^>]+>/g, " ")
.replace(/\s+/g, " ")
.trim();
return oneLineTextOf(html);
}

/**
Expand Down
Loading