From 4e64fbfb0fbd7f54fce813350fe2133499546786 Mon Sep 17 00:00:00 2001 From: Christian Date: Wed, 30 Sep 2026 16:29:27 -0500 Subject: [PATCH 01/15] Compose streamed Next.js and GTM script rewrites Apply matching script stages in sequence and emit each fragment once. Keep GTM buffering response-local and bounded, and rewrite captured Flight payloads only after reading their original record boundaries. Preserve unsafe originals and recompute decoded record lengths. Cover Pages and App Router data, bootstrap controls, escaped queries, source-order fallback, and complete identity/gzip adapter responses. Refs #1206, #1207, #1208 --- .../tests/routes.rs | 45 + .../trusted-server-adapter-fastly/src/app.rs | 46 + .../trusted-server-core/src/html_processor.rs | 150 +++- .../src/integrations/google_tag_manager.rs | 383 +++++++- .../src/integrations/nextjs/mod.rs | 823 +++++++++++++++++- .../src/integrations/nextjs/rsc.rs | 156 +++- .../integrations/nextjs/rsc_placeholders.rs | 595 +++++++++++-- .../src/integrations/nextjs/rsc_stream.rs | 79 +- .../integrations/nextjs/script_rewriter.rs | 8 +- .../src/integrations/registry.rs | 10 +- .../trusted-server-core/src/test_support.rs | 172 +++- 11 files changed, 2276 insertions(+), 191 deletions(-) diff --git a/crates/trusted-server-adapter-axum/tests/routes.rs b/crates/trusted-server-adapter-axum/tests/routes.rs index 7126b2a714..42d1b1b6b5 100644 --- a/crates/trusted-server-adapter-axum/tests/routes.rs +++ b/crates/trusted-server-adapter-axum/tests/routes.rs @@ -7,8 +7,10 @@ use axum::body::Body as AxumBody; use axum::http::Request; use edgezero_adapter_axum::service::EdgeZeroAxumService; +use std::sync::Arc; use tower::{Service as _, ServiceExt as _}; use trusted_server_adapter_axum::app::TrustedServerApp; +use trusted_server_core::test_support::nextjs_auction; const LEGACY_ADMIN_DENY_METHODS: &[&str] = &["GET", "POST", "HEAD", "OPTIONS", "PUT", "PATCH", "DELETE"]; @@ -938,3 +940,46 @@ async fn nextjs_auction_output_holds_until_the_structural_body_close() { "should not leak generated placeholders: {html}" ); } + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn nextjs_gtm_html_response_is_complete() { + for gzip in [false, true] { + let origin = Arc::new(nextjs_auction::NextJsAuctionOrigin::with_html_response( + &nextjs_auction::script_composition_html(), + gzip, + false, + )); + let router = TrustedServerApp::routes_with_settings_and_services( + nextjs_auction::script_composition_settings(), + nextjs_auction::services(origin), + ) + .expect("should build mixed integration router"); + let mut service = EdgeZeroAxumService::new(router); + let request = Request::builder() + .method("GET") + .uri("/article") + .header("host", nextjs_auction::PUBLISHER_HOST) + .header("accept", "text/html") + .body(AxumBody::empty()) + .expect("should build navigation"); + let response = service + .ready() + .await + .expect("should be ready") + .call(request) + .await + .expect("should serve HTML"); + assert_eq!(response.status().as_u16(), 200); + assert_eq!( + response + .headers() + .get("content-encoding") + .map(|value| value.to_str().expect("should have ASCII coding")), + gzip.then_some("gzip") + ); + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .expect("should consume entire adapter body"); + nextjs_auction::assert_script_composition_response(&body, gzip); + } +} diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index fcb4c28562..0e7eb08363 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -1352,6 +1352,7 @@ mod tests { use std::net::{IpAddr, Ipv4Addr}; use error_stack::Report; + use futures::StreamExt as _; use futures::executor::block_on; use trusted_server_core::constants::HEADER_X_GEO_INFO_AVAILABLE; use trusted_server_core::ec::device::DeviceSignals; @@ -1368,6 +1369,7 @@ mod tests { TemplateCacheMiss, TemplateCacheReservation, TemplateEntry, TemplateMetadata, }; use trusted_server_core::settings::Settings; + use trusted_server_core::test_support::nextjs_auction; #[test] fn hooks_store_metadata_matches_edgezero_manifest() { @@ -3092,6 +3094,50 @@ mod tests { ); } + #[test] + fn dispatch_fallback_nextjs_gtm_response_is_complete() { + for gzip in [false, true] { + let state = build_state_from_settings(nextjs_auction::script_composition_settings()) + .expect("should build mixed state"); + let origin = Arc::new(nextjs_auction::NextJsAuctionOrigin::with_html_response( + &nextjs_auction::script_composition_html(), + gzip, + true, + )); + let services = nextjs_auction::services(origin); + let request = request_builder() + .method(Method::GET) + .uri(format!( + "https://{}/article", + nextjs_auction::PUBLISHER_HOST + )) + .header("host", nextjs_auction::PUBLISHER_HOST) + .header("accept", "text/html") + .body(Body::empty()) + .expect("should build navigation"); + let response = block_on(super::dispatch_fallback(&state, &services, request)); + assert_eq!(response.status(), StatusCode::OK); + assert!(!response.headers().contains_key(header::CONTENT_LENGTH)); + assert_eq!( + response + .headers() + .get(header::CONTENT_ENCODING) + .map(|value| value.to_str().expect("should have ASCII coding")), + gzip.then_some("gzip") + ); + let Body::Stream(mut stream) = response.into_body() else { + panic!("should retain lazy Fastly body"); + }; + let mut bytes = Vec::new(); + block_on(async { + while let Some(chunk) = stream.next().await { + bytes.extend_from_slice(&chunk.expect("should process every stream item")); + } + }); + nextjs_auction::assert_script_composition_response(&bytes, gzip); + } + } + #[test] fn dispatch_fallback_streams_publisher_body_without_buffering() { // Regression guard for the publisher streaming cutover (#849): a diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 3889d7d592..79a2bfaa6f 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -1,7 +1,8 @@ //! Simplified HTML processor that combines URL replacement and integration injection //! //! This module provides a `StreamProcessor` implementation for HTML content. -use std::cell::Cell; +use std::borrow::Cow; +use std::cell::{Cell, RefCell}; use std::io; use std::rc::Rc; use std::sync::Arc; @@ -661,38 +662,62 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso })); } - for script_rewriter in script_rewriters { - let selector = script_rewriter.selector(); - let rewriter = script_rewriter.clone(); - let patterns = patterns.clone(); - let document_state = document_state.clone(); - element_content_handlers.push(text!(selector, { - let rewriter = rewriter.clone(); - let patterns = patterns.clone(); - let document_state = document_state.clone(); - move |text| { - let ctx = IntegrationScriptContext { - selector, - request_host: &patterns.request_host, - request_scheme: &patterns.request_scheme, - origin_host: &patterns.origin_host, - is_last_in_text_node: text.last_in_text_node(), - max_buffered_script_bytes: config.max_buffered_body_bytes, - document_state: &document_state, - }; - match rewriter.rewrite(text.as_str(), &ctx) { - ScriptRewriteAction::Keep => {} - ScriptRewriteAction::Replace(rewritten) => { - text.replace(&rewritten, ContentType::Text); - } - ScriptRewriteAction::RemoveNode => { - text.remove(); - } - } - Ok(()) - } + // Selector callbacks retain lol_html's CSS matching; only the composed text + // callback mutates a fragment. Start each script with fresh match state. + let script_matches = Rc::new(RefCell::new(vec![false; script_rewriters.len()])); + let reset_matches = script_matches.clone(); + element_content_handlers.push(element!("script", move |_el| { + reset_matches.borrow_mut().fill(false); + Ok(()) + })); + for (index, rewriter) in script_rewriters.iter().enumerate() { + let matches = script_matches.clone(); + element_content_handlers.push(element!(rewriter.selector(), move |_el| { + matches.borrow_mut()[index] = true; + Ok(()) })); } + let script_patterns = patterns.clone(); + let script_document_state = document_state.clone(); + element_content_handlers.push(text!("script", move |text| { + let mut working = Cow::Borrowed(text.as_str()); + let mut changed = false; + for (rewriter, matched) in script_rewriters.iter().zip(script_matches.borrow().iter()) { + if !matched { + continue; + } + let ctx = IntegrationScriptContext { + selector: rewriter.selector(), + request_host: &script_patterns.request_host, + request_scheme: &script_patterns.request_scheme, + origin_host: &script_patterns.origin_host, + is_last_in_text_node: text.last_in_text_node(), + max_buffered_script_bytes: config.max_buffered_body_bytes, + document_state: &script_document_state, + }; + match rewriter.rewrite(&working, &ctx) { + ScriptRewriteAction::Keep => {} + ScriptRewriteAction::Replace(rewritten) => { + working = Cow::Owned(rewritten); + changed = true; + } + ScriptRewriteAction::RemoveNode => { + working = Cow::Borrowed(""); + changed = true; + } + } + } + if changed { + if working.is_empty() { + text.remove(); + } else if let Cow::Owned(rewritten) = working { + // Script is raw text: HTML entities would change JS/JSON data. + // Emit composed source verbatim rather than escaping it again. + text.replace(&rewritten, ContentType::Html); + } + } + Ok(()) + })); let rewriter_settings = RewriterSettings { document_content_handlers, @@ -723,7 +748,7 @@ mod tests { use super::*; use crate::integrations::{ AttributeRewriteAction, IntegrationAttributeContext, IntegrationAttributeRewriter, - IntegrationHeadInjector, IntegrationHtmlContext, + IntegrationHeadInjector, IntegrationHtmlContext, IntegrationScriptRewriter, }; use crate::streaming_processor::{Compression, PipelineConfig, StreamingPipeline}; use crate::test_support::tests::create_test_settings; @@ -752,6 +777,67 @@ mod tests { } } + struct SuppressingScript; + impl IntegrationScriptRewriter for SuppressingScript { + fn integration_id(&self) -> &'static str { + "suppress" + } + fn selector(&self) -> &'static str { + "script[data-suppress='yes']" + } + fn rewrite(&self, _text: &str, ctx: &IntegrationScriptContext<'_>) -> ScriptRewriteAction { + assert_eq!(ctx.selector, self.selector()); + ScriptRewriteAction::RemoveNode + } + } + + struct FlushingScript; + impl IntegrationScriptRewriter for FlushingScript { + fn integration_id(&self) -> &'static str { + "flush" + } + fn selector(&self) -> &'static str { + "script#flush" + } + fn rewrite(&self, text: &str, ctx: &IntegrationScriptContext<'_>) -> ScriptRewriteAction { + assert_eq!(ctx.selector, self.selector()); + assert!(text.is_empty(), "should see upstream suppression"); + if ctx.is_last_in_text_node { + ScriptRewriteAction::Replace("flushed-once".into()) + } else { + ScriptRewriteAction::Keep + } + } + } + + #[test] + fn script_actions_compose_with_selector_reset_and_empty_final_flush() { + let mut config = create_test_config(); + config.integrations = IntegrationRegistry::from_rewriters( + Vec::new(), + vec![Arc::new(SuppressingScript), Arc::new(FlushingScript)], + ); + let mut processor = create_html_processor(config); + let mut output = processor + .process_chunk(b"", true) + .expect("should finalize all matching stages"), + ); + let output = String::from_utf8(output).expect("should emit UTF-8"); + assert!( + output.contains(""), + "should flush downstream even on suppressed final: {output}" + ); + assert!( + output.contains(""), + "should reset CSS match state between scripts" + ); + assert_eq!(output.matches("flushed-once").count(), 1); + } + #[test] fn integration_attribute_rewriter_can_remove_elements() { struct RemovingLinkRewriter; diff --git a/crates/trusted-server-core/src/integrations/google_tag_manager.rs b/crates/trusted-server-core/src/integrations/google_tag_manager.rs index ae9ceb7cb3..6661df1a67 100644 --- a/crates/trusted-server-core/src/integrations/google_tag_manager.rs +++ b/crates/trusted-server-core/src/integrations/google_tag_manager.rs @@ -143,6 +143,91 @@ static GTM_QUOTED_URL_PATTERNS: LazyLock> = LazyLock::new(|| { .collect() }); +// Flight spans are JS string source, not decoded JSON. Match only whole or +// quoted routed URL tokens and preserve their delimiters, suffix and slash layer. +static GTM_RSC_WHOLE_URL_PATTERN: LazyLock = LazyLock::new(|| { + Regex::new( + r"^(?:https?:)?(?P\\{0,8}/)(?:\\{0,8}/)(?:www\.(?:googletagmanager|google-analytics)\.com|analytics\.google\.com)(?P(?:\\{0,8}/)(?:gtm\.js|gtag(?:\\{0,8}/|\.)js|g(?:\\{0,8}/)collect|collect)|)(?P[?#].*)?$" + ).expect("should compile whole Flight URL pattern") +}); +static GTM_RSC_QUOTED_URL_START: LazyLock = LazyLock::new(|| { + Regex::new(r#"(?P\\{0,8}["'`])(?:https?:)?\\{0,8}/\\{0,8}/(?:www\.(?:googletagmanager|google-analytics)\.com|analytics\.google\.com)"#) + .expect("should compile quoted Flight URL start") +}); + +// Match the opening quote's escape layer. Higher-layer instances of that +// delimiter belong to the URL (e.g. escaped quotes inside a JSON query). +fn flight_url_token_end( + content: &str, + start: usize, + delimiter: u8, + layer: usize, +) -> Option<(usize, usize)> { + let bytes = content.as_bytes(); + let mut slash_run = 0; + for index in start..bytes.len() { + let byte = bytes[index]; + if byte == delimiter { + if slash_run == layer { + return Some((index - slash_run, index + 1)); + } + if slash_run < layer { + return None; + } + } else if delimiter == b'`' && byte == b'$' { + return (bytes.get(index + 1) == Some(&b'{')).then_some((index, index + 2)); + } + slash_run = if byte == b'\\' { slash_run + 1 } else { 0 }; + } + None +} + +/// Rewrite supported GTM/GA tokens in a captured Flight source span. +/// +/// Call only after original T-record boundaries are known, before lengths are +/// recomputed. Originals used for fallback must never pass through this function. +pub(crate) fn rewrite_gtm_rsc_span(content: &str) -> String { + fn route(token: &str) -> Option { + let captures = GTM_RSC_WHOLE_URL_PATTERN.captures(token)?; + let slash = &captures["slash"]; + let path = &captures["path"]; + let suffix = captures.name("suffix").map_or("", |value| value.as_str()); + Some(format!( + "{slash}integrations{slash}google_tag_manager{path}{suffix}" + )) + } + if let Some(rewritten) = route(content) { + return rewritten; + } + let mut rewritten = String::with_capacity(content.len()); + let mut cursor = 0; + let mut search = 0; + while let Some(captures) = GTM_RSC_QUOTED_URL_START.captures(&content[search..]) { + let open = captures.name("open").expect("should capture quote layer"); + let open_start = search + open.start(); + let url_start = search + open.end(); + let delimiter = content.as_bytes()[url_start - 1]; + let layer = open.len() - 1; + search = url_start; + // Never accept a suffix of an unsupported, deeper opening layer. + if open_start > 0 && content.as_bytes()[open_start - 1] == b'\\' { + continue; + } + let Some((url_end, token_end)) = flight_url_token_end(content, url_start, delimiter, layer) + else { + continue; + }; + search = token_end; + if let Some(url) = route(&content[url_start..url_end]) { + rewritten.push_str(&content[cursor..url_start]); + rewritten.push_str(&url); + cursor = url_end; + } + } + rewritten.push_str(&content[cursor..]); + rewritten +} + #[derive(Debug, Clone, Deserialize, Serialize, Validate)] pub struct GoogleTagManagerConfig { #[serde(default = "default_enabled")] @@ -356,14 +441,12 @@ pub struct GoogleTagManagerIntegration { /// allowlist as "any host", so without this a 3xx from the upstream would /// let an arbitrary origin's body be re-served as first-party JavaScript. proxy_allowed_domains: Vec, - /// Accumulates text fragments when `lol_html` splits a text node across - /// chunk boundaries. Drained on `is_last_in_text_node`. - /// - /// Uses `Mutex` to satisfy the `Sync` bound on `IntegrationScriptRewriter`. - /// The pipeline is single-threaded (`lol_html::HtmlRewriter` is `!Send`), - /// so the lock is uncontended. `lol_html` delivers text chunks sequentially - /// per element — the buffer is always empty when a new element's text begins. - accumulated_text: Mutex, +} + +#[derive(Default)] +struct GtmScriptState { + held: String, + bypass_until_last: bool, } impl GoogleTagManagerIntegration { @@ -372,7 +455,6 @@ impl GoogleTagManagerIntegration { Arc::new(Self { config, proxy_allowed_domains, - accumulated_text: Mutex::new(String::new()), }) } @@ -1029,39 +1111,57 @@ impl IntegrationScriptRewriter for GoogleTagManagerIntegration { } fn rewrite(&self, content: &str, ctx: &IntegrationScriptContext<'_>) -> ScriptRewriteAction { - let mut buf = self - .accumulated_text + let state = ctx + .document_state + .get_or_insert_with(GTM_INTEGRATION_ID, || Mutex::new(GtmScriptState::default())); + let mut state = state .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); - - // Cheap gate: only engage the accumulation path for scripts whose - // running text could plausibly contain a GTM/GA domain. Unrelated - // scripts (the vast majority) return Keep on every fragment so they - // stay visible in `lol_html`'s output unchanged — and, critically, - // so a Replace from this rewriter does not clobber a Replace from - // another rewriter on the same element (e.g., `NextJsNextDataRewriter` - // on `script#__NEXT_DATA__`). See `might_contain_gtm_prefix` for the - // boundary-safe substring check. - let prior_and_current_might_match = - might_contain_gtm_prefix(&buf) || might_contain_gtm_prefix(content); - - if !ctx.is_last_in_text_node { - // Intermediate fragment. Only accumulate + suppress when the script - // might still resolve to a GTM match. Otherwise return Keep so the - // fragment is emitted as-is by `lol_html` and untouched by us. - if prior_and_current_might_match { - buf.push_str(content); - return ScriptRewriteAction::RemoveNode; + if crate::integrations::nextjs::protects_current_flight_fragment(ctx.document_state) { + if ctx.is_last_in_text_node { + state.bypass_until_last = false; + } + if state.held.is_empty() { + return ScriptRewriteAction::Keep; } - return ScriptRewriteAction::keep(); + let mut restored = std::mem::take(&mut state.held); + restored.push_str(content); + return ScriptRewriteAction::Replace(restored); } - - // Last fragment. If we accumulated prior fragments, combine them. - let full_content: Option = if buf.is_empty() { + if state.bypass_until_last { + if ctx.is_last_in_text_node { + state.bypass_until_last = false; + } + return ScriptRewriteAction::Keep; + } + // Once holding begins, retain every following fragment in source order, + // even if the tentative hostname prefix turns out not to match. + if state.held.is_empty() && !might_contain_gtm_prefix(content) { + return ScriptRewriteAction::Keep; + } + if state + .held + .len() + .checked_add(content.len()) + .is_none_or(|length| length > ctx.max_buffered_script_bytes) + { + state.bypass_until_last = !ctx.is_last_in_text_node; + if state.held.is_empty() { + return ScriptRewriteAction::Keep; + } + let mut restored = std::mem::take(&mut state.held); + restored.push_str(content); + return ScriptRewriteAction::Replace(restored); + } + if !ctx.is_last_in_text_node { + state.held.push_str(content); + return ScriptRewriteAction::RemoveNode; + } + let full_content = if state.held.is_empty() { None } else { - buf.push_str(content); - Some(std::mem::take(&mut *buf)) + state.held.push_str(content); + Some(std::mem::take(&mut state.held)) }; let text = full_content.as_deref().unwrap_or(content); @@ -1098,6 +1198,7 @@ mod tests { }; use crate::platform::test_support::{StubHttpClient, build_services_with_http_client}; use crate::settings::Settings; + use crate::streaming_processor::StreamProcessor as _; use crate::streaming_processor::{Compression, PipelineConfig, StreamingPipeline}; #[test] @@ -3414,6 +3515,216 @@ container_id = "GTM-DEFAULT" } } + #[test] + fn captured_flight_preserves_quoted_query_delimiters_and_template_interpolation() { + for (source, expected) in [ + ( + r#"\"http://www.googletagmanager.com/gtm.js?q='text'\""#, + r#"\"/integrations/google_tag_manager/gtm.js?q='text'\""#, + ), + ( + r#"\"http://www.googletagmanager.com/gtm.js?q=`text`\""#, + r#"\"/integrations/google_tag_manager/gtm.js?q=`text`\""#, + ), + ( + r#"'http://www.googletagmanager.com/gtm.js?q=\"text\"'"#, + r#"'/integrations/google_tag_manager/gtm.js?q=\"text\"'"#, + ), + ( + r#"\"http://www.googletagmanager.com/gtm.js?q=\\\"text\\\"\""#, + r#"\"/integrations/google_tag_manager/gtm.js?q=\\\"text\\\"\""#, + ), + ( + r"'http://www.googletagmanager.com/gtm.js?q=\'text\''", + r"'/integrations/google_tag_manager/gtm.js?q=\'text\''", + ), + ( + "`http://www.googletagmanager.com/gtm.js?q=$literal`", + "`http://www.googletagmanager.com/gtm.js?q=$literal`", + ), + ( + "`http://www.googletagmanager.com/gtm.js${id}`", + "`/integrations/google_tag_manager/gtm.js${id}`", + ), + ] { + assert_eq!(rewrite_gtm_rsc_span(source), expected); + } + } + + #[test] + fn captured_flight_token_rewrite_preserves_escape_layers_and_allowed_paths() { + for slash in ["/", r"\/", r"\\/"] { + for quote in ["", "'", "\"", r#"\""#, "`"] { + let source = format!( + "{quote}http:{slash}{slash}www.googletagmanager.com{slash}gtm.js{quote}" + ); + let expected = format!( + "{quote}{slash}integrations{slash}google_tag_manager{slash}gtm.js{quote}" + ); + assert_eq!( + rewrite_gtm_rsc_span(&source), + expected, + "should preserve source escape layer: {source}" + ); + } + } + for source in [ + r#""https://www.googletagmanager.com/ns.html?id=GTM-MIX1""#, + "text http://www.googletagmanager.com/gtm.js text", + "'http://www.googletagmanager.com/gtm.js/path'", + "'http://www.googletagmanager.com/collect;matrix'", + "'http://www.googletagmanager.com.evil/gtm.js'", + "'http://www.googletagmanager.com/gtm.js\"", + ] { + assert_eq!( + rewrite_gtm_rsc_span(source), + source, + "should not route unsupported or unbounded token" + ); + } + } + + #[test] + fn inline_gtm_split_after_underscore_emits_once() { + let mut settings = make_settings(); + settings + .integrations + .insert_config( + "google_tag_manager", + &serde_json::json!({"enabled": true, "container_id": "GTM-MIX1"}), + ) + .expect("should enable GTM"); + settings + .integrations + .insert_config("nextjs", &serde_json::json!({"enabled": true})) + .expect("should enable Next.js"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new(crate::auction::compile_auction_plan(&settings).expect("should compile plan")), + ) + .expect("should build registry"); + let mut processor = create_html_processor(config_from_settings(&settings, ®istry)); + let mut bytes = processor + .process_chunk( + b"", true) + .expect("should consume final"), + ); + let html = String::from_utf8(bytes).expect("should emit UTF-8"); + assert!(html.contains("var url='/integrations/google_tag_manager/gtm.js';var tail_=1;")); + assert_eq!( + html.matches("var url=").count(), + 1, + "should emit script text once" + ); + } + + #[test] + fn sticky_gtm_four_fragments_preserve_order_after_false_google_prefix() { + let integration = GoogleTagManagerIntegration::new(tag_config("GTM-MIX1", &[])); + let state = IntegrationDocumentState::default(); + let fragments = [ + "var falsePrefix='google", + "X';first=1;", + "second=2;var tag='http://www.googletagmanager.com/gtm.js", + "';last=3;", + ]; + let mut emitted = String::new(); + for (index, fragment) in fragments.iter().enumerate() { + let context = IntegrationScriptContext { + selector: "script", + request_host: "test.example.com", + request_scheme: "https", + origin_host: "origin.example.com", + is_last_in_text_node: index == 3, + max_buffered_script_bytes: 1000, + document_state: &state, + }; + match IntegrationScriptRewriter::rewrite(&*integration, fragment, &context) { + ScriptRewriteAction::Keep => emitted.push_str(fragment), + ScriptRewriteAction::Replace(value) => emitted.push_str(&value), + ScriptRewriteAction::RemoveNode => {} + } + } + assert_eq!( + emitted, + "var falsePrefix='googleX';first=1;second=2;var tag='/integrations/google_tag_manager/gtm.js';last=3;" + ); + } + + #[test] + fn sticky_gtm_holding_is_bounded_and_document_local() { + let integration = GoogleTagManagerIntegration::new(tag_config("GTM-MIX1", &[])); + let a = IntegrationDocumentState::default(); + let b = IntegrationDocumentState::default(); + let context = |last, budget, state| IntegrationScriptContext { + selector: "script", + request_host: "test.example.com", + request_scheme: "https", + origin_host: "origin.example.com", + is_last_in_text_node: last, + max_buffered_script_bytes: budget, + document_state: state, + }; + let fragments = ["prefix google", "X-middle", "later", "-tail"]; + let mut emitted = String::new(); + for (i, fragment) in fragments.iter().enumerate() { + let action = IntegrationScriptRewriter::rewrite( + &*integration, + fragment, + &context(i == 3, 100, &a), + ); + match action { + ScriptRewriteAction::Keep => emitted.push_str(fragment), + ScriptRewriteAction::Replace(value) => emitted.push_str(&value), + ScriptRewriteAction::RemoveNode => {} + } + if i == 0 { + assert_eq!( + IntegrationScriptRewriter::rewrite( + &*integration, + "other", + &context(true, 100, &b) + ), + ScriptRewriteAction::Keep, + "should isolate documents" + ); + } + } + assert_eq!( + emitted, + fragments.concat(), + "should preserve source order after false prefix" + ); + assert_eq!( + IntegrationScriptRewriter::rewrite(&*integration, "google", &context(false, 6, &a)), + ScriptRewriteAction::RemoveNode + ); + assert_eq!( + IntegrationScriptRewriter::rewrite(&*integration, "X", &context(false, 6, &a)), + ScriptRewriteAction::Replace("googleX".into()), + "should restore on overflow" + ); + assert_eq!( + IntegrationScriptRewriter::rewrite(&*integration, "tail", &context(true, 6, &a)), + ScriptRewriteAction::Keep + ); + assert_eq!( + IntegrationScriptRewriter::rewrite(&*integration, "google", &context(false, 6, &a)), + ScriptRewriteAction::RemoveNode + ); + assert_eq!( + IntegrationScriptRewriter::rewrite(&*integration, "", &context(true, 6, &a)), + ScriptRewriteAction::Replace("google".into()), + "should flush empty final at exact limit" + ); + } + #[test] fn fragmented_gtm_snippet_is_accumulated_and_rewritten() { let config = tag_config("GTM-FRAG1", &[]); diff --git a/crates/trusted-server-core/src/integrations/nextjs/mod.rs b/crates/trusted-server-core/src/integrations/nextjs/mod.rs index f6f36f67c4..4036e5e768 100644 --- a/crates/trusted-server-core/src/integrations/nextjs/mod.rs +++ b/crates/trusted-server-core/src/integrations/nextjs/mod.rs @@ -63,6 +63,21 @@ pub(super) fn configuration_error(message: impl Into) -> Report bool { + state + .get::>(NEXTJS_INTEGRATION_ID) + .is_some_and(|state| { + state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .current_fragment_protected + }) +} + /// Register the Next.js integration when enabled. /// /// # Errors @@ -91,7 +106,18 @@ pub fn register( // then substitute them through the bounded output stream processor. let placeholders = Arc::new(NextJsRscPlaceholderRewriter::new(config.clone())); - let stream_processor = Arc::new(NextJsRscStreamProcessorFactory::new(config.clone())); + let gtm_transform = settings + .integration_config::( + "google_tag_manager", + )? + .filter(|config| config.enabled) + .map(|_| { + crate::integrations::google_tag_manager::rewrite_gtm_rsc_span as fn(&str) -> String + }); + let stream_processor = Arc::new(NextJsRscStreamProcessorFactory::new( + config.clone(), + gtm_transform, + )); let builder = IntegrationRegistration::builder(NEXTJS_INTEGRATION_ID) .with_script_rewriter(structured) @@ -115,10 +141,13 @@ mod tests { use super::*; use crate::html_processor::{HtmlProcessorConfig, create_html_processor}; use crate::integrations::IntegrationRegistry; - use crate::streaming_processor::{Compression, PipelineConfig, StreamingPipeline}; + use crate::streaming_processor::{ + Compression, PipelineConfig, StreamProcessor as _, StreamingPipeline, + }; use crate::test_support::tests::create_test_settings; + use flate2::{read::GzDecoder, write::GzEncoder}; use serde_json::json; - use std::io::Cursor; + use std::io::{Cursor, Read as _, Write as _}; fn config_from_settings( settings: &Settings, @@ -133,6 +162,794 @@ mod tests { ) } + fn mixed_output(html: &str, chunk_size: usize) -> String { + mixed_output_with_budget(html, chunk_size, 10 * 1024 * 1024, 16 * 1024 * 1024) + } + + fn mixed_config(group_limit: usize, script_limit: usize) -> HtmlProcessorConfig { + let mut settings = create_test_settings(); + for (id, config) in [ + ( + "nextjs", + json!({"enabled": true, "max_combined_payload_bytes": group_limit}), + ), + ( + "google_tag_manager", + json!({"enabled": true, "container_id": "GTM-MIX1"}), + ), + ] { + settings + .integrations + .insert_config(id, &config) + .expect("should configure integration"); + } + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new(crate::auction::compile_auction_plan(&settings).expect("should compile plan")), + ) + .expect("should create registry"); + let mut config = config_from_settings(&settings, ®istry); + config.max_buffered_body_bytes = script_limit; + config + } + + fn mixed_output_with_budget( + html: &str, + chunk_size: usize, + group_limit: usize, + script_limit: usize, + ) -> String { + let config = mixed_config(group_limit, script_limit); + let mut pipeline = StreamingPipeline::new( + PipelineConfig { + input_compression: Compression::None, + output_compression: Compression::None, + chunk_size, + }, + create_html_processor(config), + ); + let mut output = Vec::new(); + pipeline + .process(Cursor::new(html.as_bytes()), &mut output) + .expect("should consume complete mixed response"); + String::from_utf8(output).expect("should emit UTF-8") + } + + #[test] + fn mixed_next_data_flight_mentions_do_not_claim_json_or_hide_gtm() { + for text in [ + "self.__next_f", + "window.__next_f", + "self.__next_f.push([1,content])", + ] { + for chunk in [32, 64, 1000, 8192] { + let data = json!({"props": {"pageProps": { + "text": text, "href": "https://origin.example.com/page", + "tag": "http://www.googletagmanager.com/gtm.js", "padding": "_".repeat(10000), + }}}); + let output = mixed_output( + &format!(r#""#), + chunk, + ); + let script = output + .split(r#"") + .next() + .expect("should close Pages script"); + let mut expected = data; + expected["props"]["pageProps"]["href"] = json!("https://test.example.com/page"); + expected["props"]["pageProps"]["tag"] = + json!("/integrations/google_tag_manager/gtm.js"); + assert_eq!( + serde_json::from_str::(script) + .expect("should preserve Pages JSON"), + expected, + "should not claim quoted Flight mention: {text}, chunk={chunk}" + ); + } + } + } + + #[test] + fn protected_flight_then_next_data_then_ordinary_gtm_resets_ownership() { + let payload = "1:Tffff,'http://www.googletagmanager.com/gtm.js'"; + let pages = json!({"text": "self.__next_f.push([1,content])", "href": "https://origin.example.com/page", "tag": "http://www.googletagmanager.com/gtm.js"}); + // Keep the ordinary URL's opening quote and >=6 marker bytes together; + // standalone GTM deliberately does not hold shorter tentative prefixes. + let alignment = " ".repeat(23); + let html = format!( + r#"{alignment}"#, + serde_json::to_string(payload).expect("should encode raw Flight") + ); + for chunk in [32, 8192] { + let output = mixed_output_with_budget(&html, chunk, 24, 10000); + assert_eq!( + flight_payloads(&output), + [payload], + "should preserve raw Flight" + ); + let source = output + .split(r#"") + .next() + .expect("should close Pages script"); + let mut expected = pages.clone(); + expected["href"] = json!("https://test.example.com/page"); + expected["tag"] = json!("/integrations/google_tag_manager/gtm.js"); + assert_eq!( + serde_json::from_str::(source).expect("should retain JSON"), + expected + ); + assert!( + output.ends_with( + "" + ), + "should rewrite later ordinary GTM at chunk={chunk}: {output}" + ); + } + } + + #[test] + fn push_head_after_tentative_reference_overflow_protects_raw_continuation() { + let payload = "1:Tffff,'http://www.googletagmanager.com/gtm.js'"; + let html = format!( + r#""#, + " ".repeat(512), + serde_json::to_string(payload).expect("should encode raw Flight") + ); + for chunk in [32, 8192] { + assert_eq!( + mixed_output_with_budget(&html, chunk, 100, 10000), + html, + "should protect a call confirmed after tentative probe overflow and preserve its later raw continuation" + ); + } + } + + #[test] + fn ordinary_next_f_reference_does_not_bypass_later_flight() { + let body = "http://www.googletagmanager.com/gtm.js"; + let payload = format!("1:T{:x},{body}", body.len()); + for padding in [0, 400] { + for chunk in [32, 8192] { + let html = format!( + "{}", + "_".repeat(padding), + flight_html(&[&payload]) + ); + let output = mixed_output_with_budget(&html, chunk, 100, 10000); + assert_eq!( + flight_payloads(&output), + [format!( + "1:T{:x},/integrations/google_tag_manager/gtm.js", + "/integrations/google_tag_manager/gtm.js".len() + )], + "should not bypass captured Flight because of a prior ordinary reference: padding={padding}, chunk={chunk}" + ); + } + } + } + + #[test] + fn ordinary_next_f_reference_does_not_hide_gtm() { + let script = "var ref=self.__next_f;var tag='http://www.googletagmanager.com/gtm.js';"; + for chunk in [32, 64, 1000, 8192] { + let output = mixed_output(&format!(""), chunk); + assert_eq!( + output, + "", + "should not protect a non-push reference, chunk={chunk}" + ); + } + } + + #[test] + fn mixed_next_data_preserves_json_and_both_rewrites() { + for chunk in [32, 64, 1000, 8192] { + for marker in ["_", "__", "__n", "__ne", "__nex", "__next", "__next_"] { + for (url, routed) in [ + ( + "https://www.googletagmanager.com/gtm.js?id=GTM-MIX1", + "/integrations/google_tag_manager/gtm.js?id=GTM-MIX1", + ), + ( + "http://www.google-analytics.com/g/collect?v=2", + "/integrations/google_tag_manager/g/collect?v=2", + ), + ] { + for late in [false, true] { + let padding = json!(format!("{}{marker}", "_".repeat(10000))); + let source = if late { + format!( + r#"{{"padding":{padding},"href":"https://origin.example.com/page","tag":"{url}"}}"# + ) + } else { + format!( + r#"{{"tag":"{url}","padding":{padding},"href":"https://origin.example.com/page"}}"# + ) + }; + let html = format!( + r#"

suffix

"# + ); + let output = mixed_output(&html, chunk); + let script = output + .split(r#"") + .next() + .expect("should close script"); + let expected = json!({"padding": format!("{}{marker}", "_".repeat(10000)), "href": "https://test.example.com/page", "tag": routed}); + assert_eq!( + serde_json::from_str::(script) + .expect("should preserve valid JSON"), + expected, + "chunk={chunk}, marker={marker}, late={late}" + ); + assert!(output.ends_with(""), "should finish response"); + } + } + } + } + } + + fn flight_payloads(output: &str) -> Vec { + let mut payloads = Vec::new(); + for part in output.split("").next().expect("should close script"); + while let Some((start, end)) = shared::find_rsc_push_payload_range(script) { + payloads.push( + serde_json::from_str::(&script[start - 1..=end]) + .expect("should retain valid push string JSON"), + ); + script = &script[end + 1..]; + } + } + payloads + } + + fn assert_mixed_t_model(payload: &str, expected: &serde_json::Value) { + let (header, body) = payload.split_once(',').expect("should retain T record"); + assert_eq!( + usize::from_str_radix(&header[3..], 16).expect("should retain hex length"), + body.len(), + "should count decoded transformed T bytes" + ); + assert_eq!( + &serde_json::from_str::(body).expect("should retain model JSON"), + expected + ); + } + + #[test] + fn normal_flight_bootstrap_controls_allow_later_payload_rewriting() { + let controls = "(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])"; + let data = json!({"href": "https://origin.example.com/page", "tag": "http://www.googletagmanager.com/gtm.js", "ga": "//www.google-analytics.com/g/collect?v=2", "text": "é😀"}); + let body = data.to_string(); + let push = format!( + "self.__next_f.push([1,{}])", + serde_json::to_string(&format!("1:T{:x},{body}", body.len())) + .expect("should encode Flight") + ); + let mut expected = data; + expected["href"] = json!("https://test.example.com/page"); + expected["tag"] = json!("/integrations/google_tag_manager/gtm.js"); + expected["ga"] = json!("/integrations/google_tag_manager/g/collect?v=2"); + for together in [false, true] { + let scripts = if together { + format!("") + } else { + format!("") + }; + let html = format!("{scripts}

suffix

"); + for compression in [Compression::None, Compression::Gzip] { + let input = if compression == Compression::Gzip { + let mut encoder = GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder + .write_all(html.as_bytes()) + .expect("should encode gzip bootstrap document"); + encoder.finish().expect("should finish gzip input") + } else { + html.as_bytes().to_vec() + }; + for chunk_size in [32, 64, 1000, 8192] { + let mut pipeline = StreamingPipeline::new( + PipelineConfig { + input_compression: compression, + output_compression: compression, + chunk_size, + }, + create_html_processor(mixed_config(100000, 100000)), + ); + let mut bytes = Vec::new(); + pipeline + .process(Cursor::new(&input), &mut bytes) + .expect("should consume bootstrap document"); + let mut decoded = Vec::new(); + if compression == Compression::Gzip { + GzDecoder::new(bytes.as_slice()) + .read_to_end(&mut decoded) + .expect("should decode gzip bootstrap response"); + } else { + decoded = bytes; + } + let output = String::from_utf8(decoded).expect("should emit UTF-8"); + assert!( + output.contains(controls), + "should preserve inert bootstrap controls" + ); + let payloads = flight_payloads(&output); + assert_eq!(payloads.len(), 1, "should emit payload exactly once"); + assert_mixed_t_model(&payloads[0], &expected); + assert!(!output.contains("__ts_rsc_")); + assert!(output.ends_with("

suffix

")); + } + } + } + } + + #[test] + fn same_delimiter_escaped_flight_queries_preserve_data_and_t_lengths() { + let data = json!({"tag": "http://www.googletagmanager.com/gtm.js?q=\"x\"&v=é", "href": "https://origin.example.com/page", "url": "http://origin.example.com/other"}); + let body = data.to_string(); + let payload = format!("1:T{:x},{body}", body.len()); + let mut expected = data; + expected["tag"] = json!("/integrations/google_tag_manager/gtm.js?q=\"x\"&v=é"); + expected["href"] = json!("https://test.example.com/page"); + expected["url"] = json!("https://test.example.com/other"); + for chunk in [32, 64, 1000, 8192] { + let output = mixed_output(&flight_html(&[&payload]), chunk); + let payloads = flight_payloads(&output); + assert_mixed_t_model(&payloads[0], &expected); + let body = r"'http://www.googletagmanager.com/gtm.js?q=\'x\''"; + let payload = format!("1:T{:x},{body}", body.len()); + let output = mixed_output(&flight_html(&[&payload]), chunk); + let payloads = flight_payloads(&output); + let (header, transformed) = payloads[0].split_once(',').expect("should retain T"); + assert_eq!( + usize::from_str_radix(&header[3..], 16).expect("should parse hex"), + transformed.len() + ); + assert_eq!( + transformed, + r"'/integrations/google_tag_manager/gtm.js?q=\'x\''" + ); + } + } + + #[test] + fn sticky_false_google_prefix_preserves_raw_script_metacharacters() { + let fragments = [ + "", + ]; + let mut processor = create_html_processor(mixed_config(10000, 10000)); + let mut bytes = Vec::new(); + for (index, fragment) in fragments.iter().enumerate() { + bytes.extend( + processor + .process_chunk(fragment.as_bytes(), index == fragments.len() - 1) + .expect("should consume raw script fragment"), + ); + } + assert_eq!( + String::from_utf8(bytes).expect("should emit UTF-8"), + fragments.concat(), + "holding/restoration must preserve raw JS source, not HTML-escape it" + ); + } + + #[test] + fn mixed_flight_first_party_urls_preserve_t_lengths() { + for scheme in ["https:", "http:", ""] { + for host_path in [ + "www.googletagmanager.com/gtm.js", + "www.googletagmanager.com/gtag/js?id=G-MIX1", + "www.google-analytics.com/g/collect", + "analytics.google.com/collect?v=2", + ] { + for quoted in [false, true] { + let url = format!("{scheme}//{host_path}"); + let body = if quoted { format!("\"{url}\"") } else { url }; + let payload = format!("1:T{:x},{body}", body.len()); + let encoded = serde_json::to_string(&payload).expect("should encode payload"); + let html = format!( + "

suffix

" + ); + for chunk in [32, 64, 1000, 8192] { + let output = mixed_output(&html, chunk); + let payloads = flight_payloads(&output); + assert_eq!(payloads.len(), 1); + let (header, body) = + payloads[0].split_once(',').expect("should retain T record"); + assert!( + body.contains("/integrations/google_tag_manager/"), + "should rewrite captured GTM-only flight: {body}" + ); + assert_eq!( + usize::from_str_radix(&header[3..], 16) + .expect("should keep hex length"), + body.len(), + "should count transformed decoded bytes" + ); + } + } + } + } + let payload = "0:{\"url\":\"https://www.googletagmanager.com/gtm.js\",\"href\":\"https://origin.example.com/page\"}\n"; + let encoded = serde_json::to_string(&payload).expect("should encode payload"); + let output = mixed_output( + &format!(""), + 32, + ); + let payloads = flight_payloads(&output); + assert!(payloads[0].contains("test.example.com/page")); + assert!( + payloads[0].contains("/integrations/google_tag_manager/gtm.js"), + "should rewrite queryless escaped quote URL" + ); + } + + #[test] + fn mixed_flight_batch_rewrites_every_payload() { + let bodies = [ + "http://www.googletagmanager.com/gtm.js", + "//www.google-analytics.com/collect?v=2", + ]; + let pushes: Vec = bodies + .iter() + .enumerate() + .map(|(index, body)| { + let payload = format!("{index}:T{:x},{body}", body.len()); + format!( + "self.__next_f.push([1,{}])", + serde_json::to_string(&payload).expect("should encode payload") + ) + }) + .collect(); + let output = mixed_output(&format!("", pushes.join(";")), 8192); + let script = output + .split("") + .next() + .expect("should close script"); + for push in script.split(';') { + let json = push + .strip_prefix("self.__next_f.push(") + .expect("should retain push") + .strip_suffix(')') + .expect("should close push"); + let value: serde_json::Value = + serde_json::from_str(json).expect("should preserve push JSON"); + let (header, body) = value[1] + .as_str() + .expect("should retain payload") + .split_once(',') + .expect("should retain T header"); + assert!( + body.contains("/integrations/google_tag_manager/"), + "should rewrite every captured push" + ); + assert_eq!( + usize::from_str_radix(&header[3..], 16).expect("should parse hex"), + body.len(), + "should recount every payload" + ); + } + } + + #[test] + fn mixed_next_data_prefix_tails_and_false_google_preserve_payload() { + for chunk in [32, 64, 1000, 8192] { + for prefix in [ + "_", "__", "__n", "__ne", "__nex", "__next", "__next_", "google", + ] { + for alignment in [0, 1, 13, 1023, 1024] { + let data = json!({"padding": format!("{}{prefix}", "_".repeat(12000)), "href": "https://origin.example.com/page"}); + let start = format!( + "{}"), chunk); + let script = output + .split("") + .next() + .expect("should close script"); + data["href"] = json!("https://test.example.com/page"); + assert_eq!( + serde_json::from_str::(script) + .expect("should keep valid JSON"), + data, + "prefix={prefix}, chunk={chunk}, alignment={alignment}" + ); + } + } + } + } + + #[test] + fn mixed_next_data_identity_and_gzip_chunk_sweep() { + let data = json!({"href": "https://origin.example.com/page", "tag": "https://www.google-analytics.com/collect?v=2", "padding": "_".repeat(12000)}); + let html = format!( + "

suffix

" + ); + for compression in [Compression::None, Compression::Gzip] { + let input = if compression == Compression::Gzip { + let mut encoder = GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder + .write_all(html.as_bytes()) + .expect("should encode actual gzip input"); + encoder.finish().expect("should finish gzip input") + } else { + html.as_bytes().to_vec() + }; + for chunk_size in [32, 64, 1000, 8192] { + let mut pipeline = StreamingPipeline::new( + PipelineConfig { + input_compression: compression, + output_compression: compression, + chunk_size, + }, + create_html_processor(mixed_config(100000, 100000)), + ); + let mut output = Vec::new(); + pipeline + .process(Cursor::new(&input), &mut output) + .expect("should process complete compressed document"); + let mut decoded = Vec::new(); + if compression == Compression::Gzip { + GzDecoder::new(output.as_slice()) + .read_to_end(&mut decoded) + .expect("should decode actual gzip output"); + } else { + decoded = output; + } + let output = String::from_utf8(decoded).expect("should emit UTF-8"); + let script = output + .split("") + .next() + .expect("should close script"); + let mut expected = data.clone(); + expected["href"] = json!("https://test.example.com/page"); + expected["tag"] = json!("/integrations/google_tag_manager/collect?v=2"); + assert_eq!( + serde_json::from_str::(script) + .expect("should retain valid JSON"), + expected + ); + assert!(output.ends_with("")); + } + } + } + + #[test] + fn mixed_registry_isolates_interleaved_html_documents() { + let config = mixed_config(10000, 10000); + let mut a = create_html_processor(config.clone()); + let mut b = create_html_processor(config); + let mut output_a = a + .process_chunk(b"", true) + .expect("should finish document A"), + ); + output_b.extend( + b.process_chunk(b"-analytics.com/collect?v=2';", true) + .expect("should finish document B"), + ); + assert_eq!( + String::from_utf8(output_a).expect("should emit UTF-8"), + "" + ); + assert_eq!( + String::from_utf8(output_b).expect("should emit UTF-8"), + "" + ); + } + + #[test] + fn structured_and_probe_composition_at_every_identifier_prefix_tail() { + for prefix in ["_", "__", "__n", "__ne", "__nex", "__next", "__next_"] { + let mut processor = create_html_processor(mixed_config(100000, 100000)); + let first = format!(""; + let mut bytes = processor + .process_chunk(first.as_bytes(), false) + .expect("should accept explicit prefix tail"); + bytes.extend( + processor + .process_chunk(second.as_bytes(), true) + .expect("should finish suppressed text"), + ); + let html = String::from_utf8(bytes).expect("should emit UTF-8"); + let script = html + .split("") + .next() + .expect("should close script"); + assert_eq!( + serde_json::from_str::(script).expect("should keep valid JSON"), + json!({"padding": prefix, "href": "https://test.example.com/page"}) + ); + } + } + + #[test] + fn mixed_existing_app_fixture_completes_with_both_enabled() { + let fixture = include_str!("../../html_processor.test.html"); + let data = json!({"href": "https://origin.example.com/fixture-proof", "tag": "http://www.googletagmanager.com/gtm.js", "ga": "//www.google-analytics.com/g/collect?v=2"}); + let body = data.to_string(); + let push = format!( + "", + serde_json::to_string(&format!("f:T{:x},{body}", body.len())) + .expect("should encode fixture proof") + ); + let (before, after) = fixture + .rsplit_once("") + .expect("should retain structural body close"); + let fixture_copy = format!("{before}{push}{after}"); + assert!( + fixture_copy.contains( + "(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])" + ), + "should retain real bootstrap sequence" + ); + let mut expected = data; + expected["href"] = json!("https://test.example.com/fixture-proof"); + expected["tag"] = json!("/integrations/google_tag_manager/gtm.js"); + expected["ga"] = json!("/integrations/google_tag_manager/g/collect?v=2"); + for chunk in [32, 8192] { + let output = mixed_output(&fixture_copy, chunk); + let payloads = flight_payloads(&output); + let proof = payloads + .iter() + .find(|payload| payload.starts_with("f:T")) + .expect("should retain designated Flight proof"); + assert_mixed_t_model(proof, &expected); + assert!(!output.contains("__ts_rsc_")); + assert!( + output.contains("googletagmanager.com/ns.html"), + "should retain unsupported iframe routing" + ); + assert!(output.trim_end().ends_with("")); + } + } + + fn flight_html(payloads: &[&str]) -> String { + let scripts: String = payloads + .iter() + .map(|payload| { + format!( + "", + serde_json::to_string(payload).expect("should encode Flight") + ) + }) + .collect(); + format!("{scripts}

suffix

") + } + + #[test] + fn mixed_flight_cross_script_lengths_and_split_hostname_safety() { + let body = r#"{"tag":"http://www.googletagmanager.com/gtm.js","url":"https://origin.example.com/page","text":"é😀"}"#; + let payload = format!("1:T{:x},{body}", body.len()); + for split in [ + (payload.find("http:").expect("should locate URL") - 1), + payload + .find("tagmanager") + .expect("should locate hostname split"), + ] { + for chunk in [32, 64, 1000, 8192] { + let output = + mixed_output(&flight_html(&[&payload[..split], &payload[split..]]), chunk); + let combined = flight_payloads(&output).concat(); + let (header, rewritten) = combined.split_once(',').expect("should retain T header"); + assert_eq!( + usize::from_str_radix(&header[3..], 16).expect("should parse length"), + rewritten.len() + ); + let data: serde_json::Value = + serde_json::from_str(rewritten).expect("should preserve model JSON"); + if split == (payload.find("http:").expect("should locate URL") - 1) { + assert_eq!(data["tag"], "/integrations/google_tag_manager/gtm.js"); + } else { + assert_eq!( + data["tag"], "http://www.googletagmanager.com/gtm.js", + "should safely preserve URL split across actual payload scripts" + ); + } + assert_eq!(data["url"], "https://test.example.com/page"); + assert_eq!(data["text"], "é😀"); + } + } + let escaped = body.replace('/', r"\/"); + let payload = format!("1:T{:x},{escaped}", escaped.len()); + let output = mixed_output(&flight_html(&[&payload]), 32); + let payloads = flight_payloads(&output); + let (header, body) = payloads[0].split_once(',').expect("should retain T"); + assert_eq!( + usize::from_str_radix(&header[3..], 16).expect("should parse length"), + body.len() + ); + let data: serde_json::Value = + serde_json::from_str(body).expect("should preserve escaped model JSON"); + assert_eq!(data["tag"], "/integrations/google_tag_manager/gtm.js"); + } + + #[test] + fn mixed_flight_fallback_preserves_original_payloads_and_later_scripts() { + let url = "'http://www.googletagmanager.com/gtm.js?id=GTM-MIX1'"; + let incomplete = format!("1:Tffff,{url}"); + let oversized = format!("1:T{:x},{url}", url.len()); + for (payloads, group_limit, script_limit) in [ + (vec![incomplete.as_str()], 10000, 10000), + (vec!["1:T", url], 10000, 10000), + (vec![oversized.as_str()], 24, 10000), + (vec![oversized.as_str()], 10000, 10), + ] { + for chunk in [32, 64, 1000, 8192] { + let html = flight_html(&payloads).replace("

suffix

", "

suffix

"); + let output = mixed_output_with_budget(&html, chunk, group_limit, script_limit); + assert_eq!( + flight_payloads(&output), + payloads, + "should preserve fallback payload bytes: group={group_limit}, script={script_limit}, chunk={chunk}" + ); + if script_limit > 10 { + assert!( + output.contains("var ordinary='/integrations/google_tag_manager/gtm.js';"), + "should rewrite ordinary script after Flight fallback" + ); + } + assert!(output.ends_with("")); + } + } + } + + #[test] + fn mixed_flight_response_completes() { + let payload = r#"0:{\"href\":\"https://origin.example.com/page\",\"url\":\"https://www.googletagmanager.com/gtm.js?id=GTM-MIX1\"}\n"#; + let html = format!( + r#"

suffix

"# + ); + let output = mixed_output(&html, 8192); + assert!( + output.contains("test.example.com/page"), + "should rewrite origin" + ); + assert!(!output.contains("__ts_rsc_"), "should resolve placeholders"); + assert!(output.ends_with(""), "should finish response"); + } + #[test] fn html_processor_rewrites_nextjs_script_when_enabled() { let html = r#" diff --git a/crates/trusted-server-core/src/integrations/nextjs/rsc.rs b/crates/trusted-server-core/src/integrations/nextjs/rsc.rs index ab144083c1..9a53d5e8d2 100644 --- a/crates/trusted-server-core/src/integrations/nextjs/rsc.rs +++ b/crates/trusted-server-core/src/integrations/nextjs/rsc.rs @@ -398,12 +398,46 @@ fn find_tchunks_with_markers(content: &str) -> Option> { // Single-script T-chunk processing // ============================================================================= -pub(crate) fn rewrite_rsc_tchunks_with_rewriter( +fn rewrite_span( content: &str, rewriter: &RscUrlRewriter, origin_host: &str, request_host: &str, request_scheme: &str, + transform: Option String>, +) -> String { + let rewritten = rewriter.rewrite_to_string(content, origin_host, request_host, request_scheme); + match transform { + Some(transform) => transform(&rewritten), + None => rewritten, + } +} + +#[cfg(test)] +fn rewrite_rsc_tchunks_with_rewriter( + content: &str, + rewriter: &RscUrlRewriter, + origin_host: &str, + request_host: &str, + request_scheme: &str, +) -> String { + rewrite_rsc_tchunks_with_transform( + content, + rewriter, + origin_host, + request_host, + request_scheme, + None, + ) +} + +fn rewrite_rsc_tchunks_with_transform( + content: &str, + rewriter: &RscUrlRewriter, + origin_host: &str, + request_host: &str, + request_scheme: &str, + transform: Option String>, ) -> String { let Some(chunks) = find_tchunks(content) else { log::warn!( @@ -413,7 +447,14 @@ pub(crate) fn rewrite_rsc_tchunks_with_rewriter( }; if chunks.is_empty() { - return rewriter.rewrite_to_string(content, origin_host, request_host, request_scheme); + return rewrite_span( + content, + rewriter, + origin_host, + request_host, + request_scheme, + transform, + ); } let mut result = String::with_capacity(content.len()); @@ -421,15 +462,24 @@ pub(crate) fn rewrite_rsc_tchunks_with_rewriter( for chunk in &chunks { let before = &content[last_end..chunk.match_start]; - result.push_str( - rewriter - .rewrite(before, origin_host, request_host, request_scheme) - .as_ref(), - ); + result.push_str(&rewrite_span( + before, + rewriter, + origin_host, + request_host, + request_scheme, + transform, + )); let chunk_content = &content[chunk.header_end..chunk.content_end]; - let rewritten_content = - rewriter.rewrite_to_string(chunk_content, origin_host, request_host, request_scheme); + let rewritten_content = rewrite_span( + chunk_content, + rewriter, + origin_host, + request_host, + request_scheme, + transform, + ); let new_length = calculate_unescaped_byte_length(&rewritten_content); let new_length_hex = format!("{new_length:x}"); @@ -444,11 +494,14 @@ pub(crate) fn rewrite_rsc_tchunks_with_rewriter( } let remaining = &content[last_end..]; - result.push_str( - rewriter - .rewrite(remaining, origin_host, request_host, request_scheme) - .as_ref(), - ); + result.push_str(&rewrite_span( + remaining, + rewriter, + origin_host, + request_host, + request_scheme, + transform, + )); result } @@ -519,23 +572,44 @@ pub(crate) fn rewrite_rsc_scripts_combined_with_limit( request_host: &str, request_scheme: &str, max_combined_payload_bytes: usize, +) -> Vec { + rewrite_rsc_scripts_combined_with_transform( + payloads, + rewriter, + origin_host, + request_host, + request_scheme, + max_combined_payload_bytes, + None, + ) +} + +pub(super) fn rewrite_rsc_scripts_combined_with_transform( + payloads: &[&str], + rewriter: &RscUrlRewriter, + origin_host: &str, + request_host: &str, + request_scheme: &str, + max_combined_payload_bytes: usize, + transform: Option String>, ) -> Vec { let Some((_, preceding_payloads)) = payloads.split_last() else { return Vec::new(); }; // Early exit if no payload contains the origin host - avoids regex compilation - if !payloads.iter().any(|p| p.contains(origin_host)) { + if transform.is_none() && !payloads.iter().any(|p| p.contains(origin_host)) { return payloads.iter().map(|p| (*p).to_owned()).collect(); } if payloads.len() == 1 { - return vec![rewrite_rsc_tchunks_with_rewriter( + return vec![rewrite_rsc_tchunks_with_transform( payloads[0], rewriter, origin_host, request_host, request_scheme, + transform, )]; } @@ -570,12 +644,13 @@ pub(crate) fn rewrite_rsc_scripts_combined_with_limit( return payloads .iter() .map(|p| { - rewrite_rsc_tchunks_with_rewriter( + rewrite_rsc_tchunks_with_transform( p, rewriter, origin_host, request_host, request_scheme, + transform, ) }) .collect(); @@ -615,7 +690,16 @@ pub(crate) fn rewrite_rsc_scripts_combined_with_limit( if chunks.is_empty() { return payloads .iter() - .map(|p| rewriter.rewrite_to_string(p, origin_host, request_host, request_scheme)) + .map(|p| { + rewrite_span( + p, + rewriter, + origin_host, + request_host, + request_scheme, + transform, + ) + }) .collect(); } @@ -624,15 +708,24 @@ pub(crate) fn rewrite_rsc_scripts_combined_with_limit( for chunk in &chunks { let before = &combined[last_end..chunk.match_start]; - result.push_str( - rewriter - .rewrite(before, origin_host, request_host, request_scheme) - .as_ref(), - ); + result.push_str(&rewrite_span( + before, + rewriter, + origin_host, + request_host, + request_scheme, + transform, + )); let chunk_content = &combined[chunk.header_end..chunk.content_end]; - let rewritten_content = - rewriter.rewrite_to_string(chunk_content, origin_host, request_host, request_scheme); + let rewritten_content = rewrite_span( + chunk_content, + rewriter, + origin_host, + request_host, + request_scheme, + transform, + ); let new_length = calculate_unescaped_byte_length_skip_markers(&rewritten_content); let new_length_hex = format!("{new_length:x}"); @@ -647,11 +740,14 @@ pub(crate) fn rewrite_rsc_scripts_combined_with_limit( } let remaining = &combined[last_end..]; - result.push_str( - rewriter - .rewrite(remaining, origin_host, request_host, request_scheme) - .as_ref(), - ); + result.push_str(&rewrite_span( + remaining, + rewriter, + origin_host, + request_host, + request_scheme, + transform, + )); result.split(RSC_MARKER).map(String::from).collect() } diff --git a/crates/trusted-server-core/src/integrations/nextjs/rsc_placeholders.rs b/crates/trusted-server-core/src/integrations/nextjs/rsc_placeholders.rs index 7fd8d70a8b..9b1c783115 100644 --- a/crates/trusted-server-core/src/integrations/nextjs/rsc_placeholders.rs +++ b/crates/trusted-server-core/src/integrations/nextjs/rsc_placeholders.rs @@ -1,4 +1,6 @@ -use std::sync::Arc; +use std::sync::{Arc, LazyLock}; + +use regex::Regex; use crate::integrations::{ IntegrationScriptContext, IntegrationScriptRewriter, ScriptRewriteAction, @@ -12,11 +14,37 @@ use super::rsc_stream::{ capture_fragment, classify_rsc_group, document_state, rsc_payload_placeholder, }; use super::shared::{ - RSC_RECEIVER_CONTEXT_BYTES, find_rsc_push_payload_range, find_trimmed_rsc_push_payload_range, + RSC_PUSH_CALL_PATTERN, RSC_PUSH_CALL_PATTERN_TRIMMED, RSC_RECEIVER_CONTEXT_BYTES, + find_rsc_push_payload_range, find_trimmed_rsc_push_payload_range, receiver_context_is_flight_push, }; use super::{NEXTJS_INTEGRATION_ID, NextJsIntegrationConfig}; +// Observe actual qualified push calls, not bare property references. Unlike +// capture, ownership also covers unsupported channels and whitespace spellings +// because raw fallback must retain their original T lengths. +static QUALIFIED_FLIGHT_PUSH_PATTERN: LazyLock = LazyLock::new(|| { + Regex::new(r"(?:self|window)\.__next_f\s*(?:\.\s*push\s*\(|=\s*(?:self|window)\.__next_f\s*\|\|\s*\[\]\s*\)\s*\.\s*push\s*\()") + .expect("should compile qualified Flight push observation") +}); + +// The receiver may already have streamed, leaving a verified trimmed head. +static TRIMMED_FLIGHT_PUSH_PATTERN: LazyLock = LazyLock::new(|| { + Regex::new(r"^__next_f\s*(?:\.\s*push\s*\(|=\s*(?:self|window)\.__next_f\s*\|\|\s*\[\]\s*\)\s*\.\s*push\s*\()") + .expect("should compile trimmed Flight push observation") +}); + +// Only completed inert bootstrap controls may be skipped. Opaque channel-2 +// strings and incomplete calls must retain the conservative raw fallback. +static INERT_FLIGHT_CONTROL_ARGUMENTS: LazyLock = LazyLock::new(|| { + Regex::new(r"^\s*\[\s*(?:0|2\s*,\s*null)\s*\]\s*\)") + .expect("should compile inert Flight control arguments") +}); + +// Collapsing whitespace runs bounds either supported call head well below this +// size, while keeping the receiver's preceding boundary for qualification. +const FLIGHT_QUALIFIER_TAIL_BYTES: usize = 96; + pub(super) struct NextJsRscPlaceholderRewriter { config: Arc, } @@ -42,67 +70,103 @@ impl NextJsRscPlaceholderRewriter { }; } - let range = if state.rsc_receiver_trimmed { - find_trimmed_rsc_push_payload_range(content) - } else { - find_rsc_push_payload_range(content) - }; - state.rsc_receiver_trimmed = false; - let Some((payload_start, payload_end)) = range else { - return if was_buffered { - ScriptRewriteAction::replace(content.to_owned()) + // Discover all ranges and validate the queue budget before committing any + // capture. A malformed or unrecognized remainder stays original/protected. + let mut ranges = Vec::new(); + let mut cursor = 0; + let mut trimmed = std::mem::take(&mut state.rsc_receiver_trimmed); + let mut queued_bytes = state.captured_payload_bytes; + while content[cursor..].contains("__next_f") { + let remaining = &content[cursor..]; + let head = if trimmed { + TRIMMED_FLIGHT_PUSH_PATTERN.find(remaining) } else { - ScriptRewriteAction::Keep + first_qualified_flight_push(remaining) }; - }; - - if payload_start > payload_end - || payload_end > content.len() - || !content.is_char_boundary(payload_start) - || !content.is_char_boundary(payload_end) - { - state.bypass_rsc = true; - return if was_buffered { - ScriptRewriteAction::replace(content.to_owned()) + if let Some(head) = head + && let Some(arguments) = + INERT_FLIGHT_CONTROL_ARGUMENTS.find(&remaining[head.end()..]) + { + cursor += head.end() + arguments.end(); + trimmed = false; + continue; + } + let range = if trimmed { + find_trimmed_rsc_push_payload_range(remaining) } else { - ScriptRewriteAction::Keep + find_rsc_push_payload_range(remaining) + }; + let call = if trimmed { + RSC_PUSH_CALL_PATTERN_TRIMMED.find(remaining) + } else { + RSC_PUSH_CALL_PATTERN.find(remaining) + }; + trimmed = false; + let Some((start, end)) = range.filter(|(start, _)| { + call.is_some_and(|call| { + call.end() == *start && !contains_qualified_flight(&remaining[..call.start()]) + }) + }) else { + // A raw qualified push cannot safely continue a pending T group + // or be skipped by later captures. A bare property reference is + // ordinary script text and must not disable later Flight capture. + state.bypass_rsc |= state.flight_node_owned; + return if was_buffered { + ScriptRewriteAction::replace(content.to_owned()) + } else { + ScriptRewriteAction::Keep + }; }; + let start = cursor + start; + let end = cursor + end; + let length = end.saturating_sub(start); + // Group bytes and queued parser-held bytes have distinct limits: + // independent groups parsed in one call must not consume each other's + // group allowance before the output processor can release them. + let total = queued_bytes.checked_add(length); + if start > end + || end >= content.len() + || !content.is_char_boundary(start) + || !content.is_char_boundary(end) + || length > limit + || state.captured_payloads.len() + ranges.len() >= MAX_UNRESOLVED_RSC_PAYLOADS + || total.is_none_or(|bytes| bytes > max_queued_payload_bytes) + { + state.bypass_rsc = true; + return if was_buffered { + ScriptRewriteAction::replace(content.to_owned()) + } else { + ScriptRewriteAction::Keep + }; + } + queued_bytes = total.expect("should have validated queue size"); + ranges.push((start, end)); + cursor = end + 1; } - - let payload = &content[payload_start..payload_end]; - // `limit` bounds one script here and the downstream unresolved group in - // `classify_rsc_group`. It deliberately does not bound this queue in - // aggregate: the processor cannot decrement a resolved group until the - // whole parser call returns, so a shared `limit` budget would make - // independent payloads bypass each other purely because they shared a - // source chunk. The queue is parser-held script text, so it is bounded - // by the parser's own script-buffer budget instead — the same budget - // `NextJsNextDataRewriter` buffers against — plus a payload count. - let exceeds_limit = payload.len() > limit - || state.captured_payloads.len() >= MAX_UNRESOLVED_RSC_PAYLOADS - || state - .captured_payload_bytes - .checked_add(payload.len()) - .is_none_or(|queued| queued > max_queued_payload_bytes); - if exceeds_limit { - state.bypass_rsc = true; + if ranges.is_empty() { return if was_buffered { ScriptRewriteAction::replace(content.to_owned()) } else { ScriptRewriteAction::Keep }; } - - let placeholder = rsc_payload_placeholder(&state.namespace, state.next_placeholder_index); - state.next_placeholder_index = state.next_placeholder_index.saturating_add(1); - state.captured_payload_bytes += payload.len(); - state.captured_payloads.push_back(CapturedPayload { - placeholder: placeholder.clone(), - original: payload.to_owned(), - }); - - let mut rewritten = content.to_owned(); - rewritten.replace_range(payload_start..payload_end, &placeholder); + let mut rewritten = String::with_capacity(content.len()); + cursor = 0; + for (start, end) in ranges { + let placeholder = + rsc_payload_placeholder(&state.namespace, state.next_placeholder_index); + state.next_placeholder_index = state.next_placeholder_index.saturating_add(1); + state.captured_payloads.push_back(CapturedPayload { + placeholder: placeholder.clone(), + original: content[start..end].to_owned(), + }); + rewritten.push_str(&content[cursor..start]); + rewritten.push_str(&placeholder); + cursor = end; + } + state.captured_payload_bytes = queued_bytes; + rewritten.push_str(&content[cursor..]); + state.current_fragment_protected = false; ScriptRewriteAction::replace(rewritten) } @@ -124,12 +188,15 @@ impl NextJsRscPlaceholderRewriter { FragmentCapture::Suppress => ScriptRewriteAction::RemoveNode, FragmentCapture::Restore(restored) => { state.rsc_receiver_trimmed = false; - state.bypass_rsc = true; + state.bypass_rsc |= state.flight_node_owned; ScriptRewriteAction::replace(restored) } FragmentCapture::PassThrough => { let trimmed = state.rsc_receiver_trimmed; state.rsc_receiver_trimmed = false; + if !state.flight_node_owned { + return ScriptRewriteAction::Keep; + } if is_last && content.len() > limit && content.contains("__next_f") { let mut remaining = content; let mut receiver_trimmed = trimmed; @@ -167,26 +234,12 @@ impl NextJsRscPlaceholderRewriter { } } } -} - -impl IntegrationScriptRewriter for NextJsRscPlaceholderRewriter { - fn integration_id(&self) -> &'static str { - NEXTJS_INTEGRATION_ID - } - - fn selector(&self) -> &'static str { - "script" - } - - fn rewrite(&self, content: &str, ctx: &IntegrationScriptContext<'_>) -> ScriptRewriteAction { - if !self.config.enabled || self.config.rewrite_attributes.is_empty() { - return ScriptRewriteAction::keep(); - } - - let state = document_state(ctx.document_state); - let mut state = state - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); + fn rewrite_fragment( + &self, + content: &str, + ctx: &IntegrationScriptContext<'_>, + state: &mut super::rsc_stream::NextJsDocumentState, + ) -> ScriptRewriteAction { if state.bypass_rsc { // The downstream processor can enter bypass after the parser has // suppressed part of this script. Restore it before passing through @@ -213,7 +266,7 @@ impl IntegrationScriptRewriter for NextJsRscPlaceholderRewriter { return self.rewrite_claimed_fragment( content, ctx.is_last_in_text_node, - &mut state, + state, limit, ctx.max_buffered_script_bytes, ); @@ -299,7 +352,7 @@ impl IntegrationScriptRewriter for NextJsRscPlaceholderRewriter { let action = self.rewrite_claimed_fragment( claimed, ctx.is_last_in_text_node, - &mut state, + state, limit, ctx.max_buffered_script_bytes, ); @@ -315,6 +368,105 @@ impl IntegrationScriptRewriter for NextJsRscPlaceholderRewriter { } } +impl IntegrationScriptRewriter for NextJsRscPlaceholderRewriter { + fn integration_id(&self) -> &'static str { + NEXTJS_INTEGRATION_ID + } + + fn selector(&self) -> &'static str { + "script" + } + + fn rewrite(&self, content: &str, ctx: &IntegrationScriptContext<'_>) -> ScriptRewriteAction { + let shared = document_state(ctx.document_state); + let mut state = shared + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + if std::mem::take(&mut state.next_data_fragment) { + // Pages data is not executable Flight, even when a JSON string + // contains a complete push spelling. Clear the downstream snapshot + // as well so a prior protected script cannot suppress Pages GTM. + state.current_fragment_protected = false; + state.flight_node_owned = false; + state.flight_qualifier_tail.clear(); + return ScriptRewriteAction::Keep; + } + if !self.config.enabled || self.config.rewrite_attributes.is_empty() { + state.current_fragment_protected = false; + return ScriptRewriteAction::Keep; + } + observe_flight_qualifier(&mut state, content); + // A call head can complete after a tentative property probe overflowed. + // Once it proves Flight, preserve its entire raw continuation/group. + if state.flight_node_owned + && matches!( + state.rsc_script, + super::rsc_stream::FragmentState::BypassUntilLast + ) + { + state.bypass_rsc = true; + } + state.current_fragment_protected = state.flight_node_owned; + let action = self.rewrite_fragment(content, ctx, &mut state); + if ctx.is_last_in_text_node { + state.flight_node_owned = false; + state.flight_qualifier_tail.clear(); + state.rsc_probe.clear(); + state.rsc_receiver_context.clear(); + state.rsc_receiver_trimmed = false; + } + action + } +} + +fn contains_qualified_flight(source: &str) -> bool { + first_qualified_flight_push(source).is_some() +} + +fn first_qualified_flight_push(source: &str) -> Option> { + QUALIFIED_FLIGHT_PUSH_PATTERN + .find_iter(source) + .find(|call| { + let identifier = call.start() + + call + .as_str() + .find("__next_f") + .expect("should find identifier in qualified push pattern"); + receiver_context_is_flight_push(&source[..identifier]) + }) +} + +/// Observe a bounded whitespace-normalized call head without buffering script +/// source, including initializer heads split after document bypass. +fn observe_flight_qualifier(state: &mut super::rsc_stream::NextJsDocumentState, content: &str) { + if state.flight_node_owned { + return; + } + for character in content.chars() { + let character = if character.is_whitespace() { + ' ' + } else { + character + }; + if character == ' ' && state.flight_qualifier_tail.ends_with(' ') { + continue; + } + state.flight_qualifier_tail.push(character); + if state.flight_qualifier_tail.len() > FLIGHT_QUALIFIER_TAIL_BYTES { + let start = state.flight_qualifier_tail.len() - FLIGHT_QUALIFIER_TAIL_BYTES; + let start = (start..state.flight_qualifier_tail.len()) + .find(|index| state.flight_qualifier_tail.is_char_boundary(*index)) + .expect("should find bounded qualifier character boundary"); + state.flight_qualifier_tail.drain(..start); + } + if character == '(' && contains_qualified_flight(&state.flight_qualifier_tail) { + state.flight_node_owned = true; + state.flight_qualifier_tail.clear(); + break; + } + } +} + /// Bytes to withhold so a `__next_f` identifier split across text fragments can /// still be recognized once the next fragment arrives. fn longest_identifier_prefix(bytes: &[u8]) -> usize { @@ -370,6 +522,295 @@ mod tests { }) } + #[test] + fn unrecognized_qualified_prefix_makes_the_whole_batch_raw_and_protected() { + let state = IntegrationDocumentState::default(); + let rewriter = NextJsRscPlaceholderRewriter::new(test_config()); + let script = r#"self.__next_f.push([2,"1:T40,'http://www.googletagmanager.com/gtm.js'"]);self.__next_f.push([1,"later"])"#; + assert_eq!( + rewriter.rewrite(script, &ctx(true, &state)), + ScriptRewriteAction::Keep, + "should not partially hide a batch with uncaptured qualified payloads" + ); + let shared = document_state(&state); + let state = shared.lock().expect("should lock state"); + assert!(state.captured_payloads.is_empty()); + assert!(state.current_fragment_protected); + } + + #[test] + fn inert_controls_do_not_make_unsafe_batches_partially_capturable() { + for unsafe_call in [ + r#"self.__next_f.push([2,"1:T40,'http://www.googletagmanager.com/gtm.js'"])"#, + "self.__next_f.push([0,1])", + "self.__next_f.push([2,null", + "self.__next_f.push([0]", + ] { + for unsafe_first in [false, true] { + let state = IntegrationDocumentState::default(); + let rewriter = NextJsRscPlaceholderRewriter::new(test_config()); + let control = "self.__next_f.push([0]);self.__next_f.push([2,null])"; + let supported = r#"self.__next_f.push([1,"later"])"#; + let script = if unsafe_first { + format!("{control};{unsafe_call};{supported}") + } else { + format!("{control};{supported};{unsafe_call}") + }; + assert_eq!( + rewriter.rewrite(&script, &ctx(true, &state)), + ScriptRewriteAction::Keep, + "should preserve unsafe batch: {script}" + ); + let shared = document_state(&state); + let state = shared.lock().expect("should lock state"); + assert!(state.captured_payloads.is_empty()); + assert!(state.current_fragment_protected); + assert!(state.bypass_rsc); + } + } + } + + #[test] + fn inert_controls_are_preserved_without_document_bypass() { + for script in [ + "(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])", + "window.__next_f . push ( [ 0 ] );window.__next_f.push( [ 2 , null ] )", + ] { + let state = IntegrationDocumentState::default(); + let rewriter = NextJsRscPlaceholderRewriter::new(test_config()); + assert_eq!( + rewriter.rewrite(script, &ctx(true, &state)), + ScriptRewriteAction::Keep + ); + let shared = document_state(&state); + let state = shared.lock().expect("should lock state"); + assert!(state.captured_payloads.is_empty()); + assert!(state.current_fragment_protected); + assert!(!state.bypass_rsc); + } + } + + #[test] + fn batched_capture_is_atomic_at_byte_and_count_limits() { + let payload = "x".repeat(40); + let push = format!(r#"self.__next_f.push([1,"{payload}"])"#); + for (count, budget, captured) in + [(2, 80, 2), (2, 79, 0), (256, 20000, 256), (257, 20000, 0)] + { + let state = IntegrationDocumentState::default(); + let rewriter = NextJsRscPlaceholderRewriter::new(test_config()); + let script = vec![push.as_str(); count].join(";"); + let action = rewriter.rewrite( + &script, + &IntegrationScriptContext { + max_buffered_script_bytes: budget, + ..ctx(true, &state) + }, + ); + let shared = document_state(&state); + let state = shared.lock().expect("should lock state"); + assert_eq!( + state.captured_payloads.len(), + captured, + "should commit every push or none" + ); + assert_eq!(state.captured_payload_bytes, captured * payload.len()); + assert_eq!( + matches!(action, ScriptRewriteAction::Replace(_)), + captured > 0 + ); + assert_eq!( + state.current_fragment_protected, + captured == 0, + "should protect rejected raw batch" + ); + } + let state = IntegrationDocumentState::default(); + let rewriter = NextJsRscPlaceholderRewriter::new(test_config()); + let script = format!("{push};self.__next_f.push([1,\"unfinished"); + assert_eq!( + rewriter.rewrite(&script, &ctx(true, &state)), + ScriptRewriteAction::Keep, + "should leave unsafe remainder original" + ); + assert!( + document_state(&state) + .lock() + .expect("should lock state") + .captured_payloads + .is_empty() + ); + } + + #[test] + fn bypass_ownership_recognizes_every_split_and_resets_on_empty_final() { + let mut settings = crate::test_support::tests::create_test_settings(); + settings + .integrations + .insert_config( + "google_tag_manager", + &serde_json::json!({"enabled": true, "container_id": "GTM-MIX1"}), + ) + .expect("should enable GTM"); + let registry = crate::integrations::IntegrationRegistry::with_plan( + &settings, + Arc::new(crate::auction::compile_auction_plan(&settings).expect("should compile plan")), + ) + .expect("should create registry"); + let gtm = registry + .script_rewriters() + .pop() + .expect("should register GTM"); + let rewriter = NextJsRscPlaceholderRewriter::new(test_config()); + let spaced_initializer = format!( + r#"(self.__next_f{}={}window.__next_f{}||{}[]{}).{}push{}([1,"1:T40,'http://www.googletagmanager.com/gtm.js'"])"#, + " ".repeat(256), + " ".repeat(256), + " ".repeat(256), + " ".repeat(256), + " ".repeat(256), + " ".repeat(256), + " ".repeat(256) + ); + for script in [ + r#"self.__next_f.push([1,"1:T40,'http://www.googletagmanager.com/gtm.js'"])"#, + r#"window.__next_f.push([1,"1:T40,'http://www.googletagmanager.com/gtm.js'"])"#, + r#"(self.__next_f=self.__next_f||[]).push([1,"1:T40,'http://www.googletagmanager.com/gtm.js'"])"#, + r#"(window.__next_f=window.__next_f||[]).push([1,"1:T40,'http://www.googletagmanager.com/gtm.js'"])"#, + &spaced_initializer, + ] { + for split in 1..script.len() { + let state = IntegrationDocumentState::default(); + document_state(&state) + .lock() + .expect("should lock state") + .bypass_rsc = true; + for fragment in [&script[..split], &script[split..], ""] { + let final_fragment = fragment.is_empty(); + assert_eq!( + rewriter.rewrite(fragment, &ctx(final_fragment, &state)), + ScriptRewriteAction::Keep + ); + assert_eq!( + IntegrationScriptRewriter::rewrite( + &*gtm, + fragment, + &ctx(final_fragment, &state) + ), + ScriptRewriteAction::Keep, + "should preserve raw Flight at split {split}" + ); + assert!( + document_state(&state) + .lock() + .expect("should lock state") + .flight_qualifier_tail + .len() + <= FLIGHT_QUALIFIER_TAIL_BYTES, + "should bound fallback qualification observation" + ); + } + let ordinary = "var ordinary='http://www.googletagmanager.com/gtm.js';"; + assert_eq!( + rewriter.rewrite(ordinary, &ctx(true, &state)), + ScriptRewriteAction::Keep + ); + assert!( + matches!(IntegrationScriptRewriter::rewrite(&*gtm, ordinary, &ctx(true, &state)), ScriptRewriteAction::Replace(ref text) if text.contains("/integrations/google_tag_manager/gtm.js")) + ); + } + } + } + + #[test] + fn raw_flight_fallback_is_protected_but_later_ordinary_gtm_is_not() { + let state = IntegrationDocumentState::default(); + let rewriter = NextJsRscPlaceholderRewriter::new(Arc::new(NextJsIntegrationConfig { + max_combined_payload_bytes: 24, + ..(*test_config()).clone() + })); + let mut settings = crate::test_support::tests::create_test_settings(); + settings + .integrations + .insert_config( + "google_tag_manager", + &serde_json::json!({"enabled": true, "container_id": "GTM-MIX1"}), + ) + .expect("should configure GTM"); + let registry = crate::integrations::IntegrationRegistry::with_plan( + &settings, + Arc::new(crate::auction::compile_auction_plan(&settings).expect("should compile plan")), + ) + .expect("should create registry"); + let gtm = registry + .script_rewriters() + .pop() + .expect("should register script hook"); + let raw = r#"self.__next_f.push([1,"1:T2,'http://www.googletagmanager.com/gtm.js?id=GTM-MIX1'"] )"#; + assert_eq!( + rewriter.rewrite(raw, &ctx(true, &state)), + ScriptRewriteAction::Keep + ); + assert_eq!( + IntegrationScriptRewriter::rewrite(&*gtm, raw, &ctx(true, &state)), + ScriptRewriteAction::Keep, + "should not change original T bytes on overflow" + ); + for part in [ + "self.__n", + "ext_f.push([1,\"", + "1:T2,'http://www.googletagmanager.com/gtm.js?id=GTM-MIX1'\"])", + ] { + let last = part.ends_with(")"); + assert_eq!( + rewriter.rewrite(part, &ctx(last, &state)), + ScriptRewriteAction::Keep + ); + assert_eq!( + IntegrationScriptRewriter::rewrite(&*gtm, part, &ctx(last, &state)), + ScriptRewriteAction::Keep, + "should recognize and protect split qualified flight after bypass" + ); + } + let prefix = "var before='google"; + assert_eq!( + rewriter.rewrite(prefix, &ctx(false, &state)), + ScriptRewriteAction::Keep + ); + assert_eq!( + IntegrationScriptRewriter::rewrite(&*gtm, prefix, &ctx(false, &state)), + ScriptRewriteAction::RemoveNode + ); + let flight = r#"';self.__next_f.push([1,"1:T2,'http://www.googletagmanager.com/gtm.js'"])"#; + assert_eq!( + rewriter.rewrite(flight, &ctx(false, &state)), + ScriptRewriteAction::Keep + ); + assert_eq!( + IntegrationScriptRewriter::rewrite(&*gtm, flight, &ctx(false, &state)), + ScriptRewriteAction::Replace(format!("{prefix}{flight}")), + "should drain GTM-held prefix unchanged before protected raw text" + ); + assert_eq!( + rewriter.rewrite("", &ctx(true, &state)), + ScriptRewriteAction::Keep + ); + assert_eq!( + IntegrationScriptRewriter::rewrite(&*gtm, "", &ctx(true, &state)), + ScriptRewriteAction::Keep, + "should preserve empty final protection" + ); + let ordinary = "var url='http://www.googletagmanager.com/gtm.js';"; + assert_eq!( + rewriter.rewrite(ordinary, &ctx(true, &state)), + ScriptRewriteAction::Keep + ); + assert!( + matches!(IntegrationScriptRewriter::rewrite(&*gtm, ordinary, &ctx(true, &state)), ScriptRewriteAction::Replace(ref value) if value.contains("/integrations/google_tag_manager/gtm.js")), + "should not leak flight ownership to ordinary scripts" + ); + } + #[test] fn inserts_placeholder_and_records_payload() { let state = IntegrationDocumentState::default(); @@ -459,13 +900,13 @@ mod tests { })); assert_eq!( - rewriter.rewrite("self.__next_f", &ctx(false, &state)), + rewriter.rewrite("self.__next_f.push(", &ctx(false, &state)), ScriptRewriteAction::RemoveNode, "should initially suppress the script prefix", ); assert_eq!( rewriter.rewrite("-payload-overflow", &ctx(false, &state)), - ScriptRewriteAction::Replace("self.__next_f-payload-overflow".to_owned()), + ScriptRewriteAction::Replace("self.__next_f.push(-payload-overflow".to_owned()), "should restore suppressed text before overflow", ); assert_eq!( diff --git a/crates/trusted-server-core/src/integrations/nextjs/rsc_stream.rs b/crates/trusted-server-core/src/integrations/nextjs/rsc_stream.rs index ab12e13f1d..aa3262b373 100644 --- a/crates/trusted-server-core/src/integrations/nextjs/rsc_stream.rs +++ b/crates/trusted-server-core/src/integrations/nextjs/rsc_stream.rs @@ -9,7 +9,7 @@ use crate::streaming_processor::StreamProcessor; use super::rsc::{ DEFAULT_MAX_COMBINED_PAYLOAD_BYTES, PendingTChunk, TChunkStep, next_tchunk, - rewrite_rsc_scripts_combined_with_limit, + rewrite_rsc_scripts_combined_with_transform, }; use super::shared::RscUrlRewriter; use super::{NEXTJS_INTEGRATION_ID, NextJsIntegrationConfig}; @@ -36,6 +36,9 @@ pub(super) struct CapturedPayload { pub(super) struct NextJsDocumentState { pub(super) namespace: String, pub(super) next_data: FragmentState, + /// One-shot discriminator set by the matched Pages stage and consumed by + /// the immediately following Flight stage, including empty/pass-through text. + pub(super) next_data_fragment: bool, pub(super) rsc_script: FragmentState, pub(super) rsc_probe: String, /// Tail of script text already released for the current text node, kept so a @@ -49,6 +52,12 @@ pub(super) struct NextJsDocumentState { pub(super) captured_payload_bytes: usize, pub(super) next_placeholder_index: usize, pub(super) bypass_rsc: bool, + /// Qualified Flight ownership lasts only for the current script text node. + pub(super) flight_node_owned: bool, + /// Snapshot consumed by downstream GTM, including the final raw fragment. + pub(super) current_fragment_protected: bool, + /// Bounded qualifier observation remains active even after document bypass. + pub(super) flight_qualifier_tail: String, } impl Default for NextJsDocumentState { @@ -56,6 +65,7 @@ impl Default for NextJsDocumentState { Self { namespace: uuid::Uuid::new_v4().simple().to_string(), next_data: FragmentState::Idle, + next_data_fragment: false, rsc_script: FragmentState::Idle, rsc_probe: String::new(), rsc_receiver_context: String::new(), @@ -64,6 +74,9 @@ impl Default for NextJsDocumentState { captured_payload_bytes: 0, next_placeholder_index: 0, bypass_rsc: false, + flight_node_owned: false, + current_fragment_protected: false, + flight_qualifier_tail: String::new(), } } } @@ -145,11 +158,15 @@ pub(super) fn capture_fragment<'a>( pub(super) struct NextJsRscStreamProcessorFactory { config: Arc, + transform: Option String>, } impl NextJsRscStreamProcessorFactory { - pub(super) fn new(config: Arc) -> Self { - Self { config } + pub(super) fn new( + config: Arc, + transform: Option String>, + ) -> Self { + Self { config, transform } } } @@ -170,6 +187,7 @@ impl IntegrationHtmlStreamProcessorFactory for NextJsRscStreamProcessorFactory { context.request_host, context.request_scheme, limit, + self.transform, )) } } @@ -185,6 +203,7 @@ pub(super) struct NextJsRscStreamProcessor { group: Vec, classifier: RscGroupClassifier, rewriter: RscUrlRewriter, + transform: Option String>, } impl NextJsRscStreamProcessor { @@ -194,6 +213,7 @@ impl NextJsRscStreamProcessor { request_host: String, request_scheme: String, limit: usize, + transform: Option String>, ) -> Self { Self { state, @@ -206,6 +226,7 @@ impl NextJsRscStreamProcessor { group: Vec::new(), classifier: RscGroupClassifier::new(limit), rewriter: RscUrlRewriter::new(), + transform, } } @@ -325,13 +346,14 @@ impl NextJsRscStreamProcessor { .iter() .map(|payload| payload.original.as_str()) .collect(); - let rewritten = rewrite_rsc_scripts_combined_with_limit( + let rewritten = rewrite_rsc_scripts_combined_with_transform( &payloads, &self.rewriter, &self.origin_host, &self.request_host, &self.request_scheme, self.limit, + self.transform, ); if rewritten.len() != self.group.len() { log::warn!( @@ -1126,11 +1148,60 @@ mod tests { "proxy.example.com".to_owned(), "https".to_owned(), limit, + None, ), placeholders, ) } + #[test] + fn enabled_gtm_transform_never_mutates_fallback_originals() { + let body = "'http://www.googletagmanager.com/gtm.js'"; + let invalid = format!("1:Tzz,{body}"); + let incomplete = format!("1:Tffff,{body}"); + let header_tail = format!("T{:x},{body}", body.len()); + let escape_tail = format!("41{body}"); + let marker_tail = format!("c\n\0SPLIT\0{body}"); + for payloads in [ + vec![invalid.as_str()], + vec![incomplete.as_str()], + vec!["1:", header_tail.as_str()], + vec![r"1:T2,\u00", escape_tail.as_str()], + vec!["1:T3,ab", marker_tail.as_str()], + ] { + let (mut processor, placeholders) = processor_with_payloads(&payloads, 10000); + processor.transform = + Some(crate::integrations::google_tag_manager::rewrite_gtm_rsc_span); + let output = processor + .process_chunk(placeholders.concat().as_bytes(), true) + .expect("should restore fallback group"); + assert_eq!( + output, + payloads.concat().as_bytes(), + "should never transform original fallback bytes" + ); + } + let (mut processor, placeholders) = processor_with_payloads(&[&incomplete, body], 120); + processor.transform = Some(crate::integrations::google_tag_manager::rewrite_gtm_rsc_span); + let interstitial = "x".repeat(100); + let input = format!("{}{interstitial}{}", placeholders[0], placeholders[1]); + assert_eq!( + processor + .process_chunk(input.as_bytes(), true) + .expect("should restore held-output overflow"), + format!("{incomplete}{interstitial}{body}").as_bytes() + ); + let (mut processor, _) = processor_with_payloads(&[body], 1000); + processor.transform = Some(crate::integrations::google_tag_manager::rewrite_gtm_rsc_span); + assert!( + processor + .process_chunk(&[], true) + .expect_err("should reject missing captured placeholder") + .to_string() + .contains("not present") + ); + } + #[test] fn stream_processor_emits_ordinary_html_before_eof() { let (mut processor, _) = processor_with_payloads(&[], 1024); diff --git a/crates/trusted-server-core/src/integrations/nextjs/script_rewriter.rs b/crates/trusted-server-core/src/integrations/nextjs/script_rewriter.rs index b6de84449c..e454626205 100644 --- a/crates/trusted-server-core/src/integrations/nextjs/script_rewriter.rs +++ b/crates/trusted-server-core/src/integrations/nextjs/script_rewriter.rs @@ -62,14 +62,14 @@ impl IntegrationScriptRewriter for NextJsNextDataRewriter { } fn rewrite(&self, content: &str, ctx: &IntegrationScriptContext<'_>) -> ScriptRewriteAction { - if self.config.rewrite_attributes.is_empty() { - return ScriptRewriteAction::keep(); - } - let state = document_state(ctx.document_state); let mut state = state .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); + state.next_data_fragment = true; + if self.config.rewrite_attributes.is_empty() { + return ScriptRewriteAction::keep(); + } match capture_fragment( &mut state.next_data, diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 47c521e610..a41f986f16 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -55,11 +55,18 @@ pub enum AttributeRewriteOutcome { RemoveElement, } -/// Action returned by inline script rewriters to describe how to mutate the node. +/// Action on the current working script text fragment, composed in registration order. +/// +/// Every matching stage receives the previous stage's output, including empty +/// suppressed text and the original final-fragment flag. Only the composed result +/// is applied to the parser fragment. #[derive(Debug, Clone, PartialEq, Eq)] pub enum ScriptRewriteAction { + /// Preserve the current working text, not the original parser text. Keep, + /// Pass replacement text to subsequent matching stages. Replace(String), + /// Suppress this fragment; downstream stages still run and may flush held text. RemoveNode, } @@ -93,6 +100,7 @@ pub struct IntegrationAttributeContext<'a> { /// Context passed to script/text rewriters for inline HTML handling. #[derive(Debug)] pub struct IntegrationScriptContext<'a> { + /// The individual rewriter's CSS selector, matched by `lol_html`. pub selector: &'a str, pub request_host: &'a str, pub request_scheme: &'a str, diff --git a/crates/trusted-server-core/src/test_support.rs b/crates/trusted-server-core/src/test_support.rs index d6e85a167f..44889501fd 100644 --- a/crates/trusted-server-core/src/test_support.rs +++ b/crates/trusted-server-core/src/test_support.rs @@ -66,11 +66,14 @@ pub mod tests { /// drive byte-identical input. #[cfg(any(test, feature = "test-utils"))] pub mod nextjs_auction { + use std::io::{Read as _, Write as _}; use std::net::IpAddr; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; + use bytes::Bytes; use error_stack::Report; + use flate2::{read::GzDecoder, write::GzEncoder}; use crate::geo::GeoInfo; use crate::platform::{ @@ -194,13 +197,150 @@ pub mod nextjs_auction { ) } + /// Isolated Next.js/GTM settings without auction dispatch. + /// + /// # Panics + /// + /// Panics if the static GTM configuration is invalid. + #[must_use] + pub fn script_composition_settings() -> Settings { + let mut settings = settings(); + settings.auction.enabled = false; + settings.creative_opportunities = None; + settings + .integrations + .insert_config( + "google_tag_manager", + &serde_json::json!({ + "enabled": true, "container_id": "GTM-MIX1" + }), + ) + .expect("should enable fixture GTM"); + settings + } + + const SCRIPT_COMPOSITION_BOOTSTRAP: &str = + "(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])"; + + /// HTML with standard bootstrap controls and a mixed Flight T record, + /// crossing parser-internal boundaries with same-delimiter query escapes. + #[must_use] + pub fn script_composition_html() -> String { + let content = serde_json::json!({ + "url": format!("https://{ORIGIN_HOST}/app"), + "tag": "http://www.googletagmanager.com/gtm.js?q=\"x\"&v=é", + "ga": "//www.google-analytics.com/g/collect?v=2", + "text": "é😀", "padding": "_".repeat(11000), + }) + .to_string(); + let payload = serde_json::json!(format!("1:T{:x},{content}", content.len())); + format!( + "

script-composition-suffix

" + ) + } + + /// Assert fully consumed adapter output, decoded JSON and recomputed T length. + /// + /// # Panics + /// + /// Panics if response coding, Flight syntax, rewrites, or completion are wrong. + pub fn assert_script_composition_response(bytes: &[u8], gzip: bool) { + let mut decoded = Vec::new(); + if gzip { + GzDecoder::new(bytes) + .read_to_end(&mut decoded) + .expect("should decode complete gzip response"); + } else { + decoded.extend_from_slice(bytes); + } + let html = String::from_utf8(decoded).expect("should emit UTF-8 HTML"); + assert!( + html.contains("

script-composition-suffix

"), + "should deliver suffix sentinel" + ); + assert!(html.ends_with(""), "should finish HTML"); + assert!( + !html.contains("__ts_rsc_"), + "should resolve every captured payload" + ); + assert!( + html.contains(SCRIPT_COMPOSITION_BOOTSTRAP), + "should preserve inert bootstrap controls" + ); + assert_eq!( + html.matches("self.__next_f.push([1,").count(), + 1, + "should emit Flight once" + ); + let json = html + .rsplit("self.__next_f.push(") + .next() + .expect("should keep push") + .split(")") + .next() + .expect("should close push"); + let push: serde_json::Value = + serde_json::from_str(json).expect("should retain valid push JSON"); + let payload = push[1].as_str().expect("should retain payload"); + let (header, body) = payload.split_once(',').expect("should retain T record"); + assert_eq!( + usize::from_str_radix(&header[3..], 16).expect("should parse hex length"), + body.len(), + "should recount decoded T bytes" + ); + let data: serde_json::Value = + serde_json::from_str(body).expect("should preserve valid model JSON"); + assert!( + data["url"] + .as_str() + .expect("should retain URL") + .contains(&format!("{PUBLISHER_HOST}/app")), + "should rewrite origin" + ); + assert_eq!( + data["tag"], + "/integrations/google_tag_manager/gtm.js?q=\"x\"&v=é" + ); + assert_eq!(data["ga"], "/integrations/google_tag_manager/g/collect?v=2"); + assert_eq!(data["text"], "é😀"); + assert_eq!(data["padding"], "_".repeat(11000)); + } + /// Upstream that serves [`origin_html`] and one deterministic bid. #[derive(Default)] pub struct NextJsAuctionOrigin { auction_requests: AtomicUsize, + html_response: Option>, + gzip: bool, + streaming: bool, } impl NextJsAuctionOrigin { + /// Serve canned HTML instead of the default fixture, optionally gzip-encoded + /// and streamed in small chunks. The default fixture remains unchanged. + /// + /// # Panics + /// + /// Panics if in-memory gzip encoding fails. + #[must_use] + pub fn with_html_response(html: &str, gzip: bool, streaming: bool) -> Self { + let bytes = if gzip { + let mut encoder = GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder + .write_all(html.as_bytes()) + .expect("should encode fixture gzip"); + encoder.finish().expect("should finish fixture gzip") + } else { + html.as_bytes().to_vec() + }; + Self { + html_response: Some(bytes), + gzip, + streaming, + ..Self::default() + } + } + /// Number of auction requests this fixture has answered. #[must_use] pub fn auction_requests(&self) -> usize { @@ -215,6 +355,10 @@ pub mod nextjs_auction { #[async_trait::async_trait(?Send)] impl PlatformHttpClient for NextJsAuctionOrigin { + fn supports_streaming_responses(&self) -> bool { + self.streaming + } + async fn send( &self, request: PlatformHttpRequest, @@ -241,11 +385,31 @@ pub mod nextjs_auction { .attach(format!("unexpected fixture upstream: {host:?}"))); } }; + let canned_origin = + request.request.uri().host() == Some(ORIGIN_HOST) && self.html_response.is_some(); + let bytes = if canned_origin { + self.html_response + .as_ref() + .expect("should have canned response") + .clone() + } else { + body.into_bytes() + }; + let mut response = http::Response::builder() + .status(200) + .header("content-type", content_type); + if canned_origin && self.gzip { + response = response.header("content-encoding", "gzip"); + } + let body = if request.stream_response && self.streaming { + let chunks: Vec = bytes.chunks(97).map(Bytes::copy_from_slice).collect(); + edgezero_core::body::Body::stream(futures::stream::iter(chunks)) + } else { + edgezero_core::body::Body::from(bytes) + }; Ok(PlatformResponse::new( - http::Response::builder() - .status(200) - .header("content-type", content_type) - .body(edgezero_core::body::Body::from(body)) + response + .body(body) .expect("should build deterministic upstream response"), ) .with_backend_name(request.backend_name)) From 770f3d62871e95990b9d8a1a911aadd24d9ef168 Mon Sep 17 00:00:00 2001 From: Christian Date: Wed, 30 Sep 2026 17:27:10 -0500 Subject: [PATCH 02/15] Share compressed HTML processing in Next.js tests --- .../src/integrations/nextjs/mod.rs | 99 ++++++++----------- 1 file changed, 39 insertions(+), 60 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/nextjs/mod.rs b/crates/trusted-server-core/src/integrations/nextjs/mod.rs index 4036e5e768..07c687745f 100644 --- a/crates/trusted-server-core/src/integrations/nextjs/mod.rs +++ b/crates/trusted-server-core/src/integrations/nextjs/mod.rs @@ -215,6 +215,43 @@ mod tests { String::from_utf8(output).expect("should emit UTF-8") } + fn mixed_output_with_compression( + html: &str, + compression: Compression, + chunk_size: usize, + ) -> String { + let input = if compression == Compression::Gzip { + let mut encoder = GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder + .write_all(html.as_bytes()) + .expect("should encode gzip input"); + encoder.finish().expect("should finish gzip input") + } else { + html.as_bytes().to_vec() + }; + let mut pipeline = StreamingPipeline::new( + PipelineConfig { + input_compression: compression, + output_compression: compression, + chunk_size, + }, + create_html_processor(mixed_config(100000, 100000)), + ); + let mut output = Vec::new(); + pipeline + .process(Cursor::new(&input), &mut output) + .expect("should consume complete mixed response"); + let mut decoded = Vec::new(); + if compression == Compression::Gzip { + GzDecoder::new(output.as_slice()) + .read_to_end(&mut decoded) + .expect("should decode gzip output"); + } else { + decoded = output; + } + String::from_utf8(decoded).expect("should emit UTF-8") + } + #[test] fn mixed_next_data_flight_mentions_do_not_claim_json_or_hide_gtm() { for text in [ @@ -447,37 +484,8 @@ mod tests { }; let html = format!("{scripts}

suffix

"); for compression in [Compression::None, Compression::Gzip] { - let input = if compression == Compression::Gzip { - let mut encoder = GzEncoder::new(Vec::new(), flate2::Compression::default()); - encoder - .write_all(html.as_bytes()) - .expect("should encode gzip bootstrap document"); - encoder.finish().expect("should finish gzip input") - } else { - html.as_bytes().to_vec() - }; for chunk_size in [32, 64, 1000, 8192] { - let mut pipeline = StreamingPipeline::new( - PipelineConfig { - input_compression: compression, - output_compression: compression, - chunk_size, - }, - create_html_processor(mixed_config(100000, 100000)), - ); - let mut bytes = Vec::new(); - pipeline - .process(Cursor::new(&input), &mut bytes) - .expect("should consume bootstrap document"); - let mut decoded = Vec::new(); - if compression == Compression::Gzip { - GzDecoder::new(bytes.as_slice()) - .read_to_end(&mut decoded) - .expect("should decode gzip bootstrap response"); - } else { - decoded = bytes; - } - let output = String::from_utf8(decoded).expect("should emit UTF-8"); + let output = mixed_output_with_compression(&html, compression, chunk_size); assert!( output.contains(controls), "should preserve inert bootstrap controls" @@ -695,37 +703,8 @@ mod tests { "

suffix

" ); for compression in [Compression::None, Compression::Gzip] { - let input = if compression == Compression::Gzip { - let mut encoder = GzEncoder::new(Vec::new(), flate2::Compression::default()); - encoder - .write_all(html.as_bytes()) - .expect("should encode actual gzip input"); - encoder.finish().expect("should finish gzip input") - } else { - html.as_bytes().to_vec() - }; for chunk_size in [32, 64, 1000, 8192] { - let mut pipeline = StreamingPipeline::new( - PipelineConfig { - input_compression: compression, - output_compression: compression, - chunk_size, - }, - create_html_processor(mixed_config(100000, 100000)), - ); - let mut output = Vec::new(); - pipeline - .process(Cursor::new(&input), &mut output) - .expect("should process complete compressed document"); - let mut decoded = Vec::new(); - if compression == Compression::Gzip { - GzDecoder::new(output.as_slice()) - .read_to_end(&mut decoded) - .expect("should decode actual gzip output"); - } else { - decoded = output; - } - let output = String::from_utf8(decoded).expect("should emit UTF-8"); + let output = mixed_output_with_compression(&html, compression, chunk_size); let script = output .split("{flight}", + &prefix[split..] + ); + let mut bytes = processor + .process_chunk(first.as_bytes(), false) + .expect("should accept non-code prefix"); + bytes.extend( + processor + .process_chunk(last.as_bytes(), true) + .expect("should complete ordinary and Flight scripts"), + ); + let output = String::from_utf8(bytes).expect("should emit UTF-8"); + assert!( + output.contains(&format!("")), + "should preserve non-code source and rewrite ordinary GTM at split {split}: {output}" + ); + assert!( + output.contains(r#"\"url\":\"https://test.example.com/page\""#), + "should still rewrite later real Flight at split {split}: {output}" + ); + assert!( + !output.contains("__ts_rsc_"), + "should resolve every placeholder" + ); + } + } + } + + #[test] + fn lexical_filter_captures_only_executable_pushes_in_the_same_script() { + let fake = r#"const demo='self.__next_f.push([1,"0:{\"url\":\"https://origin.example.com/fake\"}\n"])';"#; + let push = r#"self.__next_f.push([1,"0:{\"url\":\"https://origin.example.com/page\"}\n"])"#; + let rewritten_push = push.replace("origin.example.com", "test.example.com"); + for (source, expected) in [ + ( + format!("{fake}{push};/* self.__next_f.push( */"), + format!("{fake}{rewritten_push};/* self.__next_f.push( */"), + ), + ( + format!( + "const out=`text self.__next_f.push( ${{ {{a: {push}, b: `nested ${{{push}}}`}} }} tail`;" + ), + format!( + "const out=`text self.__next_f.push( ${{ {{a: {rewritten_push}, b: `nested ${{{rewritten_push}}}`}} }} tail`;" + ), + ), + ( + format!( + r#"const re=/["'`/]/;if (ready) /["'`/]/.test(input);const ratio=i++ / 2;const value=obj.return / 2;{push};"# + ), + format!( + r#"const re=/["'`/]/;if (ready) /["'`/]/.test(input);const ratio=i++ / 2;const value=obj.return / 2;{rewritten_push};"# + ), + ), + ] { + for split in 0..=source.len() { + let mut processor = create_html_processor(mixed_config(10000, 10000)); + let first = format!("", &source[split..]); + let mut bytes = processor + .process_chunk(first.as_bytes(), false) + .expect("should accept script prefix"); + bytes.extend( + processor + .process_chunk(last.as_bytes(), true) + .expect("should complete executable Flight"), + ); + assert_eq!( + String::from_utf8(bytes).expect("should emit UTF-8"), + format!(""), + "should rewrite only executable calls at split {split}" + ); + } + } + } + #[test] fn mixed_next_data_flight_mentions_do_not_claim_json_or_hide_gtm() { for text in [ diff --git a/crates/trusted-server-core/src/integrations/nextjs/rsc_placeholders.rs b/crates/trusted-server-core/src/integrations/nextjs/rsc_placeholders.rs index 9b1c783115..7abe3aa7ca 100644 --- a/crates/trusted-server-core/src/integrations/nextjs/rsc_placeholders.rs +++ b/crates/trusted-server-core/src/integrations/nextjs/rsc_placeholders.rs @@ -13,6 +13,7 @@ use super::rsc_stream::{ CapturedPayload, FragmentCapture, MAX_UNRESOLVED_RSC_PAYLOADS, RscGroupStatus, capture_fragment, classify_rsc_group, document_state, rsc_payload_placeholder, }; +use super::script_lexer::ScriptLexer; use super::shared::{ RSC_PUSH_CALL_PATTERN, RSC_PUSH_CALL_PATTERN_TRIMMED, RSC_RECEIVER_CONTEXT_BYTES, find_rsc_push_payload_range, find_trimmed_rsc_push_payload_range, @@ -62,7 +63,8 @@ impl NextJsRscPlaceholderRewriter { limit: usize, max_queued_payload_bytes: usize, ) -> ScriptRewriteAction { - if !content.contains("__next_f") { + let code = state.rsc_lexical_start.clone().mask(content); + if !code.contains("__next_f") { return if was_buffered { ScriptRewriteAction::replace(content.to_owned()) } else { @@ -76,12 +78,13 @@ impl NextJsRscPlaceholderRewriter { let mut cursor = 0; let mut trimmed = std::mem::take(&mut state.rsc_receiver_trimmed); let mut queued_bytes = state.captured_payload_bytes; - while content[cursor..].contains("__next_f") { + while code[cursor..].contains("__next_f") { let remaining = &content[cursor..]; + let remaining_code = &code[cursor..]; let head = if trimmed { - TRIMMED_FLIGHT_PUSH_PATTERN.find(remaining) + TRIMMED_FLIGHT_PUSH_PATTERN.find(remaining_code) } else { - first_qualified_flight_push(remaining) + first_qualified_flight_push(remaining_code) }; if let Some(head) = head && let Some(arguments) = @@ -91,20 +94,17 @@ impl NextJsRscPlaceholderRewriter { trimmed = false; continue; } - let range = if trimmed { - find_trimmed_rsc_push_payload_range(remaining) - } else { - find_rsc_push_payload_range(remaining) - }; + let range = executable_payload_range(remaining, remaining_code, trimmed); let call = if trimmed { - RSC_PUSH_CALL_PATTERN_TRIMMED.find(remaining) + RSC_PUSH_CALL_PATTERN_TRIMMED.find(remaining_code) } else { - RSC_PUSH_CALL_PATTERN.find(remaining) + RSC_PUSH_CALL_PATTERN.find(remaining_code) }; trimmed = false; let Some((start, end)) = range.filter(|(start, _)| { call.is_some_and(|call| { - call.end() == *start && !contains_qualified_flight(&remaining[..call.start()]) + call.end() == *start + && !contains_qualified_flight(&remaining_code[..call.start()]) }) }) else { // A raw qualified push cannot safely continue a pending T group @@ -198,14 +198,13 @@ impl NextJsRscPlaceholderRewriter { return ScriptRewriteAction::Keep; } if is_last && content.len() > limit && content.contains("__next_f") { + let code = state.rsc_lexical_start.clone().mask(content); let mut remaining = content; + let mut remaining_code = code.as_str(); let mut receiver_trimmed = trimmed; let unsafe_continuation = loop { - let range = if receiver_trimmed { - find_trimmed_rsc_push_payload_range(remaining) - } else { - find_rsc_push_payload_range(remaining) - }; + let range = + executable_payload_range(remaining, remaining_code, receiver_trimmed); let Some((start, end)) = range else { break true; }; @@ -219,8 +218,9 @@ impl NextJsRscPlaceholderRewriter { // payload is not classified as another call. Only the // first receiver can have streamed in an earlier fragment. remaining = &remaining[end + 1..]; + remaining_code = &remaining_code[end + 1..]; receiver_trimmed = false; - if !remaining.contains("__next_f") { + if !remaining_code.contains("__next_f") { break false; } }; @@ -237,6 +237,8 @@ impl NextJsRscPlaceholderRewriter { fn rewrite_fragment( &self, content: &str, + code: &str, + lexical_start: ScriptLexer, ctx: &IntegrationScriptContext<'_>, state: &mut super::rsc_stream::NextJsDocumentState, ) -> ScriptRewriteAction { @@ -272,15 +274,15 @@ impl NextJsRscPlaceholderRewriter { ); } - if state.rsc_probe.is_empty() && !content.contains("__next_f") { + if state.rsc_probe.is_empty() && !code.contains("__next_f") { if ctx.is_last_in_text_node { state.rsc_receiver_context.clear(); return ScriptRewriteAction::Keep; } - let probe_length = longest_identifier_prefix(content.as_bytes()); + let probe_length = longest_identifier_prefix(code.as_bytes()); let ready_length = content.len() - probe_length; state.rsc_probe.push_str(&content[ready_length..]); - remember_released(&mut state.rsc_receiver_context, &content[..ready_length]); + remember_released(&mut state.rsc_receiver_context, &code[..ready_length]); return if probe_length == 0 { ScriptRewriteAction::Keep } else if ready_length == 0 { @@ -293,7 +295,8 @@ impl NextJsRscPlaceholderRewriter { let prior_probe = std::mem::take(&mut state.rsc_probe); let mut combined = prior_probe.clone(); combined.push_str(content); - if !combined.contains("__next_f") { + let combined_code = format!("{prior_probe}{code}"); + if !combined_code.contains("__next_f") { if ctx.is_last_in_text_node { state.rsc_receiver_context.clear(); return if prior_probe.is_empty() { @@ -302,10 +305,13 @@ impl NextJsRscPlaceholderRewriter { ScriptRewriteAction::replace(combined) }; } - let probe_length = longest_identifier_prefix(combined.as_bytes()); + let probe_length = longest_identifier_prefix(combined_code.as_bytes()); let ready_length = combined.len() - probe_length; state.rsc_probe.push_str(&combined[ready_length..]); - remember_released(&mut state.rsc_receiver_context, &combined[..ready_length]); + remember_released( + &mut state.rsc_receiver_context, + &combined_code[..ready_length], + ); if prior_probe.is_empty() && probe_length == 0 { return ScriptRewriteAction::Keep; } @@ -316,18 +322,18 @@ impl NextJsRscPlaceholderRewriter { }; } - let identifier_start = combined + let identifier_start = combined_code .find("__next_f") .expect("should find the identifier that selected this branch"); let mut context = state.rsc_receiver_context.clone(); - context.push_str(&combined[..identifier_start]); + context.push_str(&combined_code[..identifier_start]); if !receiver_context_is_flight_push(&context) { // Some other object owns a `__next_f` property. Release the text // unchanged rather than claiming an unrelated publisher script. if ctx.is_last_in_text_node { state.rsc_receiver_context.clear(); } else { - remember_released(&mut state.rsc_receiver_context, &combined); + remember_released(&mut state.rsc_receiver_context, &combined_code); } return if prior_probe.is_empty() { ScriptRewriteAction::Keep @@ -340,7 +346,7 @@ impl NextJsRscPlaceholderRewriter { // one that already streamed leaves a trimmed claim whose receiver this // verified context stands in for. state.rsc_receiver_trimmed = - !receiver_context_is_flight_push(&combined[..identifier_start]); + !receiver_context_is_flight_push(&combined_code[..identifier_start]); state.rsc_receiver_context.clear(); let claimed_start = if state.rsc_receiver_trimmed { identifier_start @@ -349,6 +355,8 @@ impl NextJsRscPlaceholderRewriter { }; let prefix = &combined[..claimed_start]; let claimed = &combined[claimed_start..]; + state.rsc_lexical_start = lexical_start; + state.rsc_lexical_start.mask(&combined[..claimed_start]); let action = self.rewrite_claimed_fragment( claimed, ctx.is_last_in_text_node, @@ -389,13 +397,23 @@ impl IntegrationScriptRewriter for NextJsRscPlaceholderRewriter { state.current_fragment_protected = false; state.flight_node_owned = false; state.flight_qualifier_tail.clear(); + state.flight_lexical = ScriptLexer::default(); return ScriptRewriteAction::Keep; } if !self.config.enabled || self.config.rewrite_attributes.is_empty() { state.current_fragment_protected = false; return ScriptRewriteAction::Keep; } - observe_flight_qualifier(&mut state, content); + let lexical_start = state.flight_lexical.clone(); + let code = state.flight_lexical.mask(content); + if state.flight_lexical.is_opaque() { + // At the lexical nesting limit, retain the existing conservative raw + // qualification. Ordinary scripts without a push do not bypass Flight. + observe_flight_qualifier(&mut state, content); + state.bypass_rsc |= state.flight_node_owned; + } else { + observe_flight_qualifier(&mut state, &code); + } // A call head can complete after a tentative property probe overflowed. // Once it proves Flight, preserve its entire raw continuation/group. if state.flight_node_owned @@ -407,18 +425,35 @@ impl IntegrationScriptRewriter for NextJsRscPlaceholderRewriter { state.bypass_rsc = true; } state.current_fragment_protected = state.flight_node_owned; - let action = self.rewrite_fragment(content, ctx, &mut state); + let action = self.rewrite_fragment(content, &code, lexical_start, ctx, &mut state); if ctx.is_last_in_text_node { state.flight_node_owned = false; state.flight_qualifier_tail.clear(); state.rsc_probe.clear(); state.rsc_receiver_context.clear(); state.rsc_receiver_trimmed = false; + state.flight_lexical = ScriptLexer::default(); + state.rsc_lexical_start = ScriptLexer::default(); } action } } +/// Match a call head only in executable source, then read its original string. +fn executable_payload_range(content: &str, code: &str, trimmed: bool) -> Option<(usize, usize)> { + if trimmed { + RSC_PUSH_CALL_PATTERN_TRIMMED.find(code)?; + return find_trimmed_rsc_push_payload_range(content); + } + let call = RSC_PUSH_CALL_PATTERN.find(code)?; + let identifier = call.start() + call.as_str().find("__next_f")?; + if !receiver_context_is_flight_push(&code[..identifier]) { + return None; + } + let (start, end) = find_rsc_push_payload_range(&content[call.start()..])?; + Some((call.start() + start, call.start() + end)) +} + fn contains_qualified_flight(source: &str) -> bool { first_qualified_flight_push(source).is_some() } @@ -522,6 +557,16 @@ mod tests { }) } + #[test] + fn quoted_flight_spellings_do_not_claim_ownership() { + let state = IntegrationDocumentState::default(); + let rewriter = NextJsRscPlaceholderRewriter::new(test_config()); + rewriter.rewrite(r#"const demo="self.__next_f.push(";"#, &ctx(false, &state)); + let shared = document_state(&state); + let state = shared.lock().expect("should lock state"); + assert!(!state.flight_node_owned, "quoted text must not own Flight"); + } + #[test] fn unrecognized_qualified_prefix_makes_the_whole_batch_raw_and_protected() { let state = IntegrationDocumentState::default(); diff --git a/crates/trusted-server-core/src/integrations/nextjs/rsc_stream.rs b/crates/trusted-server-core/src/integrations/nextjs/rsc_stream.rs index aa3262b373..22812c707e 100644 --- a/crates/trusted-server-core/src/integrations/nextjs/rsc_stream.rs +++ b/crates/trusted-server-core/src/integrations/nextjs/rsc_stream.rs @@ -11,6 +11,7 @@ use super::rsc::{ DEFAULT_MAX_COMBINED_PAYLOAD_BYTES, PendingTChunk, TChunkStep, next_tchunk, rewrite_rsc_scripts_combined_with_transform, }; +use super::script_lexer::ScriptLexer; use super::shared::RscUrlRewriter; use super::{NEXTJS_INTEGRATION_ID, NextJsIntegrationConfig}; @@ -58,6 +59,10 @@ pub(super) struct NextJsDocumentState { pub(super) current_fragment_protected: bool, /// Bounded qualifier observation remains active even after document bypass. pub(super) flight_qualifier_tail: String, + /// Lexical context advances with original parser fragments, never replacements. + pub(super) flight_lexical: ScriptLexer, + /// Snapshot at the beginning of a claimed buffer for source-offset matching. + pub(super) rsc_lexical_start: ScriptLexer, } impl Default for NextJsDocumentState { @@ -77,6 +82,8 @@ impl Default for NextJsDocumentState { flight_node_owned: false, current_fragment_protected: false, flight_qualifier_tail: String::new(), + flight_lexical: ScriptLexer::default(), + rsc_lexical_start: ScriptLexer::default(), } } } diff --git a/crates/trusted-server-core/src/integrations/nextjs/script_lexer.rs b/crates/trusted-server-core/src/integrations/nextjs/script_lexer.rs new file mode 100644 index 0000000000..7061448682 --- /dev/null +++ b/crates/trusted-server-core/src/integrations/nextjs/script_lexer.rs @@ -0,0 +1,318 @@ +//! Bounded lexical context for recognizing executable Flight call heads. +//! +//! This is not a JavaScript parser. It masks strings, comments and template text +//! without changing byte offsets, and exposes code inside template expressions. + +const MAX_TEMPLATE_DEPTH: usize = 32; + +#[derive(Clone, Debug, Default)] +enum Mode { + #[default] + Code, + Slash { + regex: bool, + }, + Regex { + escaped: bool, + class: bool, + }, + String { + quote: char, + escaped: bool, + }, + LineComment, + BlockComment { + star: bool, + }, + Template { + escaped: bool, + dollar: bool, + }, + Opaque, +} + +#[derive(Clone, Debug)] +pub(super) struct ScriptLexer { + mode: Mode, + template_braces: Vec, + expression_start: bool, + word: String, + word_overflow: bool, + word_property: bool, + after_dot: bool, + control_pending: bool, + parentheses: Vec, + operator: Option<(char, bool)>, +} + +impl Default for ScriptLexer { + fn default() -> Self { + Self { + mode: Mode::Code, + template_braces: Vec::new(), + expression_start: true, + word: String::new(), + word_overflow: false, + word_property: false, + after_dot: false, + control_pending: false, + parentheses: Vec::new(), + operator: None, + } + } +} + +impl ScriptLexer { + /// Mask non-code bytes while preserving source offsets and opening quotes. + /// Opening quotes remain visible so the Flight payload regex can match them. + pub(super) fn mask(&mut self, source: &str) -> String { + let mut code = String::with_capacity(source.len()); + for character in source.chars() { + if self.is_code(character) { + code.push(character); + } else { + // A non-whitespace sentinel prevents matching across inert text. + code.extend(std::iter::repeat_n('#', character.len_utf8())); + } + } + code + } + + /// Excessively nested templates are unsupported rather than unbounded. + pub(super) fn is_opaque(&self) -> bool { + matches!(self.mode, Mode::Opaque) + } + + fn is_code(&mut self, character: char) -> bool { + match &mut self.mode { + Mode::Code => { + if character.is_alphanumeric() || matches!(character, '_' | '$') { + if self.word.is_empty() { + self.word_property = self.after_dot; + self.after_dot = false; + } + self.operator = None; + if self.word.len() < 10 && !self.word_overflow { + self.word.push(character); + } else { + self.word_overflow = true; + } + self.expression_start = false; + return true; + } + if !self.word.is_empty() { + self.expression_start = !self.word_overflow + && !self.word_property + && matches!( + self.word.as_str(), + "return" + | "throw" + | "case" + | "delete" + | "void" + | "typeof" + | "new" + | "yield" + | "await" + | "else" + | "do" + ); + self.control_pending = !self.word_overflow + && !self.word_property + && matches!( + self.word.as_str(), + "if" | "while" | "for" | "with" | "switch" | "catch" + ); + self.word.clear(); + self.word_overflow = false; + } + let prior_operator = self.operator.take(); + if !character.is_whitespace() && character != '/' { + self.after_dot = character == '.'; + } + match character { + '(' => { + if self.parentheses.len() == MAX_TEMPLATE_DEPTH { + self.mode = Mode::Opaque; + } else { + self.parentheses.push(self.control_pending); + } + self.control_pending = false; + self.expression_start = true; + } + ')' => self.expression_start = self.parentheses.pop().unwrap_or(false), + '+' | '-' => { + if let Some((operator, prior_expression)) = prior_operator + && operator == character + { + self.expression_start = prior_expression; + } else { + self.operator = Some((character, self.expression_start)); + self.expression_start = true; + } + } + '\'' | '"' => { + self.mode = Mode::String { + quote: character, + escaped: false, + }; + } + '/' => { + self.mode = Mode::Slash { + regex: self.expression_start, + } + } + '`' => { + if self.template_braces.len() == MAX_TEMPLATE_DEPTH { + self.mode = Mode::Opaque; + } else { + self.template_braces.push(0); + self.mode = Mode::Template { + escaped: false, + dollar: false, + }; + } + } + '{' => { + self.expression_start = true; + if let Some(braces) = self.template_braces.last_mut() { + *braces = braces.saturating_add(1); + } + } + '}' => { + if let Some(braces) = self.template_braces.last_mut() { + if *braces == 0 { + self.mode = Mode::Template { + escaped: false, + dollar: false, + }; + } else { + *braces -= 1; + } + } + } + _ => { + if !character.is_whitespace() { + self.expression_start = !matches!(character, ')' | ']' | '.'); + } + } + } + true + } + Mode::Slash { regex } => { + match character { + '/' => self.mode = Mode::LineComment, + '*' => self.mode = Mode::BlockComment { star: false }, + _ if *regex => { + self.mode = Mode::Regex { + escaped: false, + class: false, + }; + return self.is_code(character); + } + _ => { + self.mode = Mode::Code; + self.expression_start = true; + self.after_dot = false; + return self.is_code(character); + } + } + false + } + Mode::Regex { escaped, class } => { + if *escaped { + *escaped = false; + } else if character == '\\' { + *escaped = true; + } else if character == '[' { + *class = true; + } else if character == ']' { + *class = false; + } else if character == '/' && !*class { + self.mode = Mode::Code; + self.expression_start = false; + } + false + } + Mode::String { quote, escaped } => { + if *escaped { + *escaped = false; + } else if character == '\\' { + *escaped = true; + } else if character == *quote { + self.mode = Mode::Code; + self.expression_start = false; + } + false + } + Mode::LineComment => { + if matches!(character, '\n' | '\r' | '\u{2028}' | '\u{2029}') { + self.mode = Mode::Code; + } + false + } + Mode::BlockComment { star } => { + if *star && character == '/' { + self.mode = Mode::Code; + } else { + *star = character == '*'; + } + false + } + Mode::Template { escaped, dollar } => { + if *escaped { + *escaped = false; + *dollar = false; + } else if character == '\\' { + *escaped = true; + *dollar = false; + } else if character == '`' { + self.template_braces.pop(); + self.mode = Mode::Code; + self.expression_start = false; + } else if *dollar && character == '{' { + self.mode = Mode::Code; + self.expression_start = true; + } else { + *dollar = character == '$'; + } + false + } + Mode::Opaque => false, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn masks_inert_text_at_every_fragment_boundary() { + let source = r#"'é self.__next_f.push('/* self.__next_f.push( */`😀 ${ {nested: `self.__next_f.push(`} } ${self.__next_f.push([1,"data"])} `"#; + let expected = ScriptLexer::default().mask(source); + assert_eq!(expected.len(), source.len(), "should preserve byte offsets"); + assert_eq!(expected.matches("self.__next_f.push(").count(), 1); + for split in (0..=source.len()).filter(|split| source.is_char_boundary(*split)) { + let mut lexer = ScriptLexer::default(); + let actual = format!( + "{}{}", + lexer.mask(&source[..split]), + lexer.mask(&source[split..]) + ); + assert_eq!( + actual, expected, + "should preserve lexical state at split {split}" + ); + } + } + + #[test] + fn bounds_nested_templates() { + let mut lexer = ScriptLexer::default(); + lexer.mask(&"`${".repeat(MAX_TEMPLATE_DEPTH + 1)); + assert!(lexer.is_opaque(), "should refuse excessive nesting"); + assert_eq!(lexer.template_braces.len(), MAX_TEMPLATE_DEPTH); + assert!(!lexer.mask("self.__next_f.push(").contains("__next_f")); + } +} From d738d93f70e3a1e79bfa9256cc730a317d14381a Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 1 Oct 2026 09:01:30 -0500 Subject: [PATCH 05/15] Distinguish JavaScript blocks from expression-valued braces Keep regex literals after statement blocks inert without hiding Flight calls after object, function, or class expressions. Bound brace and pending-class context alongside existing lexical nesting limits. Add fragment-boundary and identity/gzip regressions for origin/GTM rewrites and decoded Flight T lengths. Both reported brace/slash regressions were reproduced before the fix. --- .../src/integrations/nextjs/mod.rs | 67 ++++++ .../src/integrations/nextjs/script_lexer.rs | 214 +++++++++++++++++- 2 files changed, 270 insertions(+), 11 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/nextjs/mod.rs b/crates/trusted-server-core/src/integrations/nextjs/mod.rs index b37b3a8f23..4cf78475fd 100644 --- a/crates/trusted-server-core/src/integrations/nextjs/mod.rs +++ b/crates/trusted-server-core/src/integrations/nextjs/mod.rs @@ -264,6 +264,9 @@ mod tests { "// self.__next_f.push(\n", "/* self.__next_f.push( */", "const demo=/self.__next_f.push()/;", + "if (ready) { work() } /self.__next_f.push()/;", + "function demo() { return 1 } /self.__next_f.push()/;", + "if (ready) { if (nested) { work() } } /self.__next_f.push()/;", r#"if (ready) {} else /self.__next_f.push([2,"opaque"])/;"#, r#"do /self.__next_f.push([2,"opaque"])/; while (ready);"#, "const demo=`self.__next_f.push(`;", @@ -312,6 +315,10 @@ mod tests { format!("{fake}{push};/* self.__next_f.push( */"), format!("{fake}{rewritten_push};/* self.__next_f.push( */"), ), + ( + format!("if (ready) {{ work() }} /self.__next_f.push()/;{push};"), + format!("if (ready) {{ work() }} /self.__next_f.push()/;{rewritten_push};"), + ), ( format!( "const out=`text self.__next_f.push( ${{ {{a: {push}, b: `nested ${{{push}}}`}} }} tail`;" @@ -350,6 +357,66 @@ mod tests { } } + #[test] + fn brace_slash_division_keeps_flight_lengths_and_both_rewrites() { + let data = json!({"url": "https://origin.example.com/page", "tag": "http://www.googletagmanager.com/gtm.js", "text": "é😀"}); + let body = data.to_string(); + let payload = format!("1:T{:x},{body}", body.len()); + let push = format!( + "self.__next_f.push([1,{}])", + serde_json::to_string(&payload).expect("should encode Flight payload") + ); + let mut expected = data; + expected["url"] = json!("https://test.example.com/page"); + expected["tag"] = json!("/integrations/google_tag_manager/gtm.js"); + for prefix in [ + "const n = {} / 2;", + "const n = {nested: {}} / 2;", + "const n = function() {} / 2;", + "const n = function named() { return 1 } / 2;", + "const n = class { method() { return 1 } } / 2;", + ] { + let source = format!("{prefix}{push}"); + for split in (0..=source.len()).filter(|split| source.is_char_boundary(*split)) { + let mut processor = create_html_processor(mixed_config(10000, 10000)); + let first = format!("", &source[split..]); + let mut bytes = processor + .process_chunk(first.as_bytes(), false) + .expect("should accept division prefix"); + bytes.extend( + processor + .process_chunk(last.as_bytes(), true) + .expect("should complete Flight after division"), + ); + let output = String::from_utf8(bytes).expect("should emit UTF-8"); + assert!(output.starts_with(&format!(""); + for compression in [Compression::None, Compression::Gzip] { + for chunk_size in [32, 1000, 8192] { + let output = mixed_output_with_compression(&html, compression, chunk_size); + let payloads = flight_payloads(&output); + assert_eq!( + payloads.len(), + 2, + "should preserve compressed Flight pushes" + ); + for payload in payloads { + assert_mixed_t_model(&payload, &expected); + } + assert!(!output.contains("__ts_rsc_")); + } + } + } + } + #[test] fn mixed_next_data_flight_mentions_do_not_claim_json_or_hide_gtm() { for text in [ diff --git a/crates/trusted-server-core/src/integrations/nextjs/script_lexer.rs b/crates/trusted-server-core/src/integrations/nextjs/script_lexer.rs index 7061448682..df1aa278a6 100644 --- a/crates/trusted-server-core/src/integrations/nextjs/script_lexer.rs +++ b/crates/trusted-server-core/src/integrations/nextjs/script_lexer.rs @@ -31,17 +31,44 @@ enum Mode { Opaque, } +#[derive(Clone, Copy, Debug)] +enum BraceKind { + StatementBlock, + ExpressionBlock, + Object, +} + +#[derive(Clone, Copy, Debug)] +enum ParenthesisKind { + Control, + Expression, + Function(BraceKind), +} + +#[derive(Clone, Copy, Debug)] +struct PendingClass { + body: BraceKind, + parenthesis_depth: usize, + brace_depth: usize, +} + #[derive(Clone, Debug)] pub(super) struct ScriptLexer { mode: Mode, template_braces: Vec, expression_start: bool, + statement_start: bool, + braces: Vec, word: String, + word_statement_start: bool, word_overflow: bool, word_property: bool, after_dot: bool, control_pending: bool, - parentheses: Vec, + parentheses: Vec, + function_pending: Option, + body_pending: Option, + classes: Vec, operator: Option<(char, bool)>, } @@ -51,12 +78,18 @@ impl Default for ScriptLexer { mode: Mode::Code, template_braces: Vec::new(), expression_start: true, + statement_start: true, + braces: Vec::new(), word: String::new(), + word_statement_start: true, word_overflow: false, word_property: false, after_dot: false, control_pending: false, parentheses: Vec::new(), + function_pending: None, + body_pending: None, + classes: Vec::new(), operator: None, } } @@ -78,7 +111,7 @@ impl ScriptLexer { code } - /// Excessively nested templates are unsupported rather than unbounded. + /// Excessive lexical nesting is unsupported rather than unbounded. pub(super) fn is_opaque(&self) -> bool { matches!(self.mode, Mode::Opaque) } @@ -89,7 +122,9 @@ impl ScriptLexer { if character.is_alphanumeric() || matches!(character, '_' | '$') { if self.word.is_empty() { self.word_property = self.after_dot; + self.word_statement_start = self.statement_start; self.after_dot = false; + self.body_pending = None; } self.operator = None; if self.word.len() < 10 && !self.word_overflow { @@ -98,6 +133,7 @@ impl ScriptLexer { self.word_overflow = true; } self.expression_start = false; + self.statement_start = false; return true; } if !self.word.is_empty() { @@ -123,24 +159,80 @@ impl ScriptLexer { self.word.as_str(), "if" | "while" | "for" | "with" | "switch" | "catch" ); + if !self.word_overflow && !self.word_property { + if matches!(self.word.as_str(), "function" | "class") { + let body = if self.word_statement_start { + BraceKind::StatementBlock + } else { + BraceKind::ExpressionBlock + }; + if self.word == "function" { + self.function_pending = Some(body); + } else if self.classes.len() == MAX_TEMPLATE_DEPTH { + self.mode = Mode::Opaque; + return false; + } else { + self.classes.push(PendingClass { + body, + parenthesis_depth: self.parentheses.len(), + brace_depth: self.braces.len(), + }); + } + } + self.statement_start = matches!(self.word.as_str(), "else" | "do") + || (self.word_statement_start + && matches!(self.word.as_str(), "async" | "export" | "default")); + } self.word.clear(); self.word_overflow = false; } let prior_operator = self.operator.take(); + let mut pending_body = None; if !character.is_whitespace() && character != '/' { self.after_dot = character == '.'; + pending_body = self.body_pending.take(); + if !matches!(character, '(' | '*') { + self.function_pending = None; + } } match character { '(' => { if self.parentheses.len() == MAX_TEMPLATE_DEPTH { self.mode = Mode::Opaque; } else { - self.parentheses.push(self.control_pending); + let kind = if self.control_pending { + ParenthesisKind::Control + } else if let Some(kind) = self.function_pending.take() { + ParenthesisKind::Function(kind) + } else { + ParenthesisKind::Expression + }; + self.parentheses.push(kind); } self.control_pending = false; self.expression_start = true; + self.statement_start = false; + } + ')' => { + let kind = self.parentheses.pop(); + self.expression_start = matches!(kind, Some(ParenthesisKind::Control)); + self.statement_start = self.expression_start; + if let Some(ParenthesisKind::Function(kind)) = kind { + self.body_pending = Some(kind); + } + } + '=' => { + self.operator = Some((character, self.expression_start)); + self.expression_start = true; + self.statement_start = false; + } + '>' => { + if prior_operator.is_some_and(|(operator, _)| operator == '=') { + self.body_pending = Some(BraceKind::ExpressionBlock); + } + self.expression_start = true; + self.statement_start = false; } - ')' => self.expression_start = self.parentheses.pop().unwrap_or(false), '+' | '-' => { if let Some((operator, prior_expression)) = prior_operator && operator == character @@ -150,8 +242,10 @@ impl ScriptLexer { self.operator = Some((character, self.expression_start)); self.expression_start = true; } + self.statement_start = false; } '\'' | '"' => { + self.statement_start = false; self.mode = Mode::String { quote: character, escaped: false, @@ -163,6 +257,7 @@ impl ScriptLexer { } } '`' => { + self.statement_start = false; if self.template_braces.len() == MAX_TEMPLATE_DEPTH { self.mode = Mode::Opaque; } else { @@ -174,26 +269,64 @@ impl ScriptLexer { } } '{' => { + // Blocks admit a new statement after closing; objects and + // function expressions finish a value and admit division. + let class_body = self.classes.last().is_some_and(|class| { + class.parenthesis_depth == self.parentheses.len() + && class.brace_depth == self.braces.len() + }); + let kind = if class_body { + self.classes.pop().expect("should have pending class").body + } else { + pending_body.unwrap_or( + if self.statement_start || !self.expression_start { + BraceKind::StatementBlock + } else { + BraceKind::Object + }, + ) + }; + if self.braces.len() == MAX_TEMPLATE_DEPTH { + self.mode = Mode::Opaque; + } else { + self.braces.push(kind); + } self.expression_start = true; + self.statement_start = !matches!(kind, BraceKind::Object); if let Some(braces) = self.template_braces.last_mut() { *braces = braces.saturating_add(1); } } '}' => { - if let Some(braces) = self.template_braces.last_mut() { - if *braces == 0 { - self.mode = Mode::Template { - escaped: false, - dollar: false, - }; - } else { + if self.template_braces.last() == Some(&0) { + self.mode = Mode::Template { + escaped: false, + dollar: false, + }; + } else { + let kind = self.braces.pop(); + self.expression_start = matches!(kind, Some(BraceKind::StatementBlock)); + self.statement_start = self.expression_start; + if let Some(braces) = self.template_braces.last_mut() { *braces -= 1; } } } + ':' => { + // A reserved word used as an object key is not a class head. + if self.classes.last().is_some_and(|class| { + class.parenthesis_depth == self.parentheses.len() + && class.brace_depth == self.braces.len() + }) { + self.classes.pop(); + } + self.expression_start = true; + self.statement_start = false; + } _ => { if !character.is_whitespace() { self.expression_start = !matches!(character, ')' | ']' | '.'); + self.statement_start = character == ';'; } } } @@ -213,6 +346,7 @@ impl ScriptLexer { _ => { self.mode = Mode::Code; self.expression_start = true; + self.statement_start = false; self.after_dot = false; return self.is_code(character); } @@ -273,6 +407,7 @@ impl ScriptLexer { } else if *dollar && character == '{' { self.mode = Mode::Code; self.expression_start = true; + self.statement_start = false; } else { *dollar = character == '$'; } @@ -307,6 +442,63 @@ mod tests { } } + #[test] + fn brace_context_survives_every_fragment_boundary() { + for prefix in [ + "{} /self.__next_f.push()/;", + "if (ready) { work() } /self.__next_f.push()/;", + "if (ready) { if (nested) { work() } } /self.__next_f.push()/;", + "function demo() { return 1 } /self.__next_f.push()/;", + "async function demo() {} /self.__next_f.push()/;", + "const n = {} / 2;", + "const n = {nested: {}} / 2;", + "const n = function() {} / 2;", + "const n = function named() { return {} } / 2;", + "const n = (() => {}) / 2;", + "class Demo { method() { return 1 } } /self.__next_f.push()/;", + "const n = class { method() { return 1 } } / 2;", + "const n = class extends mixin({}) {} / 2;", + "const n = class extends (class {}) {} / 2;", + "const n = {class: {}, method() { return 1 }} / 2;", + "const n = {method() { if (ready) { work() } /self.__next_f.push()/ }} / 2;", + "const n = `value ${ {} / 2 }`;", + ] { + let source = format!("{prefix}self.__next_f.push([1,\"data\"])"); + for split in 0..=source.len() { + let mut lexer = ScriptLexer::default(); + let code = format!( + "{}{}", + lexer.mask(&source[..split]), + lexer.mask(&source[split..]) + ); + assert_eq!(code.len(), source.len(), "should preserve source offsets"); + assert_eq!( + code.matches("self.__next_f.push(").count(), + 1, + "should expose only the executable push: prefix={prefix}, split={split}" + ); + } + } + } + + #[test] + fn bounds_nested_braces() { + let mut lexer = ScriptLexer::default(); + lexer.mask(&"{".repeat(MAX_TEMPLATE_DEPTH + 1)); + assert!(lexer.is_opaque(), "should refuse excessive brace nesting"); + assert_eq!(lexer.braces.len(), MAX_TEMPLATE_DEPTH); + assert!(!lexer.mask("self.__next_f.push(").contains("__next_f")); + } + + #[test] + fn bounds_pending_class_heads() { + let mut lexer = ScriptLexer::default(); + lexer.mask(&"class extends (".repeat(MAX_TEMPLATE_DEPTH + 1)); + assert!(lexer.is_opaque(), "should refuse excessive class nesting"); + assert_eq!(lexer.classes.len(), MAX_TEMPLATE_DEPTH); + assert!(!lexer.mask("self.__next_f.push(").contains("__next_f")); + } + #[test] fn bounds_nested_templates() { let mut lexer = ScriptLexer::default(); From 3f7c3a83ac9caebc784d6ffc63019b7534d62030 Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 1 Oct 2026 10:57:10 -0500 Subject: [PATCH 06/15] Validate request hosts before source rewriting --- .../tests/routes.rs | 101 ++++++ crates/trusted-server-core/src/http_util.rs | 317 +++++++++++++++--- .../src/integrations/prebid.rs | 1 + crates/trusted-server-core/src/proxy.rs | 13 +- crates/trusted-server-core/src/publisher.rs | 15 +- 5 files changed, 396 insertions(+), 51 deletions(-) diff --git a/crates/trusted-server-adapter-axum/tests/routes.rs b/crates/trusted-server-adapter-axum/tests/routes.rs index 42d1b1b6b5..b9c6379d5e 100644 --- a/crates/trusted-server-adapter-axum/tests/routes.rs +++ b/crates/trusted-server-adapter-axum/tests/routes.rs @@ -941,6 +941,107 @@ async fn nextjs_auction_output_holds_until_the_structural_body_close() { ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn nextjs_gtm_rejects_malformed_request_hosts() { + for header in ["host", "x-forwarded-host", "forwarded"] { + let origin = Arc::new(nextjs_auction::NextJsAuctionOrigin::with_html_response( + &nextjs_auction::script_composition_html(), + false, + true, + )); + let router = TrustedServerApp::routes_with_settings_and_services( + nextjs_auction::script_composition_settings(), + nextjs_auction::services(origin), + ) + .expect("should build script composition router"); + let mut service = EdgeZeroAxumService::new(router); + let hostile_host = "example.com"; + let value = if header == "forwarded" { + format!("host=\"{hostile_host}\"") + } else { + hostile_host.to_owned() + }; + let mut request = Request::builder() + .uri("/script-composition") + .header("host", nextjs_auction::PUBLISHER_HOST) + .body(AxumBody::empty()) + .expect("should build hostile host request"); + request.headers_mut().insert( + header, + value.parse().expect("should encode hostile host header"), + ); + let response = service + .ready() + .await + .expect("should be ready") + .call(request) + .await + .expect("should respond to hostile host request"); + assert_eq!( + response.status().as_u16(), + 400, + "should reject malformed {header} rather than rewrite script source" + ); + let body = axum::body::to_bytes(response.into_body(), 1024) + .await + .expect("should consume host rejection response"); + assert!( + !String::from_utf8_lossy(&body).contains(hostile_host), + "should not reflect the rejected host in the response" + ); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn nextjs_gtm_accepts_local_request_hosts_and_ports() { + for host in ["localhost:3000", "example.com:8443", "[::1]:3000"] { + let origin = Arc::new(nextjs_auction::NextJsAuctionOrigin::with_html_response( + &nextjs_auction::script_composition_html(), + false, + true, + )); + let router = TrustedServerApp::routes_with_settings_and_services( + nextjs_auction::script_composition_settings(), + nextjs_auction::services(origin), + ) + .expect("should build script composition router"); + let mut service = EdgeZeroAxumService::new(router); + let request = Request::builder() + .uri("/script-composition") + .header("host", host) + .body(AxumBody::empty()) + .expect("should build local host request"); + let response = service + .ready() + .await + .expect("should be ready") + .call(request) + .await + .expect("should serve local host request"); + assert_eq!( + response.status().as_u16(), + 200, + "should accept host: {host}" + ); + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .expect("should consume local host response"); + let html = String::from_utf8(body.to_vec()).expect("should emit UTF-8 HTML"); + assert!( + html.contains(&format!("http://{host}/app")), + "should preserve host and port in rewritten Flight URL: {host}" + ); + assert!( + !html.contains("__ts_rsc_"), + "should resolve every local host payload" + ); + assert!( + html.ends_with(""), + "should complete local response" + ); + } +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn nextjs_gtm_html_response_is_complete() { for gzip in [false, true] { diff --git a/crates/trusted-server-core/src/http_util.rs b/crates/trusted-server-core/src/http_util.rs index 457e503568..7e39973781 100644 --- a/crates/trusted-server-core/src/http_util.rs +++ b/crates/trusted-server-core/src/http_util.rs @@ -4,6 +4,7 @@ use edgezero_core::body::Body as EdgeBody; use error_stack::Report; use http::{Request, Response, StatusCode, header}; use sha2::{Digest as _, Sha256}; +use std::net::Ipv6Addr; use std::time::Duration; use subtle::ConstantTimeEq as _; @@ -134,7 +135,7 @@ pub struct RequestInfo { } impl RequestInfo { - /// Extract request info from a Fastly request. + /// Extract the validated effective request host and scheme. /// /// Host fallback order (first present wins): /// 1. `Forwarded` header (`host=...`) @@ -148,39 +149,108 @@ impl RequestInfo { /// 4. `Fastly-SSL` /// 5. Default `http` /// - /// In production the forwarded headers are stripped by - /// [`sanitize_forwarded_headers`] at the edge, so `Host` and - /// [`ClientInfo`] TLS detection are the only sources that fire. - pub fn from_request(req: &Request, client_info: &ClientInfo) -> Self { - let host = extract_request_host(req); + /// Fastly and Spin strip forwarded headers with [`sanitize_forwarded_headers`] + /// before routing. On those paths, `Host` and [`ClientInfo`] TLS detection + /// are the effective sources. Other adapters retain their existing policy. + /// + /// The selected host must be a DNS name, IPv4 address, or bracketed IPv6 + /// address, optionally followed by a numeric port. Forwarded-header + /// precedence is unchanged. Requests without any host retain an empty host. + /// + /// # Errors + /// + /// Returns [`TrustedServerError::BadRequest`] when the selected host is + /// malformed, rather than falling back to a lower-priority header. + pub fn from_request( + req: &Request, + client_info: &ClientInfo, + ) -> Result> { + let host = extract_request_host(req)?; let scheme = detect_request_scheme( req, client_info.tls_protocol.as_deref(), client_info.tls_cipher.as_deref(), ); - Self { host, scheme } + Ok(Self { host, scheme }) } } -fn extract_request_host(req: &Request) -> String { - req.headers() - .get("forwarded") - .and_then(|h| h.to_str().ok()) - .and_then(|value| parse_forwarded_param(value, "host")) - .or_else(|| { - req.headers() - .get("x-forwarded-host") - .and_then(|h| h.to_str().ok()) - .and_then(parse_list_header_value) - }) - .or_else(|| { - req.headers() - .get(header::HOST) - .and_then(|h| h.to_str().ok()) +/// Select the effective host without allowing malformed input into rewrite data. +/// +/// # Errors +/// +/// Returns a bad-request error for a malformed selected host or a non-text host header. +fn extract_request_host(req: &Request) -> Result> { + let invalid_host = || { + Report::new(TrustedServerError::BadRequest { + message: "Invalid request host".to_owned(), }) - .unwrap_or_default() - .to_owned() + }; + let forwarded_host = req + .headers() + .get("forwarded") + .map(|value| value.to_str().map_err(|_| invalid_host())) + .transpose()? + .and_then(|value| parse_forwarded_param(value, "host")); + let host = if let Some(host) = forwarded_host { + host + } else if let Some(value) = req.headers().get("x-forwarded-host") { + parse_list_header_value(value.to_str().map_err(|_| invalid_host())?) + .ok_or_else(invalid_host)? + } else if let Some(value) = req.headers().get(header::HOST) { + value.to_str().map_err(|_| invalid_host())? + } else { + return Ok(String::new()); + }; + if !is_valid_request_host(host) { + return Err(invalid_host()); + } + Ok(host.to_owned()) +} + +/// Accept only literal host syntax that is safe to insert into script source. +fn is_valid_request_host(value: &str) -> bool { + let port = if let Some(ipv6) = value.strip_prefix('[') { + let Some((address, suffix)) = ipv6.split_once(']') else { + return false; + }; + if address.parse::().is_err() { + return false; + } + if suffix.is_empty() { + None + } else if let Some(port) = suffix.strip_prefix(':') { + Some(port) + } else { + return false; + } + } else { + let (host, port) = value + .split_once(':') + .map_or((value, None), |(host, port)| (host, Some(port))); + let host = host.strip_suffix('.').unwrap_or(host); + if host.is_empty() + || host.len() > 253 + || host.split('.').any(|label| { + label.is_empty() + || label.len() > 63 + || !label.starts_with(|c: char| c.is_ascii_alphanumeric()) + || !label.ends_with(|c: char| c.is_ascii_alphanumeric()) + || !label + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-') + }) + { + return false; + } + port + }; + port.is_none_or(|port| { + !port.is_empty() + && port.bytes().all(|byte| byte.is_ascii_digit()) + && port.parse::().is_ok() + }) } fn parse_forwarded_param<'a>(forwarded: &'a str, param: &str) -> Option<&'a str> { @@ -189,12 +259,11 @@ fn parse_forwarded_param<'a>(forwarded: &'a str, param: &str) -> Option<&'a str> let mut iter = part.splitn(2, '='); let key = iter.next().unwrap_or("").trim(); let value = iter.next().unwrap_or("").trim(); - if key.is_empty() || value.is_empty() { - continue; - } if key.eq_ignore_ascii_case(param) { let value = strip_quotes(value); - if !value.is_empty() { + // A present but empty host must fail validation, not select a + // lower-priority host. Keep the existing empty-proto behavior. + if !value.is_empty() || param.eq_ignore_ascii_case("host") { return Some(value); } } @@ -572,12 +641,175 @@ mod tests { // RequestInfo tests + #[test] + fn request_info_accepts_valid_hosts_without_normalizing_them() { + for host in [ + "example.com", + "Sub-Domain.EXAMPLE.com.", + "xn--bcher-kva.example.com", + "example.com:443", + "localhost:3000", + "internal-proxy:8080", + "192.0.2.1:8080", + "[2001:db8::1]", + "[2001:db8::1]:443", + "[::1]:3000", + "[::ffff:192.0.2.1]:65535", + ] { + for header in ["host", "x-forwarded-host", "forwarded"] { + let mut req = build_request(Method::GET, "https://example.com/page"); + set_header(&mut req, "host", "fallback.example.com"); + let value = if header == "forwarded" { + format!("for=192.0.2.2;host=\"{host}\";proto=https") + } else { + host.to_owned() + }; + set_header(&mut req, header, &value); + let info = RequestInfo::from_request(&req, &default_client_info()) + .expect("should accept valid host syntax"); + assert_eq!(info.host, host, "should preserve {header} host: {host}"); + } + } + } + + #[test] + fn request_info_rejects_malformed_selected_hosts_without_fallback() { + for host in [ + "", + "example.com", + "example.com'", + "example.com\"", + "example.com`", + "example.com\\path", + "example.com/path", + "example.com?query", + "example.com#fragment", + "user@example.com", + "https://example.com", + "example.com%22", + "example.com:invalid", + "example.com:", + "example.com:65536", + "example.com:+443", + "example.com:443:80", + "bad host.example.com", + "bad_label.example.com", + "-bad.example.com", + "bad-.example.com", + "example..com", + ".example.com", + ".", + "::1", + "[not-ipv6]", + "[2001:db8::1", + "[2001:db8::1]suffix", + "[2001:db8::1]:", + ] { + for header in ["host", "x-forwarded-host", "forwarded"] { + let mut req = build_request(Method::GET, "https://example.com/page"); + set_header(&mut req, "host", "fallback.example.com"); + let value = if header == "forwarded" { + format!("host=\"{host}\"") + } else { + host.to_owned() + }; + set_header(&mut req, header, &value); + let error = RequestInfo::from_request(&req, &default_client_info()) + .expect_err("should reject malformed selected host"); + assert!( + matches!( + error.current_context(), + TrustedServerError::BadRequest { .. } + ), + "should reject {header} host with a bad-request error: {host}" + ); + assert!( + !format!("{error}").contains(""), + "should not reflect hostile host bytes in the error" + ); + } + } + } + + #[test] + fn request_info_rejects_non_text_selected_host_headers() { + for header in ["host", "x-forwarded-host", "forwarded"] { + let mut req = build_request(Method::GET, "https://example.com/page"); + set_header(&mut req, "host", "fallback.example.com"); + req.headers_mut().insert( + HeaderName::from_static(header), + HeaderValue::from_bytes(b"example.com\xff").expect("should build non-text header"), + ); + assert!( + RequestInfo::from_request(&req, &default_client_info()).is_err(), + "should reject non-text {header} rather than falling back" + ); + } + } + + #[test] + fn request_info_rejects_oversized_dns_names() { + for host in [ + format!("{}.example.com", "a".repeat(64)), + format!( + "{}.{}.{}.{}", + "a".repeat(63), + "b".repeat(63), + "c".repeat(63), + "d".repeat(63) + ), + ] { + let mut req = build_request(Method::GET, "https://example.com/page"); + set_header(&mut req, "host", &host); + assert!( + RequestInfo::from_request(&req, &default_client_info()).is_err(), + "should reject oversized DNS name: {host}" + ); + } + } + + #[test] + fn request_info_validates_only_the_selected_host() { + let mut req = build_request(Method::GET, "https://example.com/page"); + set_header(&mut req, "forwarded", "host=public.example.com"); + set_header(&mut req, "x-forwarded-host", "example.com"); + set_header(&mut req, "host", "example.com'"); + let info = RequestInfo::from_request(&req, &default_client_info()) + .expect("should accept valid highest-priority host"); + assert_eq!( + info.host, "public.example.com", + "should preserve precedence without using lower-priority hosts" + ); + } + + #[test] + fn request_info_preserves_empty_forwarded_proto_behavior() { + let mut req = build_request(Method::GET, "https://example.com/page"); + set_header( + &mut req, + "forwarded", + "host=example.com;proto=\"\";proto=https", + ); + let info = RequestInfo::from_request(&req, &default_client_info()) + .expect("should accept valid host and later nonempty proto"); + assert_eq!(info.scheme, "https", "should retain empty-proto fallback"); + } + + #[test] + fn request_info_preserves_absent_host_behavior() { + let req = build_request(Method::GET, "/page"); + let info = RequestInfo::from_request(&req, &default_client_info()) + .expect("should preserve request metadata without host headers"); + assert!(info.host.is_empty(), "should keep absent host empty"); + } + #[test] fn test_request_info_from_host_header() { let mut req = build_request(Method::GET, "https://test.example.com/page"); set_header(&mut req, "host", "test.example.com"); - let info = RequestInfo::from_request(&req, &default_client_info()); + let info = RequestInfo::from_request(&req, &default_client_info()) + .expect("should accept Host header"); assert_eq!( info.host, "test.example.com", "Host should use Host header when forwarded headers are missing" @@ -599,7 +831,8 @@ mod tests { "public.example.com, proxy.local", ); - let info = RequestInfo::from_request(&req, &default_client_info()); + let info = RequestInfo::from_request(&req, &default_client_info()) + .expect("should accept forwarded host"); assert_eq!( info.host, "public.example.com", "Host should prefer X-Forwarded-Host over Host" @@ -612,7 +845,8 @@ mod tests { set_header(&mut req, "host", "test.example.com"); set_header(&mut req, "x-forwarded-proto", "https, http"); - let info = RequestInfo::from_request(&req, &default_client_info()); + let info = RequestInfo::from_request(&req, &default_client_info()) + .expect("should accept request scheme metadata"); assert_eq!( info.scheme, "https", "Scheme should prefer the first X-Forwarded-Proto value" @@ -623,7 +857,8 @@ mod tests { set_header(&mut req, "host", "test.example.com"); set_header(&mut req, "x-forwarded-proto", "http"); - let info = RequestInfo::from_request(&req, &default_client_info()); + let info = RequestInfo::from_request(&req, &default_client_info()) + .expect("should accept request scheme metadata"); assert_eq!( info.scheme, "http", "Scheme should use the X-Forwarded-Proto value when present" @@ -643,7 +878,8 @@ mod tests { set_header(&mut req, "x-forwarded-host", "proxy.local"); set_header(&mut req, "x-forwarded-proto", "http"); - let info = RequestInfo::from_request(&req, &default_client_info()); + let info = RequestInfo::from_request(&req, &default_client_info()) + .expect("should accept forwarded host with port"); assert_eq!( info.host, "public.example.com:443", "Host should prefer Forwarded host over X-Forwarded-Host" @@ -659,7 +895,8 @@ mod tests { let mut req = build_request(Method::GET, "https://test.example.com/page"); set_header(&mut req, "fastly-ssl", "1"); - let info = RequestInfo::from_request(&req, &default_client_info()); + let info = RequestInfo::from_request(&req, &default_client_info()) + .expect("should accept request without host"); assert_eq!( info.scheme, "https", "Scheme should fall back to Fastly-SSL when other signals are missing" @@ -675,7 +912,8 @@ mod tests { set_header(&mut req, "x-forwarded-host", "public.example.com"); set_header(&mut req, "x-forwarded-proto", "https"); - let info = RequestInfo::from_request(&req, &default_client_info()); + let info = RequestInfo::from_request(&req, &default_client_info()) + .expect("should accept chained proxy host"); assert_eq!( info.host, "public.example.com", "Host should use X-Forwarded-Host in chained proxy scenarios" @@ -769,7 +1007,8 @@ mod tests { set_header(&mut req, "x-forwarded-proto", "http"); sanitize_forwarded_headers(&mut req); - let info = RequestInfo::from_request(&req, &default_client_info()); + let info = RequestInfo::from_request(&req, &default_client_info()) + .expect("should accept sanitized Host header"); assert_eq!( info.host, "legit.example.com", @@ -892,7 +1131,8 @@ mod tests { ..ClientInfo::default() }; - let info = RequestInfo::from_request(&req, &client_info); + let info = RequestInfo::from_request(&req, &client_info) + .expect("should accept TLS protocol metadata"); assert_eq!( info.scheme, "https", @@ -908,7 +1148,8 @@ mod tests { ..ClientInfo::default() }; - let info = RequestInfo::from_request(&req, &client_info); + let info = RequestInfo::from_request(&req, &client_info) + .expect("should accept TLS cipher metadata"); assert_eq!( info.scheme, "https", diff --git a/crates/trusted-server-core/src/integrations/prebid.rs b/crates/trusted-server-core/src/integrations/prebid.rs index a031c2554a..b001b4dcee 100644 --- a/crates/trusted-server-core/src/integrations/prebid.rs +++ b/crates/trusted-server-core/src/integrations/prebid.rs @@ -3964,6 +3964,7 @@ mod tests { fn make_request_info(context: &AuctionContext<'_>) -> RequestInfo { RequestInfo::from_request(context.request, context.services.client_info()) + .expect("should extract valid test request info") } fn config_from_settings( diff --git a/crates/trusted-server-core/src/proxy.rs b/crates/trusted-server-core/src/proxy.rs index 5d659cdbff..b834742a32 100644 --- a/crates/trusted-server-core/src/proxy.rs +++ b/crates/trusted-server-core/src/proxy.rs @@ -1621,7 +1621,8 @@ pub async fn handle_first_party_click( /// # Errors /// /// Returns an error if JSON parsing fails, the URL cannot be parsed, the URL uses an -/// unsupported scheme, the URL lacks a host, or the host violates `proxy.allowed_domains`. +/// unsupported scheme, the URL lacks a host, the host violates `proxy.allowed_domains`, +/// or request metadata needed to determine the scheme contains a malformed host. pub async fn handle_first_party_proxy_sign( settings: &Settings, _services: &RuntimeServices, @@ -1639,11 +1640,11 @@ pub async fn handle_first_party_proxy_sign( // adapter forwards verbatim — have no scheme of their own, so parsing would // hand back the placeholder base's `https` and make an HTTP dev server sign // an HTTPS target it then proxies over TLS against a plaintext service. - let request_scheme = req - .uri() - .scheme_str() - .map(str::to_ascii_lowercase) - .unwrap_or_else(|| RequestInfo::from_request(&req, _services.client_info()).scheme); + let request_scheme = if let Some(scheme) = req.uri().scheme_str() { + scheme.to_ascii_lowercase() + } else { + RequestInfo::from_request(&req, _services.client_info())?.scheme + }; let payload = if method == Method::POST { let body_bytes = request_body_bytes(req.into_body(), "first-party sign")?; diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 6f89edcd0a..40d46d47d1 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -4312,8 +4312,8 @@ pub struct AuctionDispatch<'a> { /// /// # Errors /// -/// Returns a [`TrustedServerError`] if the proxy request fails or the -/// origin backend is unreachable. +/// Returns a [`TrustedServerError`] if the request host is malformed, the proxy +/// request fails, or the origin backend is unreachable. pub async fn handle_publisher_request( settings: &Settings, services: &RuntimeServices, @@ -4333,8 +4333,8 @@ pub async fn handle_publisher_request( // Prebid.js requests are not intercepted here anymore. The HTML processor removes // publisher-supplied Prebid scripts; the unified TSJS bundle includes Prebid.js when enabled. - // Extract request host and scheme (uses Host header and TLS detection after edge sanitization) - let request_info = RequestInfo::from_request(&req, services.client_info()); + // Validate request host before it can enter HTML or Flight rewrite output. + let request_info = RequestInfo::from_request(&req, services.client_info())?; let request_host = &request_info.host; let request_scheme = &request_info.scheme; @@ -6692,7 +6692,8 @@ fn normalize_page_bids_path(raw: &str) -> String { /// /// # Errors /// -/// Returns [`TrustedServerError`] if cookie parsing or EC ID generation fails. +/// Returns [`TrustedServerError`] if the request host is malformed, cookie +/// parsing fails, or EC ID generation fails. pub async fn handle_page_bids( settings: &Settings, services: &RuntimeServices, @@ -6767,7 +6768,7 @@ pub async fn handle_page_bids( // Trusted request origin for absolute inline creative URLs — derived from the // origin the visitor is actually on (scheme, host, port), not the configured // publisher domain, which cannot carry a port and may differ by subdomain. - let request_info = RequestInfo::from_request(&req, services.client_info()); + let request_info = RequestInfo::from_request(&req, services.client_info())?; let page_bids_request_origin = request_origin(&request_info.scheme, &request_info.host); let path_param = req @@ -6799,7 +6800,7 @@ pub async fn handle_page_bids( Vec::new() }; - let request_info = crate::http_util::RequestInfo::from_request(&req, services.client_info()); + let request_info = crate::http_util::RequestInfo::from_request(&req, services.client_info())?; // Owned so the identity-graph snapshot can be stored back on `ec_context` // below without holding a borrow of it across the mutation. let ec_id = ec_context From f2a6a2a28912b8edf83403e8a09986a449e77efb Mon Sep 17 00:00:00 2001 From: Christian Date: Thu, 1 Oct 2026 10:57:10 -0500 Subject: [PATCH 07/15] Address script composition review cleanups --- crates/trusted-server-core/src/html_processor.rs | 6 +++++- .../src/integrations/google_tag_manager.rs | 6 +++++- crates/trusted-server-core/src/integrations/mod.rs | 2 ++ crates/trusted-server-core/src/integrations/nextjs/mod.rs | 3 ++- crates/trusted-server-core/src/integrations/registry.rs | 8 +++++--- 5 files changed, 19 insertions(+), 6 deletions(-) diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 79a2bfaa6f..87c0991a98 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -835,7 +835,11 @@ mod tests { output.contains(""), "should reset CSS match state between scripts" ); - assert_eq!(output.matches("flushed-once").count(), 1); + assert_eq!( + output.matches("flushed-once").count(), + 1, + "should emit the flushed fragment exactly once" + ); } #[test] diff --git a/crates/trusted-server-core/src/integrations/google_tag_manager.rs b/crates/trusted-server-core/src/integrations/google_tag_manager.rs index abba52c61b..cf4f9673b2 100644 --- a/crates/trusted-server-core/src/integrations/google_tag_manager.rs +++ b/crates/trusted-server-core/src/integrations/google_tag_manager.rs @@ -3603,7 +3603,11 @@ container_id = "GTM-DEFAULT" "`/integrations/google_tag_manager/gtm.js${id}`", ), ] { - assert_eq!(rewrite_gtm_rsc_span(source), expected); + assert_eq!( + rewrite_gtm_rsc_span(source), + expected, + "should preserve query delimiters and template interpolation: {source}" + ); } } diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index ee883b0d58..2a48e79d47 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -299,6 +299,8 @@ pub(crate) fn builders() -> &'static [IntegrationBuilder] { id: "testlight", build: testlight::register, }, + // Next.js must precede GTM: GTM reads the Flight-protection snapshot + // that the Next.js script stage records for the same fragment. IntegrationBuilder { id: "nextjs", build: nextjs::register, diff --git a/crates/trusted-server-core/src/integrations/nextjs/mod.rs b/crates/trusted-server-core/src/integrations/nextjs/mod.rs index 4cf78475fd..d68614881a 100644 --- a/crates/trusted-server-core/src/integrations/nextjs/mod.rs +++ b/crates/trusted-server-core/src/integrations/nextjs/mod.rs @@ -111,7 +111,6 @@ pub fn register( .integration_config::( "google_tag_manager", )? - .filter(|config| config.enabled) .map(|_| { crate::integrations::google_tag_manager::rewrite_gtm_rsc_span as fn(&str) -> String }); @@ -120,6 +119,8 @@ pub fn register( gtm_transform, )); + // Pages data must precede Flight capture: it sets the one-shot discriminator + // that keeps JSON strings from being treated as executable Flight source. let builder = IntegrationRegistration::builder(NEXTJS_INTEGRATION_ID) .with_script_rewriter(structured) .with_script_rewriter(placeholders) diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index a41f986f16..e9001afec0 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -540,13 +540,15 @@ pub trait IntegrationAttributeRewriter: Send + Sync { ) -> AttributeRewriteAction; } -/// Trait for integration-provided inline script/text rewrite hooks. +/// Trait for integration-provided inline script rewrite hooks. pub trait IntegrationScriptRewriter: Send + Sync { /// Identifier for logging/diagnostics. fn integration_id(&self) -> &'static str; - /// CSS selector (e.g. `script#__NEXT_DATA__`) that should trigger this rewriter. + /// CSS selector matching `", &source[split..]); + let mut bytes = processor + .process_chunk(first.as_bytes(), false) + .expect("should accept script prefix"); + bytes.extend( + processor + .process_chunk(last.as_bytes(), true) + .expect("should complete Flight with trailing reference"), + ); + let output = String::from_utf8(bytes).expect("should emit UTF-8"); + let payloads = flight_payloads(&output); + assert_eq!(payloads.len(), 2, "should emit both Flight pushes once"); + for payload in payloads { + assert_mixed_t_model(&payload, &expected); + } + assert!( + output.contains(&format!(";console.log({reference});")), + "should retain trailing reference at split {split}" + ); + assert!(!output.contains("__ts_rsc_"), "should resolve placeholders"); + } + let html = format!(""); + for compression in [Compression::None, Compression::Gzip] { + for chunk in [32, 1000, 8192] { + let output = mixed_output_with_compression(&html, compression, chunk); + let payloads = flight_payloads(&output); + assert_eq!(payloads.len(), 2, "should emit both compressed pushes once"); + for payload in payloads { + assert_mixed_t_model(&payload, &expected); + } + } + } + } + } + #[test] fn ordinary_next_f_reference_does_not_hide_gtm() { let script = "var ref=self.__next_f;var tag='http://www.googletagmanager.com/gtm.js';"; @@ -719,6 +774,113 @@ mod tests { ); } + #[test] + fn ordinary_gtm_before_captured_flight_is_chunk_invariant() { + let ordinary = "var tag='http://www.googletagmanager.com/gtm.js';"; + let expected_ordinary = "var tag='/integrations/google_tag_manager/gtm.js';"; + let data = json!({"url": "https://origin.example.com/page", "tag": "http://www.googletagmanager.com/gtm.js?id=GTM-MIX1", "text": "é😀"}); + let body = data.to_string(); + let mut expected = data; + expected["url"] = json!("https://test.example.com/page"); + expected["tag"] = json!("/integrations/google_tag_manager/gtm.js?id=GTM-MIX1"); + for payload in [format!("0:{body}\n"), format!("1:T{:x},{body}", body.len())] { + let push = format!( + "self.__next_f.push([1,{}])", + serde_json::to_string(&payload).expect("should encode Flight payload") + ); + let html = format!(""); + let whole = mixed_output(&html, 8192); + assert!( + whole.starts_with(&format!("", &push[split..]), + ]; + let mut bytes = Vec::new(); + for (index, fragment) in fragments.iter().enumerate() { + bytes.extend( + processor + .process_chunk(fragment.as_bytes(), index == fragments.len() - 1) + .expect("should complete ordinary and captured Flight source"), + ); + } + assert_eq!( + String::from_utf8(bytes).expect("should emit UTF-8"), + whole, + "should preserve both rewrites at Flight split {split}" + ); + } + for compression in [Compression::None, Compression::Gzip] { + for chunk in [128, 1000, 8192] { + assert_eq!( + mixed_output_with_compression(&html, compression, chunk), + whole, + "should preserve both rewrites for {compression:?} at chunk {chunk}" + ); + } + } + } + } + + #[test] + fn ordinary_gtm_before_flight_fallback_preserves_raw_payloads() { + let ordinary = "var tag='http://www.googletagmanager.com/gtm.js';"; + let payload = "1:Tffff,'http://www.googletagmanager.com/gtm.js?id=GTM-MIX1'"; + let encoded = serde_json::to_string(payload).expect("should encode raw Flight payload"); + for (channel, limit, captured) in [(1, 10000, true), (1, 24, false), (2, 10000, false)] { + let push = format!("self.__next_f.push([{channel},{encoded}])"); + let html = format!(""); + let whole = mixed_output_with_budget(&html, 8192, limit, 10000); + let expected_ordinary = if captured { + ordinary.replace( + "http://www.googletagmanager.com", + "/integrations/google_tag_manager", + ) + } else { + ordinary.to_owned() + }; + assert_eq!( + whole, + format!(""), + "should retain original fallback payload bytes" + ); + for split in 0..=push.len() { + let mut processor = create_html_processor(mixed_config(limit, 10000)); + let fragments = [ + format!("".to_owned(), + ]; + let mut bytes = Vec::new(); + for (index, fragment) in fragments.iter().enumerate() { + bytes.extend( + processor + .process_chunk(fragment.as_bytes(), index == fragments.len() - 1) + .expect("should restore raw Flight without changing payload lengths"), + ); + } + assert_eq!( + String::from_utf8(bytes).expect("should emit UTF-8"), + whole, + "should preserve fallback at channel {channel}, limit {limit}, split {split}" + ); + } + } + } + #[test] fn mixed_flight_first_party_urls_preserve_t_lengths() { for scheme in ["https:", "http:", ""] { diff --git a/crates/trusted-server-core/src/integrations/nextjs/rsc_placeholders.rs b/crates/trusted-server-core/src/integrations/nextjs/rsc_placeholders.rs index 7abe3aa7ca..8b596cbdd8 100644 --- a/crates/trusted-server-core/src/integrations/nextjs/rsc_placeholders.rs +++ b/crates/trusted-server-core/src/integrations/nextjs/rsc_placeholders.rs @@ -78,7 +78,7 @@ impl NextJsRscPlaceholderRewriter { let mut cursor = 0; let mut trimmed = std::mem::take(&mut state.rsc_receiver_trimmed); let mut queued_bytes = state.captured_payload_bytes; - while code[cursor..].contains("__next_f") { + loop { let remaining = &content[cursor..]; let remaining_code = &code[cursor..]; let head = if trimmed { @@ -86,10 +86,12 @@ impl NextJsRscPlaceholderRewriter { } else { first_qualified_flight_push(remaining_code) }; - if let Some(head) = head - && let Some(arguments) = - INERT_FLIGHT_CONTROL_ARGUMENTS.find(&remaining[head.end()..]) - { + // Property references are ordinary source, including after a push. + // Only another qualified call can require capture or raw fallback. + let Some(head) = head else { + break; + }; + if let Some(arguments) = INERT_FLIGHT_CONTROL_ARGUMENTS.find(&remaining[head.end()..]) { cursor += head.end() + arguments.end(); trimmed = false; continue; @@ -185,7 +187,13 @@ impl NextJsRscPlaceholderRewriter { FragmentCapture::CompleteOwned(complete) => { self.rewrite_complete(&complete, true, state, limit, max_queued_payload_bytes) } - FragmentCapture::Suppress => ScriptRewriteAction::RemoveNode, + FragmentCapture::Suppress => { + // Captured Flight bytes are withheld, so downstream stages see + // only empty source or a released receiver prefix, never a raw + // payload. Let GTM retain and rewrite preceding ordinary source. + state.current_fragment_protected = false; + ScriptRewriteAction::RemoveNode + } FragmentCapture::Restore(restored) => { state.rsc_receiver_trimmed = false; state.bypass_rsc |= state.flight_node_owned; @@ -220,7 +228,7 @@ impl NextJsRscPlaceholderRewriter { remaining = &remaining[end + 1..]; remaining_code = &remaining_code[end + 1..]; receiver_trimmed = false; - if !remaining_code.contains("__next_f") { + if !contains_qualified_flight(remaining_code) { break false; } }; @@ -1074,6 +1082,54 @@ mod tests { ); } + #[test] + fn oversized_payload_with_trailing_reference_keeps_later_capture() { + for trimmed in [false, true] { + for reference in ["self.__next_f", "window.__next_f", "foreign.__next_f"] { + let state = IntegrationDocumentState::default(); + let rewriter = + NextJsRscPlaceholderRewriter::new(Arc::new(NextJsIntegrationConfig { + max_combined_payload_bytes: 100, + ..(*test_config()).clone() + })); + let receiver = if trimmed { + assert_eq!( + rewriter.rewrite("self.", &ctx(false, &state)), + ScriptRewriteAction::Keep, + "should release the qualified receiver" + ); + "" + } else { + "self." + }; + let script = format!( + r#"{receiver}__next_f.push([1,"1:T50,{}"]);console.log({reference});"#, + "x".repeat(80) + ); + assert_eq!( + rewriter.rewrite(&script, &ctx(true, &state)), + ScriptRewriteAction::Keep, + "should retain oversized script" + ); + assert!( + !document_state(&state) + .lock() + .expect("should lock document state") + .bypass_rsc, + "should not bypass a complete group with a trailing reference" + ); + assert!( + matches!( + rewriter + .rewrite(r#"self.__next_f.push([1,"1:T3,abc"])"#, &ctx(true, &state)), + ScriptRewriteAction::Replace(_) + ), + "should still capture the next script" + ); + } + } + } + #[test] fn oversized_partial_header_bypasses_later_continuation() { for suffix in ["1", "1:", "1:T", "1:T2"] { diff --git a/crates/trusted-server-core/src/integrations/nextjs/script_lexer.rs b/crates/trusted-server-core/src/integrations/nextjs/script_lexer.rs index df1aa278a6..a4d3822cd6 100644 --- a/crates/trusted-server-core/src/integrations/nextjs/script_lexer.rs +++ b/crates/trusted-server-core/src/integrations/nextjs/script_lexer.rs @@ -275,16 +275,16 @@ impl ScriptLexer { class.parenthesis_depth == self.parentheses.len() && class.brace_depth == self.braces.len() }); - let kind = if class_body { + // A superclass function can open at the class head's + // depth. Its body must not consume the pending class. + let kind = if let Some(body) = pending_body { + body + } else if class_body { self.classes.pop().expect("should have pending class").body + } else if self.statement_start || !self.expression_start { + BraceKind::StatementBlock } else { - pending_body.unwrap_or( - if self.statement_start || !self.expression_start { - BraceKind::StatementBlock - } else { - BraceKind::Object - }, - ) + BraceKind::Object }; if self.braces.len() == MAX_TEMPLATE_DEPTH { self.mode = Mode::Opaque; @@ -458,6 +458,9 @@ mod tests { "class Demo { method() { return 1 } } /self.__next_f.push()/;", "const n = class { method() { return 1 } } / 2;", "const n = class extends mixin({}) {} / 2;", + "const n = class extends function() {} {} / 2;", + "const n = class extends function Base() {} {} / 2;", + "const n = class extends (function() {}) {} / 2;", "const n = class extends (class {}) {} / 2;", "const n = {class: {}, method() { return 1 }} / 2;", "const n = {method() { if (ready) { work() } /self.__next_f.push()/ }} / 2;", From af697810ba3e5bd22a0afe7e45fbb1830db3ad89 Mon Sep 17 00:00:00 2001 From: Christian Date: Mon, 5 Oct 2026 14:20:19 -0500 Subject: [PATCH 09/15] Fail closed on uncaptured Flight source --- .../src/integrations/nextjs/mod.rs | 160 +++++++++++++++++- .../integrations/nextjs/rsc_placeholders.rs | 157 ++++++++++++++--- .../src/integrations/nextjs/rsc_stream.rs | 10 ++ 3 files changed, 296 insertions(+), 31 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/nextjs/mod.rs b/crates/trusted-server-core/src/integrations/nextjs/mod.rs index d76b82be1c..4b6292fb12 100644 --- a/crates/trusted-server-core/src/integrations/nextjs/mod.rs +++ b/crates/trusted-server-core/src/integrations/nextjs/mod.rs @@ -255,7 +255,7 @@ mod tests { } #[test] - fn non_code_flight_spellings_preserve_gtm_and_later_flight_rewrites() { + fn non_code_flight_spellings_preserve_source_and_later_flight_rewrites() { let payload = "0:{\"url\":\"https://origin.example.com/page\"}\n"; let flight = flight_html(&[payload]); for prefix in [ @@ -291,8 +291,8 @@ mod tests { ); let output = String::from_utf8(bytes).expect("should emit UTF-8"); assert!( - output.contains(&format!("")), - "should preserve non-code source and rewrite ordinary GTM at split {split}: {output}" + output.contains(&format!("")), + "should conservatively retain GTM beside inert Flight at split {split}: {output}" ); assert!( output.contains(r#"\"url\":\"https://test.example.com/page\""#), @@ -421,6 +421,160 @@ mod tests { } } + #[test] + fn lexer_false_negatives_preserve_raw_flight_and_later_rewrites() { + let data = json!({"url": "https://origin.example.com/page", "tag": "http://www.googletagmanager.com/gtm.js?id=GTM-MIX1", "text": "é😀"}); + let body = data.to_string(); + let payload = format!("1:T{:x},{body}", body.len()); + let push = format!( + "self.__next_f.push([1,{}]);", + serde_json::to_string(&payload).expect("should encode Flight payload") + ); + let mut rewritten = data; + rewritten["url"] = json!("https://test.example.com/page"); + rewritten["tag"] = json!("/integrations/google_tag_manager/gtm.js?id=GTM-MIX1"); + for prefix in [ + "foo: { bar() } /'/.test(s);", + "switch (k) { case 1: {} /'/.test(s) }", + "const f = () => {}\n/'/.test(s);", + "const o = {while: (a) / 2, b: '/'};", + "for (const m of /'/g.exec(s) || []) {}", + ] { + // Exercise both an entirely missed script and a missed call after + // a captured one. Capturing a safe payload must not unprotect raw + // Flight elsewhere in the same parser fragment. + for leading in ["", push.as_str()] { + let source = format!("{leading}{prefix}{push}"); + let assert_output = |output: &str| { + let payloads = flight_payloads(output); + let missed = usize::from(!leading.is_empty()); + assert_eq!(payloads.len(), missed + 2, "should emit each push once"); + assert_eq!( + payloads[missed], payload, + "should retain missed Flight bytes and lengths: {prefix}" + ); + if missed != 0 { + assert_mixed_t_model(&payloads[0], &rewritten); + } + assert_mixed_t_model(&payloads[missed + 1], &rewritten); + assert!(!output.contains("__ts_rsc_"), "should resolve all captures"); + }; + for split in (0..=source.len()).filter(|split| source.is_char_boundary(*split)) { + let mut processor = create_html_processor(mixed_config(10000, 10000)); + let first = format!("", &source[split..]); + let mut bytes = processor + .process_chunk(first.as_bytes(), false) + .expect("should accept ambiguous JavaScript prefix"); + bytes.extend( + processor + .process_chunk(last.as_bytes(), true) + .expect("should preserve raw and finish later Flight"), + ); + assert_output(&String::from_utf8(bytes).expect("should emit UTF-8")); + } + let html = format!(""); + for compression in [Compression::None, Compression::Gzip] { + for chunk_size in [32, 1000, 8192] { + assert_output(&mixed_output_with_compression( + &html, + compression, + chunk_size, + )); + } + } + } + } + } + + #[test] + fn raw_flight_protection_survives_lexer_resynchronization_and_capture() { + let missed_body = "`http://www.googletagmanager.com/gtm.js`"; + let missed_payload = format!("1:T{:x},{missed_body}", missed_body.len()); + let missed_push = format!( + "self.__next_f.push([1,{}]);", + serde_json::to_string(&missed_payload).expect("should encode missed Flight") + ); + let data = json!({"url": "https://origin.example.com/page", "tag": "http://www.googletagmanager.com/gtm.js?id=GTM-MIX1"}); + let body = data.to_string(); + let push = format!( + "self.__next_f.push([1,{}]);", + serde_json::to_string(&format!("1:T{:x},{body}", body.len())) + .expect("should encode capturable Flight") + ); + let mut expected = data; + expected["url"] = json!("https://test.example.com/page"); + expected["tag"] = json!("/integrations/google_tag_manager/gtm.js?id=GTM-MIX1"); + let source = format!("foo: {{ bar() }} /'/.test(s);{missed_push} /'/.test(s);{push}"); + let assert_output = |output: &str, boundary: usize| { + let payloads = flight_payloads(output); + assert_eq!( + payloads.len(), + 3, + "should retain missed and captured pushes" + ); + assert_eq!( + payloads[0], missed_payload, + "should preserve raw continuation at boundary {boundary}" + ); + assert_mixed_t_model(&payloads[1], &expected); + assert_mixed_t_model(&payloads[2], &expected); + assert!( + !output.contains("__ts_rsc_"), + "should resolve safe captures" + ); + }; + for split in 0..=source.len() { + let mut processor = create_html_processor(mixed_config(10000, 10000)); + let first = format!("", &source[split..]); + let mut bytes = processor + .process_chunk(first.as_bytes(), false) + .expect("should accept raw Flight prefix"); + bytes.extend( + processor + .process_chunk(last.as_bytes(), true) + .expect("should finish raw continuation and safe captures"), + ); + assert_output(&String::from_utf8(bytes).expect("should emit UTF-8"), split); + } + let html = format!(""); + for compression in [Compression::None, Compression::Gzip] { + for chunk_size in [32, 1000, 8192] { + assert_output( + &mixed_output_with_compression(&html, compression, chunk_size), + chunk_size, + ); + } + } + // A qualified head can be observed but not buffered when an earlier + // unrelated property makes the tentative receiver claim fail. + let body_start = missed_push.find('`').expect("should locate raw T body"); + let first = format!( + "", &continuation[split..]); + bytes.extend( + processor + .process_chunk(last.as_bytes(), true) + .expect("should finish later safe captures"), + ); + assert_output(&String::from_utf8(bytes).expect("should emit UTF-8"), split); + } + } + #[test] fn mixed_next_data_flight_mentions_do_not_claim_json_or_hide_gtm() { for text in [ diff --git a/crates/trusted-server-core/src/integrations/nextjs/rsc_placeholders.rs b/crates/trusted-server-core/src/integrations/nextjs/rsc_placeholders.rs index 8b596cbdd8..c6d2cc0be2 100644 --- a/crates/trusted-server-core/src/integrations/nextjs/rsc_placeholders.rs +++ b/crates/trusted-server-core/src/integrations/nextjs/rsc_placeholders.rs @@ -76,6 +76,7 @@ impl NextJsRscPlaceholderRewriter { // capture. A malformed or unrecognized remainder stays original/protected. let mut ranges = Vec::new(); let mut cursor = 0; + let mut unclaimed_raw_flight = false; let mut trimmed = std::mem::take(&mut state.rsc_receiver_trimmed); let mut queued_bytes = state.captured_payload_bytes; loop { @@ -92,6 +93,8 @@ impl NextJsRscPlaceholderRewriter { break; }; if let Some(arguments) = INERT_FLIGHT_CONTROL_ARGUMENTS.find(&remaining[head.end()..]) { + unclaimed_raw_flight |= + contains_qualified_flight(&content[cursor..cursor + head.start()]); cursor += head.end() + arguments.end(); trimmed = false; continue; @@ -142,6 +145,8 @@ impl NextJsRscPlaceholderRewriter { }; } queued_bytes = total.expect("should have validated queue size"); + unclaimed_raw_flight |= + contains_qualified_flight(&content[cursor..cursor + head.start()]); ranges.push((start, end)); cursor = end + 1; } @@ -152,6 +157,9 @@ impl NextJsRscPlaceholderRewriter { ScriptRewriteAction::Keep }; } + let raw_flight_remaining = state.raw_flight_unclaimed + || unclaimed_raw_flight + || contains_qualified_flight(&content[cursor..]); let mut rewritten = String::with_capacity(content.len()); cursor = 0; for (start, end) in ranges { @@ -168,7 +176,10 @@ impl NextJsRscPlaceholderRewriter { } state.captured_payload_bytes = queued_bytes; rewritten.push_str(&content[cursor..]); - state.current_fragment_protected = false; + // Captured payloads and recognized bootstrap controls are safe, but + // raw push spellings outside them may be executable lexer misses. + // Preserve those bytes even when another call was captured successfully. + state.current_fragment_protected = raw_flight_remaining; ScriptRewriteAction::replace(rewritten) } @@ -188,10 +199,11 @@ impl NextJsRscPlaceholderRewriter { self.rewrite_complete(&complete, true, state, limit, max_queued_payload_bytes) } FragmentCapture::Suppress => { - // Captured Flight bytes are withheld, so downstream stages see - // only empty source or a released receiver prefix, never a raw - // payload. Let GTM retain and rewrite preceding ordinary source. - state.current_fragment_protected = false; + // Captured bytes are withheld, so ordinary preceding source may + // still receive GTM rewriting. An earlier unclaimed raw head can + // leave its continuation in a released prefix, however, and that + // source must stay protected even during this later capture. + state.current_fragment_protected = state.raw_flight_unclaimed; ScriptRewriteAction::RemoveNode } FragmentCapture::Restore(restored) => { @@ -395,9 +407,10 @@ impl IntegrationScriptRewriter for NextJsRscPlaceholderRewriter { fn rewrite(&self, content: &str, ctx: &IntegrationScriptContext<'_>) -> ScriptRewriteAction { let shared = document_state(ctx.document_state); - let mut state = shared + let mut guard = shared .lock() .unwrap_or_else(std::sync::PoisonError::into_inner); + let state = &mut *guard; if std::mem::take(&mut state.next_data_fragment) { // Pages data is not executable Flight, even when a JSON string // contains a complete push spelling. Clear the downstream snapshot @@ -405,6 +418,9 @@ impl IntegrationScriptRewriter for NextJsRscPlaceholderRewriter { state.current_fragment_protected = false; state.flight_node_owned = false; state.flight_qualifier_tail.clear(); + state.raw_flight_seen = false; + state.raw_flight_unclaimed = false; + state.raw_qualifier_tail.clear(); state.flight_lexical = ScriptLexer::default(); return ScriptRewriteAction::Keep; } @@ -412,16 +428,40 @@ impl IntegrationScriptRewriter for NextJsRscPlaceholderRewriter { state.current_fragment_protected = false; return ScriptRewriteAction::Keep; } + // A bounded raw observer is a safety guard, not a capture parser. Inert + // spellings may conservatively miss GTM rewriting in this script, but a + // lexical false negative must never let GTM corrupt raw Flight lengths. + let raw_head_end = observe_flight_qualifier( + content, + &mut state.raw_qualifier_tail, + &mut state.raw_flight_seen, + ); + let previously_owned = state.flight_node_owned; let lexical_start = state.flight_lexical.clone(); let code = state.flight_lexical.mask(content); - if state.flight_lexical.is_opaque() { + let code_head_end = if state.flight_lexical.is_opaque() { // At the lexical nesting limit, retain the existing conservative raw // qualification. Ordinary scripts without a push do not bypass Flight. - observe_flight_qualifier(&mut state, content); + let head_end = observe_flight_qualifier( + content, + &mut state.flight_qualifier_tail, + &mut state.flight_node_owned, + ); state.bypass_rsc |= state.flight_node_owned; + head_end } else { - observe_flight_qualifier(&mut state, &code); - } + observe_flight_qualifier( + &code, + &mut state.flight_qualifier_tail, + &mut state.flight_node_owned, + ) + }; + // An unclaimed raw head may precede this fragment's capture buffer. + // Compare first-head offsets: a later lexical claim in the same fragment + // does not prove an earlier raw head safe, nor does a later fragment. + state.raw_flight_unclaimed |= !previously_owned + && raw_head_end + .is_some_and(|raw_end| code_head_end.is_none_or(|code_end| raw_end < code_end)); // A call head can complete after a tentative property probe overflowed. // Once it proves Flight, preserve its entire raw continuation/group. if state.flight_node_owned @@ -432,11 +472,26 @@ impl IntegrationScriptRewriter for NextJsRscPlaceholderRewriter { { state.bypass_rsc = true; } - state.current_fragment_protected = state.flight_node_owned; - let action = self.rewrite_fragment(content, &code, lexical_start, ctx, &mut state); + state.current_fragment_protected = state.flight_node_owned || state.raw_flight_seen; + let action = self.rewrite_fragment(content, &code, lexical_start, ctx, state); + // Ownership alone does not prove buffering: an earlier unrelated + // property can cause the fragment to stream without a capture claim. + // Never let a later capture unprotect that raw source's continuation. + if !ctx.is_last_in_text_node + && state.raw_flight_seen + && !matches!( + state.rsc_script, + super::rsc_stream::FragmentState::Buffering(_) + ) + { + state.raw_flight_unclaimed = true; + } if ctx.is_last_in_text_node { state.flight_node_owned = false; state.flight_qualifier_tail.clear(); + state.raw_flight_seen = false; + state.raw_flight_unclaimed = false; + state.raw_qualifier_tail.clear(); state.rsc_probe.clear(); state.rsc_receiver_context.clear(); state.rsc_receiver_trimmed = false; @@ -480,34 +535,36 @@ fn first_qualified_flight_push(source: &str) -> Option> { } /// Observe a bounded whitespace-normalized call head without buffering script -/// source, including initializer heads split after document bypass. -fn observe_flight_qualifier(state: &mut super::rsc_stream::NextJsDocumentState, content: &str) { - if state.flight_node_owned { - return; +/// source, including initializer heads split after document bypass. Return its +/// original fragment byte-end so raw and masked observations can be compared. +fn observe_flight_qualifier(content: &str, tail: &mut String, seen: &mut bool) -> Option { + if *seen { + return None; } - for character in content.chars() { + for (offset, character) in content.char_indices() { let character = if character.is_whitespace() { ' ' } else { character }; - if character == ' ' && state.flight_qualifier_tail.ends_with(' ') { + if character == ' ' && tail.ends_with(' ') { continue; } - state.flight_qualifier_tail.push(character); - if state.flight_qualifier_tail.len() > FLIGHT_QUALIFIER_TAIL_BYTES { - let start = state.flight_qualifier_tail.len() - FLIGHT_QUALIFIER_TAIL_BYTES; - let start = (start..state.flight_qualifier_tail.len()) - .find(|index| state.flight_qualifier_tail.is_char_boundary(*index)) + tail.push(character); + if tail.len() > FLIGHT_QUALIFIER_TAIL_BYTES { + let start = tail.len() - FLIGHT_QUALIFIER_TAIL_BYTES; + let start = (start..tail.len()) + .find(|index| tail.is_char_boundary(*index)) .expect("should find bounded qualifier character boundary"); - state.flight_qualifier_tail.drain(..start); + tail.drain(..start); } - if character == '(' && contains_qualified_flight(&state.flight_qualifier_tail) { - state.flight_node_owned = true; - state.flight_qualifier_tail.clear(); - break; + if character == '(' && contains_qualified_flight(tail) { + *seen = true; + tail.clear(); + return Some(offset + 1); } } + None } /// Bytes to withhold so a `__next_f` identifier split across text fragments can @@ -573,6 +630,50 @@ mod tests { let shared = document_state(&state); let state = shared.lock().expect("should lock state"); assert!(!state.flight_node_owned, "quoted text must not own Flight"); + assert!( + state.raw_flight_seen, + "raw spelling should trigger only protection" + ); + assert!( + state.current_fragment_protected, + "should protect possible raw Flight" + ); + assert!( + !state.bypass_rsc, + "inert source must not bypass later scripts" + ); + } + + #[test] + fn raw_qualifier_observation_is_bounded_and_survives_every_split() { + for head in [ + "self.__next_f . push (", + "window.__next_f . push (", + "(self.__next_f = self.__next_f || []).push (", + "(window.__next_f = window.__next_f || []).push (", + ] { + let head = head.replace(' ', &" \n".repeat(64)); + for split in 0..=head.len() { + let mut tail = String::new(); + let mut seen = false; + let _ = observe_flight_qualifier(&"é😀".repeat(1000), &mut tail, &mut seen); + assert!( + tail.len() <= FLIGHT_QUALIFIER_TAIL_BYTES, + "should bound raw tail" + ); + assert!(!seen, "ordinary source must not protect Flight"); + let _ = observe_flight_qualifier(&head[..split], &mut tail, &mut seen); + let _ = observe_flight_qualifier(&head[split..], &mut tail, &mut seen); + assert!( + seen, + "should recognize normalized raw head at split {split}" + ); + assert!( + tail.is_empty(), + "should release tail once qualification succeeds" + ); + } + } } #[test] diff --git a/crates/trusted-server-core/src/integrations/nextjs/rsc_stream.rs b/crates/trusted-server-core/src/integrations/nextjs/rsc_stream.rs index 22812c707e..28a1cc45d5 100644 --- a/crates/trusted-server-core/src/integrations/nextjs/rsc_stream.rs +++ b/crates/trusted-server-core/src/integrations/nextjs/rsc_stream.rs @@ -59,6 +59,13 @@ pub(super) struct NextJsDocumentState { pub(super) current_fragment_protected: bool, /// Bounded qualifier observation remains active even after document bypass. pub(super) flight_qualifier_tail: String, + /// Raw qualification protects possible Flight even when the lexer misses it. + /// It never authorizes capture and lasts only for the current script node. + pub(super) raw_flight_seen: bool, + /// A raw head observed without a lexical claim may have already streamed. + /// Later captures cannot make its remaining source safe for ordinary GTM. + pub(super) raw_flight_unclaimed: bool, + pub(super) raw_qualifier_tail: String, /// Lexical context advances with original parser fragments, never replacements. pub(super) flight_lexical: ScriptLexer, /// Snapshot at the beginning of a claimed buffer for source-offset matching. @@ -82,6 +89,9 @@ impl Default for NextJsDocumentState { flight_node_owned: false, current_fragment_protected: false, flight_qualifier_tail: String::new(), + raw_flight_seen: false, + raw_flight_unclaimed: false, + raw_qualifier_tail: String::new(), flight_lexical: ScriptLexer::default(), rsc_lexical_start: ScriptLexer::default(), } From 514a617a0e9acd24330ad81f478c99ed02dd25ec Mon Sep 17 00:00:00 2001 From: Christian Date: Mon, 5 Oct 2026 14:20:19 -0500 Subject: [PATCH 10/15] Accept underscore request hosts safely --- .../trusted-server-adapter-axum/tests/routes.rs | 8 +++++++- crates/trusted-server-core/src/http_util.rs | 17 +++++++++++------ 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/crates/trusted-server-adapter-axum/tests/routes.rs b/crates/trusted-server-adapter-axum/tests/routes.rs index b9c6379d5e..f58013d5dd 100644 --- a/crates/trusted-server-adapter-axum/tests/routes.rs +++ b/crates/trusted-server-adapter-axum/tests/routes.rs @@ -994,7 +994,13 @@ async fn nextjs_gtm_rejects_malformed_request_hosts() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn nextjs_gtm_accepts_local_request_hosts_and_ports() { - for host in ["localhost:3000", "example.com:8443", "[::1]:3000"] { + for host in [ + "localhost:3000", + "example.com:8443", + "[::1]:3000", + "my_service:8080", + "_service_.example.com:8443", + ] { let origin = Arc::new(nextjs_auction::NextJsAuctionOrigin::with_html_response( &nextjs_auction::script_composition_html(), false, diff --git a/crates/trusted-server-core/src/http_util.rs b/crates/trusted-server-core/src/http_util.rs index 7e39973781..7c19e526df 100644 --- a/crates/trusted-server-core/src/http_util.rs +++ b/crates/trusted-server-core/src/http_util.rs @@ -153,8 +153,9 @@ impl RequestInfo { /// before routing. On those paths, `Host` and [`ClientInfo`] TLS detection /// are the effective sources. Other adapters retain their existing policy. /// - /// The selected host must be a DNS name, IPv4 address, or bracketed IPv6 - /// address, optionally followed by a numeric port. Forwarded-header + /// The selected host must be an ASCII hostname (including underscores for + /// local service names), IPv4 address, or bracketed IPv6 address, optionally + /// followed by a numeric port. Forwarded-header /// precedence is unchanged. Requests without any host retain an empty host. /// /// # Errors @@ -235,11 +236,11 @@ fn is_valid_request_host(value: &str) -> bool { || host.split('.').any(|label| { label.is_empty() || label.len() > 63 - || !label.starts_with(|c: char| c.is_ascii_alphanumeric()) - || !label.ends_with(|c: char| c.is_ascii_alphanumeric()) + || !label.starts_with(|c: char| c.is_ascii_alphanumeric() || c == '_') + || !label.ends_with(|c: char| c.is_ascii_alphanumeric() || c == '_') || !label .bytes() - .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-') + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) }) { return false; @@ -650,6 +651,11 @@ mod tests { "example.com:443", "localhost:3000", "internal-proxy:8080", + "my_service:8080", + "service_.example.com:8443", + "_service.example.com", + "_.example.com", + "bad_label.example.com", "192.0.2.1:8080", "[2001:db8::1]", "[2001:db8::1]:443", @@ -693,7 +699,6 @@ mod tests { "example.com:+443", "example.com:443:80", "bad host.example.com", - "bad_label.example.com", "-bad.example.com", "bad-.example.com", "example..com", From 4a1a569513d4256785f4c4ec710841af56cae462 Mon Sep 17 00:00:00 2001 From: ChristianPavilonis Date: Wed, 14 Oct 2026 14:03:37 -0500 Subject: [PATCH 11/15] Fix streamed Flight fallback and GTM routing, support serialized form state, and apply approved cleanup. --- .../tests/routes.rs | 9 +- .../trusted-server-adapter-fastly/src/app.rs | 14 +- .../trusted-server-core/src/html_processor.rs | 111 ++--- .../src/integrations/google_tag_manager.rs | 83 ++-- .../src/integrations/nextjs/mod.rs | 348 ++++++++++++++-- .../src/integrations/nextjs/rsc.rs | 17 +- .../integrations/nextjs/rsc_placeholders.rs | 381 +++++++++++++++--- .../src/integrations/nextjs/rsc_stream.rs | 22 +- .../src/integrations/nextjs/script_lexer.rs | 63 ++- crates/trusted-server-core/src/publisher.rs | 1 - .../trusted-server-core/src/test_support.rs | 20 +- 11 files changed, 867 insertions(+), 202 deletions(-) diff --git a/crates/trusted-server-adapter-axum/tests/routes.rs b/crates/trusted-server-adapter-axum/tests/routes.rs index f58013d5dd..3e9128f490 100644 --- a/crates/trusted-server-adapter-axum/tests/routes.rs +++ b/crates/trusted-server-adapter-axum/tests/routes.rs @@ -1076,13 +1076,18 @@ async fn nextjs_gtm_html_response_is_complete() { .call(request) .await .expect("should serve HTML"); - assert_eq!(response.status().as_u16(), 200); + assert_eq!( + response.status().as_u16(), + 200, + "should return a successful composed publisher response" + ); assert_eq!( response .headers() .get("content-encoding") .map(|value| value.to_str().expect("should have ASCII coding")), - gzip.then_some("gzip") + gzip.then_some("gzip"), + "should retain the requested response compression" ); let body = axum::body::to_bytes(response.into_body(), usize::MAX) .await diff --git a/crates/trusted-server-adapter-fastly/src/app.rs b/crates/trusted-server-adapter-fastly/src/app.rs index 06de109244..a0489ed6f5 100644 --- a/crates/trusted-server-adapter-fastly/src/app.rs +++ b/crates/trusted-server-adapter-fastly/src/app.rs @@ -3188,14 +3188,22 @@ mod tests { .body(Body::empty()) .expect("should build navigation"); let response = block_on(super::dispatch_fallback(&state, &services, request)); - assert_eq!(response.status(), StatusCode::OK); - assert!(!response.headers().contains_key(header::CONTENT_LENGTH)); + assert_eq!( + response.status(), + StatusCode::OK, + "should return a successful composed publisher response" + ); + assert!( + !response.headers().contains_key(header::CONTENT_LENGTH), + "should drop the stale content length after streaming rewrites" + ); assert_eq!( response .headers() .get(header::CONTENT_ENCODING) .map(|value| value.to_str().expect("should have ASCII coding")), - gzip.then_some("gzip") + gzip.then_some("gzip"), + "should retain the requested response compression" ); let Body::Stream(mut stream) = response.into_body() else { panic!("should retain lazy Fastly body"); diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 87c0991a98..85ad7348b9 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -664,60 +664,63 @@ pub fn create_html_processor(config: HtmlProcessorConfig) -> impl StreamProcesso // Selector callbacks retain lol_html's CSS matching; only the composed text // callback mutates a fragment. Start each script with fresh match state. - let script_matches = Rc::new(RefCell::new(vec![false; script_rewriters.len()])); - let reset_matches = script_matches.clone(); - element_content_handlers.push(element!("script", move |_el| { - reset_matches.borrow_mut().fill(false); - Ok(()) - })); - for (index, rewriter) in script_rewriters.iter().enumerate() { - let matches = script_matches.clone(); - element_content_handlers.push(element!(rewriter.selector(), move |_el| { - matches.borrow_mut()[index] = true; + // Without script rewriters, skip script text capture entirely. + if !script_rewriters.is_empty() { + let script_matches = Rc::new(RefCell::new(vec![false; script_rewriters.len()])); + let reset_matches = script_matches.clone(); + element_content_handlers.push(element!("script", move |_el| { + reset_matches.borrow_mut().fill(false); Ok(()) })); - } - let script_patterns = patterns.clone(); - let script_document_state = document_state.clone(); - element_content_handlers.push(text!("script", move |text| { - let mut working = Cow::Borrowed(text.as_str()); - let mut changed = false; - for (rewriter, matched) in script_rewriters.iter().zip(script_matches.borrow().iter()) { - if !matched { - continue; - } - let ctx = IntegrationScriptContext { - selector: rewriter.selector(), - request_host: &script_patterns.request_host, - request_scheme: &script_patterns.request_scheme, - origin_host: &script_patterns.origin_host, - is_last_in_text_node: text.last_in_text_node(), - max_buffered_script_bytes: config.max_buffered_body_bytes, - document_state: &script_document_state, - }; - match rewriter.rewrite(&working, &ctx) { - ScriptRewriteAction::Keep => {} - ScriptRewriteAction::Replace(rewritten) => { - working = Cow::Owned(rewritten); - changed = true; + for (index, rewriter) in script_rewriters.iter().enumerate() { + let matches = script_matches.clone(); + element_content_handlers.push(element!(rewriter.selector(), move |_el| { + matches.borrow_mut()[index] = true; + Ok(()) + })); + } + let script_patterns = patterns.clone(); + let script_document_state = document_state.clone(); + element_content_handlers.push(text!("script", move |text| { + let mut working = Cow::Borrowed(text.as_str()); + let mut changed = false; + for (rewriter, matched) in script_rewriters.iter().zip(script_matches.borrow().iter()) { + if !matched { + continue; } - ScriptRewriteAction::RemoveNode => { - working = Cow::Borrowed(""); - changed = true; + let ctx = IntegrationScriptContext { + selector: rewriter.selector(), + request_host: &script_patterns.request_host, + request_scheme: &script_patterns.request_scheme, + origin_host: &script_patterns.origin_host, + is_last_in_text_node: text.last_in_text_node(), + max_buffered_script_bytes: config.max_buffered_body_bytes, + document_state: &script_document_state, + }; + match rewriter.rewrite(&working, &ctx) { + ScriptRewriteAction::Keep => {} + ScriptRewriteAction::Replace(rewritten) => { + working = Cow::Owned(rewritten); + changed = true; + } + ScriptRewriteAction::RemoveNode => { + working = Cow::Borrowed(""); + changed = true; + } } } - } - if changed { - if working.is_empty() { - text.remove(); - } else if let Cow::Owned(rewritten) = working { - // Script is raw text: HTML entities would change JS/JSON data. - // Emit composed source verbatim rather than escaping it again. - text.replace(&rewritten, ContentType::Html); + if changed { + if working.is_empty() { + text.remove(); + } else if let Cow::Owned(rewritten) = working { + // Script is raw text: HTML entities would change JS/JSON data. + // Emit composed source verbatim rather than escaping it again. + text.replace(&rewritten, ContentType::Html); + } } - } - Ok(()) - })); + Ok(()) + })); + } let rewriter_settings = RewriterSettings { document_content_handlers, @@ -786,7 +789,11 @@ mod tests { "script[data-suppress='yes']" } fn rewrite(&self, _text: &str, ctx: &IntegrationScriptContext<'_>) -> ScriptRewriteAction { - assert_eq!(ctx.selector, self.selector()); + assert_eq!( + ctx.selector, + self.selector(), + "should pass the registered selector to the suppressing stage" + ); ScriptRewriteAction::RemoveNode } } @@ -800,7 +807,11 @@ mod tests { "script#flush" } fn rewrite(&self, text: &str, ctx: &IntegrationScriptContext<'_>) -> ScriptRewriteAction { - assert_eq!(ctx.selector, self.selector()); + assert_eq!( + ctx.selector, + self.selector(), + "should pass the registered selector to the flushing stage" + ); assert!(text.is_empty(), "should see upstream suppression"); if ctx.is_last_in_text_node { ScriptRewriteAction::Replace("flushed-once".into()) diff --git a/crates/trusted-server-core/src/integrations/google_tag_manager.rs b/crates/trusted-server-core/src/integrations/google_tag_manager.rs index cf4f9673b2..9b6f59d9be 100644 --- a/crates/trusted-server-core/src/integrations/google_tag_manager.rs +++ b/crates/trusted-server-core/src/integrations/google_tag_manager.rs @@ -191,7 +191,9 @@ fn flight_url_token_end( /// /// Call only after original T-record boundaries are known, before lengths are /// recomputed. Originals used for fallback must never pass through this function. -pub(crate) fn rewrite_gtm_rsc_span(content: &str) -> String { +/// `record_text` marks exactly one complete T-record body. Other spans can start +/// mid-URL at a payload-script boundary and must not be routed as whole URLs. +pub(crate) fn rewrite_gtm_rsc_span(content: &str, record_text: bool) -> String { fn route(token: &str) -> Option { let captures = GTM_RSC_WHOLE_URL_PATTERN.captures(token)?; let slash = &captures["slash"]; @@ -201,7 +203,7 @@ pub(crate) fn rewrite_gtm_rsc_span(content: &str) -> String { "{slash}integrations{slash}google_tag_manager{path}{suffix}" )) } - if let Some(rewritten) = route(content) { + if record_text && let Some(rewritten) = route(content) { return rewritten; } let mut rewritten = String::with_capacity(content.len()); @@ -448,12 +450,26 @@ pub struct GoogleTagManagerIntegration { proxy_allowed_domains: Vec, } +/// Response-local script buffering; raw fallback never transforms held source. #[derive(Default)] struct GtmScriptState { + /// Withheld fragments, retained in source order until completion or fallback. held: String, + /// An oversized script passes through until its final text fragment. bypass_until_last: bool, } +impl GtmScriptState { + fn release(&mut self, content: &str) -> ScriptRewriteAction { + if self.held.is_empty() { + return ScriptRewriteAction::Keep; + } + let mut restored = std::mem::take(&mut self.held); + restored.push_str(content); + ScriptRewriteAction::Replace(restored) + } +} + impl GoogleTagManagerIntegration { fn new(config: GoogleTagManagerConfig) -> Arc { let proxy_allowed_domains = Self::proxy_allowed_domains(&config); @@ -1126,12 +1142,7 @@ impl IntegrationScriptRewriter for GoogleTagManagerIntegration { if ctx.is_last_in_text_node { state.bypass_until_last = false; } - if state.held.is_empty() { - return ScriptRewriteAction::Keep; - } - let mut restored = std::mem::take(&mut state.held); - restored.push_str(content); - return ScriptRewriteAction::Replace(restored); + return state.release(content); } if state.bypass_until_last { if ctx.is_last_in_text_node { @@ -1144,6 +1155,7 @@ impl IntegrationScriptRewriter for GoogleTagManagerIntegration { if state.held.is_empty() && !might_contain_gtm_prefix(content) { return ScriptRewriteAction::Keep; } + // The cap also applies to a single final fragment with no held bytes. if state .held .len() @@ -1151,12 +1163,7 @@ impl IntegrationScriptRewriter for GoogleTagManagerIntegration { .is_none_or(|length| length > ctx.max_buffered_script_bytes) { state.bypass_until_last = !ctx.is_last_in_text_node; - if state.held.is_empty() { - return ScriptRewriteAction::Keep; - } - let mut restored = std::mem::take(&mut state.held); - restored.push_str(content); - return ScriptRewriteAction::Replace(restored); + return state.release(content); } if !ctx.is_last_in_text_node { state.held.push_str(content); @@ -3559,7 +3566,7 @@ container_id = "GTM-DEFAULT" for slash in ["/", r"\/", r"\\/"] { let flight_source = source.replace('/', slash); assert_eq!( - rewrite_gtm_rsc_span(&flight_source), + rewrite_gtm_rsc_span(&flight_source, true), expected.replace('/', slash), "should preserve Flight routing policy and escapes: {flight_source}" ); @@ -3571,6 +3578,27 @@ container_id = "GTM-DEFAULT" } } + #[test] + fn whole_flight_url_routing_requires_a_complete_record_body() { + let url = "//www.googletagmanager.com/gtm.js?id=GTM-MIX1"; + assert_eq!( + rewrite_gtm_rsc_span(url, false), + url, + "should preserve a possible mid-URL payload fragment" + ); + assert_eq!( + rewrite_gtm_rsc_span(url, true), + "/integrations/google_tag_manager/gtm.js?id=GTM-MIX1", + "should still route a whole T-record URL" + ); + let fragment = r#"//www.googletagmanager.com/gtm.js?id=GTM-MIX1\",\"href\":\"https://example.com/p\"}\n"#; + assert_eq!( + rewrite_gtm_rsc_span(fragment, false), + fragment, + "should not absorb trailing model source into a whole URL" + ); + } + #[test] fn captured_flight_preserves_quoted_query_delimiters_and_template_interpolation() { for (source, expected) in [ @@ -3604,7 +3632,7 @@ container_id = "GTM-DEFAULT" ), ] { assert_eq!( - rewrite_gtm_rsc_span(source), + rewrite_gtm_rsc_span(source, false), expected, "should preserve query delimiters and template interpolation: {source}" ); @@ -3622,7 +3650,7 @@ container_id = "GTM-DEFAULT" "{quote}{slash}integrations{slash}google_tag_manager{slash}gtm.js{quote}" ); assert_eq!( - rewrite_gtm_rsc_span(&source), + rewrite_gtm_rsc_span(&source, true), expected, "should preserve source escape layer: {source}" ); @@ -3633,11 +3661,11 @@ container_id = "GTM-DEFAULT" "text http://www.googletagmanager.com/gtm.js text", "'http://www.googletagmanager.com/gtm.js/path'", "'http://www.googletagmanager.com/collect;matrix'", - "'http://www.googletagmanager.com.evil/gtm.js'", + "'http://www.googletagmanager.com.evil/gtm.js'", // allow-domain: www.googletagmanager.com.evil "'http://www.googletagmanager.com/gtm.js\"", ] { assert_eq!( - rewrite_gtm_rsc_span(source), + rewrite_gtm_rsc_span(source, true), source, "should not route unsupported or unbounded token" ); @@ -3676,7 +3704,10 @@ container_id = "GTM-DEFAULT" .expect("should consume final"), ); let html = String::from_utf8(bytes).expect("should emit UTF-8"); - assert!(html.contains("var url='/integrations/google_tag_manager/gtm.js';var tail_=1;")); + assert!( + html.contains("var url='/integrations/google_tag_manager/gtm.js';var tail_=1;"), + "should rewrite the GTM URL and preserve trailing script source" + ); assert_eq!( html.matches("var url=").count(), 1, @@ -3713,7 +3744,8 @@ container_id = "GTM-DEFAULT" } assert_eq!( emitted, - "var falsePrefix='googleX';first=1;second=2;var tag='/integrations/google_tag_manager/gtm.js';last=3;" + "var falsePrefix='googleX';first=1;second=2;var tag='/integrations/google_tag_manager/gtm.js';last=3;", + "should release a false prefix and later rewrite the complete GTM URL" ); } @@ -3763,7 +3795,8 @@ container_id = "GTM-DEFAULT" ); assert_eq!( IntegrationScriptRewriter::rewrite(&*integration, "google", &context(false, 6, &a)), - ScriptRewriteAction::RemoveNode + ScriptRewriteAction::RemoveNode, + "should hold the first bounded fragment for this document" ); assert_eq!( IntegrationScriptRewriter::rewrite(&*integration, "X", &context(false, 6, &a)), @@ -3772,11 +3805,13 @@ container_id = "GTM-DEFAULT" ); assert_eq!( IntegrationScriptRewriter::rewrite(&*integration, "tail", &context(true, 6, &a)), - ScriptRewriteAction::Keep + ScriptRewriteAction::Keep, + "should pass through the final fragment after size fallback" ); assert_eq!( IntegrationScriptRewriter::rewrite(&*integration, "google", &context(false, 6, &a)), - ScriptRewriteAction::RemoveNode + ScriptRewriteAction::RemoveNode, + "should start fresh buffering after the previous script finishes" ); assert_eq!( IntegrationScriptRewriter::rewrite(&*integration, "", &context(true, 6, &a)), diff --git a/crates/trusted-server-core/src/integrations/nextjs/mod.rs b/crates/trusted-server-core/src/integrations/nextjs/mod.rs index 4b6292fb12..dce3b0f96c 100644 --- a/crates/trusted-server-core/src/integrations/nextjs/mod.rs +++ b/crates/trusted-server-core/src/integrations/nextjs/mod.rs @@ -112,7 +112,8 @@ pub fn register( "google_tag_manager", )? .map(|_| { - crate::integrations::google_tag_manager::rewrite_gtm_rsc_span as fn(&str) -> String + crate::integrations::google_tag_manager::rewrite_gtm_rsc_span + as fn(&str, bool) -> String }); let stream_processor = Arc::new(NextJsRscStreamProcessorFactory::new( config.clone(), @@ -221,6 +222,15 @@ mod tests { html: &str, compression: Compression, chunk_size: usize, + ) -> String { + html_output_with_compression(html, mixed_config(100000, 100000), compression, chunk_size) + } + + fn html_output_with_compression( + html: &str, + config: HtmlProcessorConfig, + compression: Compression, + chunk_size: usize, ) -> String { let input = if compression == Compression::Gzip { let mut encoder = GzEncoder::new(Vec::new(), flate2::Compression::default()); @@ -237,7 +247,7 @@ mod tests { output_compression: compression, chunk_size, }, - create_html_processor(mixed_config(100000, 100000)), + create_html_processor(config), ); let mut output = Vec::new(); pipeline @@ -255,9 +265,12 @@ mod tests { } #[test] - fn non_code_flight_spellings_preserve_source_and_later_flight_rewrites() { + fn non_code_flight_hints_preserve_source_and_data_heads_bypass_later_flight() { let payload = "0:{\"url\":\"https://origin.example.com/page\"}\n"; let flight = flight_html(&[payload]); + let opaque = format!("const value={}0{};", "{a:".repeat(33), "}".repeat(33)); + let opaque_hint = format!("{opaque}var hint=\"self.__next_f.push(\";"); + let opaque_data = format!("{opaque}var hint=\"self.__next_f.push([1,\";"); for prefix in [ r#"const demo="self.__next_f.push(";"#, r#"const demo='self.__next_f.push([1,"1:T7,example"])';"#, @@ -273,6 +286,8 @@ mod tests { "const demo=`self.__next_f.push(`;", "const demo=`outer ${`inner self.__next_f.push(`}`;", r"const demo=`escaped\` self.__next_f.push( \${self.__next_f.push(}`;", + opaque_hint.as_str(), + opaque_data.as_str(), ] { for split in 0..=prefix.len() { let mut processor = create_html_processor(mixed_config(10000, 10000)); @@ -294,9 +309,14 @@ mod tests { output.contains(&format!("")), "should conservatively retain GTM beside inert Flight at split {split}: {output}" ); + let host = if prefix.contains("[1,") { + "origin.example.com" + } else { + "test.example.com" + }; assert!( - output.contains(r#"\"url\":\"https://test.example.com/page\""#), - "should still rewrite later real Flight at split {split}: {output}" + output.contains(&format!(r#"\"url\":\"https://{host}/page\""#)), + "should bypass later Flight only for data heads at split {split}: {output}" ); assert!( !output.contains("__ts_rsc_"), @@ -314,7 +334,7 @@ mod tests { for (source, expected) in [ ( format!("{fake}{push};/* self.__next_f.push( */"), - format!("{fake}{rewritten_push};/* self.__next_f.push( */"), + format!("{fake}{push};/* self.__next_f.push( */"), ), ( format!("if (ready) {{ work() }} /self.__next_f.push()/;{push};"), @@ -394,13 +414,19 @@ mod tests { .expect("should complete Flight after division"), ); let output = String::from_utf8(bytes).expect("should emit UTF-8"); - assert!(output.starts_with(&format!(""); for compression in [Compression::None, Compression::Gzip] { @@ -415,14 +441,17 @@ mod tests { for payload in payloads { assert_mixed_t_model(&payload, &expected); } - assert!(!output.contains("__ts_rsc_")); + assert!( + !output.contains("__ts_rsc_"), + "should resolve every generated placeholder" + ); } } } } #[test] - fn lexer_false_negatives_preserve_raw_flight_and_later_rewrites() { + fn lexer_false_negatives_bypass_later_flight_and_restore_queued_originals() { let data = json!({"url": "https://origin.example.com/page", "tag": "http://www.googletagmanager.com/gtm.js?id=GTM-MIX1", "text": "é😀"}); let body = data.to_string(); let payload = format!("1:T{:x},{body}", body.len()); @@ -453,10 +482,16 @@ mod tests { payloads[missed], payload, "should retain missed Flight bytes and lengths: {prefix}" ); - if missed != 0 { + // A complete earlier row may already have streamed. If it + // was still queued when bypass began, it must remain raw. + if missed != 0 && payloads[0] != payload { assert_mixed_t_model(&payloads[0], &rewritten); } - assert_mixed_t_model(&payloads[missed + 1], &rewritten); + assert_eq!( + payloads[missed + 1], + payload, + "should bypass every later Flight push" + ); assert!(!output.contains("__ts_rsc_"), "should resolve all captures"); }; for split in (0..=source.len()).filter(|split| source.is_char_boundary(*split)) { @@ -487,6 +522,85 @@ mod tests { } } + #[test] + fn missed_data_head_bypasses_continued_rows_in_nextjs_only_and_mixed_responses() { + let mut settings = create_test_settings(); + settings + .integrations + .insert_config("nextjs", &json!({"enabled": true})) + .expect("should enable Next.js alone"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new(crate::auction::compile_auction_plan(&settings).expect("should compile plan")), + ) + .expect("should create registry"); + let nextjs = config_from_settings(&settings, ®istry); + let data = json!({"url": "https://origin.example.com/page", "tag": "http://www.googletagmanager.com/gtm.js?id=GTM-MIX1", "text": "é😀"}); + let body = data.to_string(); + let row = format!("1:T{:x},{body}\n", body.len()); + let split = row.find("https:").expect("should locate origin scheme") + 2; + let push = |payload: &str| { + format!( + "self.__next_f.push([1,{}]);", + serde_json::to_string(payload).expect("should serialize Flight") + ) + }; + for (mixed, config) in [(false, nextjs), (true, mixed_config(100000, 100000))] { + let mut rewritten = data.clone(); + rewritten["url"] = json!("https://test.example.com/page"); + if mixed { + rewritten["tag"] = json!("/integrations/google_tag_manager/gtm.js?id=GTM-MIX1"); + } + let rewritten_body = rewritten.to_string(); + let rewritten_row = format!("1:T{:x},{rewritten_body}\n", rewritten_body.len()); + for (prefix, rewritable) in [ + ("", true), + ("var ok=1;", true), + ("foo: { bar() } /'/.test(s);", false), + ("switch (k) { case 1: {} /'/.test(s) }", false), + ("const f = () => {}\n/'/.test(s);", false), + ("const o = {while: (a) / 2, b: '/'};", false), + ("for (const m of /'/g.exec(s) || []) {}", false), + ] { + for queued in [false, true] { + // An earlier captured partial row must be restored, even if + // its placeholder reached the downstream processor already. + let (leading, start) = if queued { + (format!("", push(&row[..8])), 8) + } else { + (String::new(), 0) + }; + let html = format!( + "{leading}", + push(&row[start..split]), + push(&row[split..]), + push(&row), + ); + let expected = if rewritable { &rewritten_row } else { &row }; + for compression in [Compression::None, Compression::Gzip] { + for chunk_size in [16, 32, 8192] { + let output = html_output_with_compression( + &html, + config.clone(), + compression, + chunk_size, + ); + assert_eq!( + flight_payloads(&output).concat(), + expected.repeat(2), + "should preserve lengths and queued originals: {prefix}, queued={queued}, mixed={mixed}" + ); + assert!( + !output.contains("__ts_rsc_"), + "should resolve every placeholder on fallback" + ); + } + } + } + } + } + } + #[test] fn raw_flight_protection_survives_lexer_resynchronization_and_capture() { let missed_body = "`http://www.googletagmanager.com/gtm.js`"; @@ -502,9 +616,7 @@ mod tests { serde_json::to_string(&format!("1:T{:x},{body}", body.len())) .expect("should encode capturable Flight") ); - let mut expected = data; - expected["url"] = json!("https://test.example.com/page"); - expected["tag"] = json!("/integrations/google_tag_manager/gtm.js?id=GTM-MIX1"); + let original_payload = format!("1:T{:x},{body}", body.len()); let source = format!("foo: {{ bar() }} /'/.test(s);{missed_push} /'/.test(s);{push}"); let assert_output = |output: &str, boundary: usize| { let payloads = flight_payloads(output); @@ -517,8 +629,14 @@ mod tests { payloads[0], missed_payload, "should preserve raw continuation at boundary {boundary}" ); - assert_mixed_t_model(&payloads[1], &expected); - assert_mixed_t_model(&payloads[2], &expected); + assert_eq!( + payloads[1], original_payload, + "should retain later raw Flight" + ); + assert_eq!( + payloads[2], original_payload, + "should bypass the next script" + ); assert!( !output.contains("__ts_rsc_"), "should resolve safe captures" @@ -575,6 +693,73 @@ mod tests { } } + #[test] + fn complete_form_state_preserves_metadata_and_allows_later_flight_rewrites() { + let data = json!({"url": "https://origin.example.com/page", "tag": "http://www.googletagmanager.com/gtm.js?id=GTM-MIX1"}); + let body = data.to_string(); + let row = format!("1:T{:x},{body}", body.len()); + let split = row.find("https:").expect("should locate scheme") + 2; + let flight = flight_html(&[&row[..split], &row[split..]]); + let mut expected = data; + expected["url"] = json!("https://test.example.com/page"); + expected["tag"] = json!("/integrations/google_tag_manager/gtm.js?id=GTM-MIX1"); + for form_state in [ + json!([{"message": "Saved"}, "k1", "7f3a", 0]), + json!({"nested": [{"quoted": "\" ) ]; self.__next_f.push([1,", "url": "https://origin.example.com/form", "tag": "http://www.googletagmanager.com/gtm.js"}], "text": "é😀"}), + json!("1:T40,'http://www.googletagmanager.com/gtm.js'"), + ] { + let bootstrap = format!( + "(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,{form_state}]);" + ); + let html = format!("{flight}"); + for compression in [Compression::None, Compression::Gzip] { + for chunk_size in [16, 32, 8192] { + let output = mixed_output_with_compression(&html, compression, chunk_size); + assert!( + output.contains(&bootstrap), + "should preserve form-state bytes verbatim" + ); + let (_, later_output) = output + .split_once("") + .expect("should retain the bootstrap script"); + assert_mixed_t_model(&flight_payloads(later_output).concat(), &expected); + assert!( + !output.contains("__ts_rsc_"), + "should resolve later captures" + ); + } + } + } + for arguments in [ + r#"[2,{"message":"Saved"}"#, + r#"[2,{"message":"Saved"},"extra"]"#, + "[2,computeFormState()]", + r#"[2,{'message':'Saved'}]"#, + r#"[3,"AAECAw=="]"#, + ] { + let unsupported = format!("self.__next_f.push({arguments});"); + let html = format!("{flight}"); + for compression in [Compression::None, Compression::Gzip] { + for chunk_size in [16, 8192] { + let output = mixed_output_with_compression(&html, compression, chunk_size); + assert!( + output.contains(&unsupported), + "should preserve unsupported arguments" + ); + assert_eq!( + flight_payloads(&output).concat(), + row, + "should retain raw later Flight" + ); + assert!( + !output.contains("__ts_rsc_"), + "should release original queued source" + ); + } + } + } + } + #[test] fn mixed_next_data_flight_mentions_do_not_claim_json_or_hide_gtm() { for text in [ @@ -642,7 +827,8 @@ mod tests { expected["tag"] = json!("/integrations/google_tag_manager/gtm.js"); assert_eq!( serde_json::from_str::(source).expect("should retain JSON"), - expected + expected, + "should preserve valid JSON and rewrite the expected model" ); assert!( output.ends_with( @@ -833,7 +1019,8 @@ mod tests { ); assert_eq!( &serde_json::from_str::(body).expect("should retain model JSON"), - expected + expected, + "should retain the complete expected Flight model" ); } @@ -868,8 +1055,14 @@ mod tests { let payloads = flight_payloads(&output); assert_eq!(payloads.len(), 1, "should emit payload exactly once"); assert_mixed_t_model(&payloads[0], &expected); - assert!(!output.contains("__ts_rsc_")); - assert!(output.ends_with("

suffix

")); + assert!( + !output.contains("__ts_rsc_"), + "should resolve every generated placeholder" + ); + assert!( + output.ends_with("

suffix

"), + "should retain the complete trailing HTML suffix" + ); } } } @@ -895,11 +1088,12 @@ mod tests { let (header, transformed) = payloads[0].split_once(',').expect("should retain T"); assert_eq!( usize::from_str_radix(&header[3..], 16).expect("should parse hex"), - transformed.len() + transformed.len(), + "should recompute the T length from transformed bytes" ); assert_eq!( - transformed, - r"'/integrations/google_tag_manager/gtm.js?q=\'x\''" + transformed, r"'/integrations/google_tag_manager/gtm.js?q=\'x\''", + "should route the tag while preserving escaped query quotes" ); } } @@ -953,7 +1147,11 @@ mod tests { if payload.starts_with("1:T") { assert_mixed_t_model(&payloads[0], &expected); } else { - assert_eq!(payloads[0], format!("0:{expected}\n")); + assert_eq!( + payloads[0], + format!("0:{expected}\n"), + "should rewrite the model and preserve its row terminator" + ); } for split in (0..=push.len()).filter(|split| push.is_char_boundary(*split)) { let mut processor = create_html_processor(mixed_config(10000, 10000)); @@ -1055,7 +1253,7 @@ mod tests { for chunk in [32, 64, 1000, 8192] { let output = mixed_output(&html, chunk); let payloads = flight_payloads(&output); - assert_eq!(payloads.len(), 1); + assert_eq!(payloads.len(), 1, "should emit exactly one Flight payload"); let (header, body) = payloads[0].split_once(',').expect("should retain T record"); assert!( @@ -1079,13 +1277,50 @@ mod tests { 32, ); let payloads = flight_payloads(&output); - assert!(payloads[0].contains("test.example.com/page")); + assert!( + payloads[0].contains("test.example.com/page"), + "should rewrite the origin URL in the captured payload" + ); assert!( payloads[0].contains("/integrations/google_tag_manager/gtm.js"), "should rewrite queryless escaped quote URL" ); } + #[test] + fn flight_url_split_after_scheme_preserves_the_whole_destination() { + for scheme in ["http:", "https:"] { + for slash in ["/", r"\/"] { + let body = json!({ + "tag": format!("{scheme}//www.googletagmanager.com/gtm.js?id=GTM-MIX1"), + "href": "https://origin.example.com/p", + }); + let payload = format!("0:{body}\n").replace('/', slash); + let tag_prefix = format!("{scheme}{slash}{slash}www.googletagmanager.com"); + let split = + payload.find(&tag_prefix).expect("should locate tag scheme") + scheme.len(); + let html = flight_html(&[&payload[..split], &payload[split..]]); + let expected = payload.replace("origin.example.com", "test.example.com"); + for compression in [Compression::None, Compression::Gzip] { + for chunk_size in [16, 32, 8192] { + let output = mixed_output_with_compression(&html, compression, chunk_size); + let payloads = flight_payloads(&output); + assert_eq!(payloads.len(), 2, "should retain both payload boundaries"); + assert_eq!( + payloads.concat(), + expected, + "should preserve the split GTM URL and rewrite the complete origin URL" + ); + assert!( + !output.contains("__ts_rsc_"), + "should resolve every captured payload" + ); + } + } + } + } + } + #[test] fn mixed_flight_batch_rewrites_every_payload() { let bodies = [ @@ -1200,9 +1435,13 @@ mod tests { assert_eq!( serde_json::from_str::(script) .expect("should retain valid JSON"), - expected + expected, + "should preserve valid Pages JSON and its expected values" + ); + assert!( + output.ends_with(""), + "should retain the complete trailing HTML suffix" ); - assert!(output.ends_with("")); } } } @@ -1213,10 +1452,10 @@ mod tests { let mut a = create_html_processor(config.clone()); let mut b = create_html_processor(config); let mut output_a = a - .process_chunk(b"", true) @@ -1228,11 +1467,13 @@ mod tests { ); assert_eq!( String::from_utf8(output_a).expect("should emit UTF-8"), - "" + "", + "should isolate document A and route its GTM URL" ); assert_eq!( String::from_utf8(output_b).expect("should emit UTF-8"), - "" + "", + "should isolate document B and route its collect URL" ); } @@ -1260,7 +1501,8 @@ mod tests { .expect("should close script"); assert_eq!( serde_json::from_str::(script).expect("should keep valid JSON"), - json!({"padding": prefix, "href": "https://test.example.com/page"}) + json!({"padding": prefix, "href": "https://test.example.com/page"}), + "should preserve padding and rewrite the Pages href" ); } } @@ -1297,12 +1539,18 @@ mod tests { .find(|payload| payload.starts_with("f:T")) .expect("should retain designated Flight proof"); assert_mixed_t_model(proof, &expected); - assert!(!output.contains("__ts_rsc_")); + assert!( + !output.contains("__ts_rsc_"), + "should resolve every generated placeholder" + ); assert!( output.contains("googletagmanager.com/ns.html"), "should retain unsupported iframe routing" ); - assert!(output.trim_end().ends_with("")); + assert!( + output.trim_end().ends_with(""), + "should retain the final HTML closing tag" + ); } } @@ -1336,20 +1584,27 @@ mod tests { let (header, rewritten) = combined.split_once(',').expect("should retain T header"); assert_eq!( usize::from_str_radix(&header[3..], 16).expect("should parse length"), - rewritten.len() + rewritten.len(), + "should recompute the T length from rewritten bytes" ); let data: serde_json::Value = serde_json::from_str(rewritten).expect("should preserve model JSON"); if split == (payload.find("http:").expect("should locate URL") - 1) { - assert_eq!(data["tag"], "/integrations/google_tag_manager/gtm.js"); + assert_eq!( + data["tag"], "/integrations/google_tag_manager/gtm.js", + "should route the captured GTM tag" + ); } else { assert_eq!( data["tag"], "http://www.googletagmanager.com/gtm.js", "should safely preserve URL split across actual payload scripts" ); } - assert_eq!(data["url"], "https://test.example.com/page"); - assert_eq!(data["text"], "é😀"); + assert_eq!( + data["url"], "https://test.example.com/page", + "should rewrite the captured origin URL" + ); + assert_eq!(data["text"], "é😀", "should preserve Unicode model text"); } } let escaped = body.replace('/', r"\/"); @@ -1359,11 +1614,15 @@ mod tests { let (header, body) = payloads[0].split_once(',').expect("should retain T"); assert_eq!( usize::from_str_radix(&header[3..], 16).expect("should parse length"), - body.len() + body.len(), + "should match the declared T length to the model bytes" ); let data: serde_json::Value = serde_json::from_str(body).expect("should preserve escaped model JSON"); - assert_eq!(data["tag"], "/integrations/google_tag_manager/gtm.js"); + assert_eq!( + data["tag"], "/integrations/google_tag_manager/gtm.js", + "should route the captured GTM tag" + ); } #[test] @@ -1391,7 +1650,10 @@ mod tests { "should rewrite ordinary script after Flight fallback" ); } - assert!(output.ends_with("")); + assert!( + output.ends_with(""), + "should retain the complete trailing HTML suffix" + ); } } } diff --git a/crates/trusted-server-core/src/integrations/nextjs/rsc.rs b/crates/trusted-server-core/src/integrations/nextjs/rsc.rs index 9a53d5e8d2..8f1e70c6a5 100644 --- a/crates/trusted-server-core/src/integrations/nextjs/rsc.rs +++ b/crates/trusted-server-core/src/integrations/nextjs/rsc.rs @@ -404,11 +404,12 @@ fn rewrite_span( origin_host: &str, request_host: &str, request_scheme: &str, - transform: Option String>, + transform: Option String>, + record_text: bool, ) -> String { let rewritten = rewriter.rewrite_to_string(content, origin_host, request_host, request_scheme); match transform { - Some(transform) => transform(&rewritten), + Some(transform) => transform(&rewritten, record_text), None => rewritten, } } @@ -437,7 +438,7 @@ fn rewrite_rsc_tchunks_with_transform( origin_host: &str, request_host: &str, request_scheme: &str, - transform: Option String>, + transform: Option String>, ) -> String { let Some(chunks) = find_tchunks(content) else { log::warn!( @@ -454,6 +455,7 @@ fn rewrite_rsc_tchunks_with_transform( request_host, request_scheme, transform, + false, ); } @@ -469,6 +471,7 @@ fn rewrite_rsc_tchunks_with_transform( request_host, request_scheme, transform, + false, )); let chunk_content = &content[chunk.header_end..chunk.content_end]; @@ -479,6 +482,7 @@ fn rewrite_rsc_tchunks_with_transform( request_host, request_scheme, transform, + true, ); let new_length = calculate_unescaped_byte_length(&rewritten_content); @@ -501,6 +505,7 @@ fn rewrite_rsc_tchunks_with_transform( request_host, request_scheme, transform, + false, )); result @@ -591,7 +596,7 @@ pub(super) fn rewrite_rsc_scripts_combined_with_transform( request_host: &str, request_scheme: &str, max_combined_payload_bytes: usize, - transform: Option String>, + transform: Option String>, ) -> Vec { let Some((_, preceding_payloads)) = payloads.split_last() else { return Vec::new(); @@ -698,6 +703,7 @@ pub(super) fn rewrite_rsc_scripts_combined_with_transform( request_host, request_scheme, transform, + false, ) }) .collect(); @@ -715,6 +721,7 @@ pub(super) fn rewrite_rsc_scripts_combined_with_transform( request_host, request_scheme, transform, + false, )); let chunk_content = &combined[chunk.header_end..chunk.content_end]; @@ -725,6 +732,7 @@ pub(super) fn rewrite_rsc_scripts_combined_with_transform( request_host, request_scheme, transform, + true, ); let new_length = calculate_unescaped_byte_length_skip_markers(&rewritten_content); @@ -747,6 +755,7 @@ pub(super) fn rewrite_rsc_scripts_combined_with_transform( request_host, request_scheme, transform, + false, )); result.split(RSC_MARKER).map(String::from).collect() diff --git a/crates/trusted-server-core/src/integrations/nextjs/rsc_placeholders.rs b/crates/trusted-server-core/src/integrations/nextjs/rsc_placeholders.rs index c6d2cc0be2..699641e569 100644 --- a/crates/trusted-server-core/src/integrations/nextjs/rsc_placeholders.rs +++ b/crates/trusted-server-core/src/integrations/nextjs/rsc_placeholders.rs @@ -1,6 +1,7 @@ use std::sync::{Arc, LazyLock}; use regex::Regex; +use serde::de::IgnoredAny; use crate::integrations::{ IntegrationScriptContext, IntegrationScriptRewriter, ScriptRewriteAction, @@ -29,19 +30,32 @@ static QUALIFIED_FLIGHT_PUSH_PATTERN: LazyLock = LazyLock::new(|| { .expect("should compile qualified Flight push observation") }); +// A missed data push may open a T row that a later captured script continues. +// Generic push hints still protect only their current script from ordinary GTM. +static QUALIFIED_FLIGHT_DATA_HEAD: LazyLock = LazyLock::new(|| { + Regex::new(r"(?:self|window)\.__next_f\s*(?:\.\s*push|=\s*(?:self|window)\.__next_f\s*\|\|\s*\[\]\s*\)\s*\.\s*push)\s*\(\s*\[\s*1\s*,") + .expect("should compile qualified Flight data head") +}); + // The receiver may already have streamed, leaving a verified trimmed head. static TRIMMED_FLIGHT_PUSH_PATTERN: LazyLock = LazyLock::new(|| { Regex::new(r"^__next_f\s*(?:\.\s*push\s*\(|=\s*(?:self|window)\.__next_f\s*\|\|\s*\[\]\s*\)\s*\.\s*push\s*\()") .expect("should compile trimmed Flight push observation") }); -// Only completed inert bootstrap controls may be skipped. Opaque channel-2 -// strings and incomplete calls must retain the conservative raw fallback. +// Fast path for the small bootstrap controls. Next.js writeInitialInstructions +// serializes channel 2 form state; nextServerDataCallback stores it separately +// from the Flight row stream. Other serialized form values are recognized below. +// Incomplete calls and binary channels retain conservative raw fallback. static INERT_FLIGHT_CONTROL_ARGUMENTS: LazyLock = LazyLock::new(|| { Regex::new(r"^\s*\[\s*(?:0|2\s*,\s*null)\s*\]\s*\)") .expect("should compile inert Flight control arguments") }); +static FLIGHT_FORM_STATE_ARGUMENTS: LazyLock = LazyLock::new(|| { + Regex::new(r"^\s*\[\s*2\s*,").expect("should compile Flight form-state arguments") +}); + // Collapsing whitespace runs bounds either supported call head well below this // size, while keeping the receiver's preceding boundary for qualification. const FLIGHT_QUALIFIER_TAIL_BYTES: usize = 96; @@ -64,6 +78,11 @@ impl NextJsRscPlaceholderRewriter { max_queued_payload_bytes: usize, ) -> ScriptRewriteAction { let code = state.rsc_lexical_start.clone().mask(content); + debug_assert_eq!( + code.len(), + content.len(), + "should preserve masked byte offsets" + ); if !code.contains("__next_f") { return if was_buffered { ScriptRewriteAction::replace(content.to_owned()) @@ -77,6 +96,7 @@ impl NextJsRscPlaceholderRewriter { let mut ranges = Vec::new(); let mut cursor = 0; let mut unclaimed_raw_flight = false; + let mut unclaimed_raw_data = state.raw_data_unclaimed; let mut trimmed = std::mem::take(&mut state.rsc_receiver_trimmed); let mut queued_bytes = state.captured_payload_bytes; loop { @@ -92,10 +112,12 @@ impl NextJsRscPlaceholderRewriter { let Some(head) = head else { break; }; - if let Some(arguments) = INERT_FLIGHT_CONTROL_ARGUMENTS.find(&remaining[head.end()..]) { + if let Some(arguments_end) = inert_flight_control_end(&remaining[head.end()..]) { unclaimed_raw_flight |= contains_qualified_flight(&content[cursor..cursor + head.start()]); - cursor += head.end() + arguments.end(); + unclaimed_raw_data |= + contains_qualified_flight_data(&content[cursor..cursor + head.start()]); + cursor += head.end() + arguments_end; trimmed = false; continue; } @@ -147,9 +169,25 @@ impl NextJsRscPlaceholderRewriter { queued_bytes = total.expect("should have validated queue size"); unclaimed_raw_flight |= contains_qualified_flight(&content[cursor..cursor + head.start()]); + unclaimed_raw_data |= + contains_qualified_flight_data(&content[cursor..cursor + head.start()]); ranges.push((start, end)); cursor = end + 1; } + // Account for every raw data head outside the captured strings and inert + // controls before committing placeholders. A later capture cannot repair + // a header that has already streamed from an uncaptured push. + unclaimed_raw_data |= contains_qualified_flight_data(&content[cursor..]); + if unclaimed_raw_data { + state.bypass_rsc = true; + state.current_fragment_protected = true; + return if was_buffered { + ScriptRewriteAction::replace(content.to_owned()) + } else { + ScriptRewriteAction::Keep + }; + } + state.raw_data_seen = false; if ranges.is_empty() { return if was_buffered { ScriptRewriteAction::replace(content.to_owned()) @@ -219,9 +257,15 @@ impl NextJsRscPlaceholderRewriter { } if is_last && content.len() > limit && content.contains("__next_f") { let code = state.rsc_lexical_start.clone().mask(content); + debug_assert_eq!( + code.len(), + content.len(), + "should preserve masked byte offsets" + ); let mut remaining = content; let mut remaining_code = code.as_str(); let mut receiver_trimmed = trimmed; + let mut unclaimed_raw_data = state.raw_data_unclaimed; let unsafe_continuation = loop { let range = executable_payload_range(remaining, remaining_code, receiver_trimmed); @@ -234,6 +278,15 @@ impl NextJsRscPlaceholderRewriter { ) { break true; } + let head_start = if receiver_trimmed { + 0 + } else { + first_qualified_flight_push(remaining_code) + .expect("should retain the matched qualified head") + .start() + }; + unclaimed_raw_data |= + contains_qualified_flight_data(&remaining[..head_start]); // Skip the entire string so push-like text inside a // payload is not classified as another call. Only the // first receiver can have streamed in an earlier fragment. @@ -241,9 +294,14 @@ impl NextJsRscPlaceholderRewriter { remaining_code = &remaining_code[end + 1..]; receiver_trimmed = false; if !contains_qualified_flight(remaining_code) { - break false; + break unclaimed_raw_data || contains_qualified_flight_data(remaining); } }; + if !unsafe_continuation { + // A recognized oversized push containing only complete + // rows cannot leave a continuation in another script. + state.raw_data_seen = false; + } state.bypass_rsc |= unsafe_continuation || state.captured_payload_bytes > 0 || !state.captured_payloads.is_empty(); @@ -421,6 +479,9 @@ impl IntegrationScriptRewriter for NextJsRscPlaceholderRewriter { state.raw_flight_seen = false; state.raw_flight_unclaimed = false; state.raw_qualifier_tail.clear(); + state.raw_data_seen = false; + state.raw_data_unclaimed = false; + state.raw_data_tail.clear(); state.flight_lexical = ScriptLexer::default(); return ScriptRewriteAction::Keep; } @@ -435,25 +496,43 @@ impl IntegrationScriptRewriter for NextJsRscPlaceholderRewriter { content, &mut state.raw_qualifier_tail, &mut state.raw_flight_seen, + &QUALIFIED_FLIGHT_PUSH_PATTERN, + '(', + ); + let raw_data_head_end = observe_flight_qualifier( + content, + &mut state.raw_data_tail, + &mut state.raw_data_seen, + &QUALIFIED_FLIGHT_DATA_HEAD, + ',', ); let previously_owned = state.flight_node_owned; let lexical_start = state.flight_lexical.clone(); let code = state.flight_lexical.mask(content); + debug_assert_eq!( + code.len(), + content.len(), + "should preserve masked byte offsets" + ); let code_head_end = if state.flight_lexical.is_opaque() { - // At the lexical nesting limit, retain the existing conservative raw - // qualification. Ordinary scripts without a push do not bypass Flight. + // Opaque source still needs per-script protection, but a generic + // push hint cannot open a T row. Only data heads bypass later scripts. let head_end = observe_flight_qualifier( content, &mut state.flight_qualifier_tail, &mut state.flight_node_owned, + &QUALIFIED_FLIGHT_PUSH_PATTERN, + '(', ); - state.bypass_rsc |= state.flight_node_owned; + state.bypass_rsc |= state.raw_data_seen; head_end } else { observe_flight_qualifier( &code, &mut state.flight_qualifier_tail, &mut state.flight_node_owned, + &QUALIFIED_FLIGHT_PUSH_PATTERN, + '(', ) }; // An unclaimed raw head may precede this fragment's capture buffer. @@ -462,6 +541,12 @@ impl IntegrationScriptRewriter for NextJsRscPlaceholderRewriter { state.raw_flight_unclaimed |= !previously_owned && raw_head_end .is_some_and(|raw_end| code_head_end.is_none_or(|code_end| raw_end < code_end)); + // A data head before the first executable push cannot belong to that + // capture, even if its receiver streamed in a prior fragment. Comparing + // against any executable push also excludes hints inside form state. + state.raw_data_unclaimed |= !previously_owned + && raw_data_head_end + .is_some_and(|raw_end| code_head_end.is_none_or(|code_end| raw_end < code_end)); // A call head can complete after a tentative property probe overflowed. // Once it proves Flight, preserve its entire raw continuation/group. if state.flight_node_owned @@ -486,7 +571,22 @@ impl IntegrationScriptRewriter for NextJsRscPlaceholderRewriter { { state.raw_flight_unclaimed = true; } + if !ctx.is_last_in_text_node + && state.raw_data_seen + && !matches!( + state.rsc_script, + super::rsc_stream::FragmentState::Buffering(_) + ) + { + state.raw_data_unclaimed = true; + } if ctx.is_last_in_text_node { + // Raw data can continue a row in the next script. Keep the entire + // remaining Flight stream raw, not merely this GTM snapshot. + state.bypass_rsc |= state.raw_data_seen || state.raw_data_unclaimed; + state.raw_data_seen = false; + state.raw_data_unclaimed = false; + state.raw_data_tail.clear(); state.flight_node_owned = false; state.flight_qualifier_tail.clear(); state.raw_flight_seen = false; @@ -502,6 +602,27 @@ impl IntegrationScriptRewriter for NextJsRscPlaceholderRewriter { } } +/// Recognize complete bootstrap/form-state arguments without changing their bytes. +/// The containing script is already bounded by the capture limit. `IgnoredAny` +/// checks serialized Next.js form state without allocating its values; binary +/// channel 3 is deliberately not accepted because it belongs to the row stream. +fn inert_flight_control_end(arguments: &str) -> Option { + if let Some(control) = INERT_FLIGHT_CONTROL_ARGUMENTS.find(arguments) { + return Some(control.end()); + } + FLIGHT_FORM_STATE_ARGUMENTS.find(arguments)?; + let mut values = serde_json::Deserializer::from_str(arguments).into_iter::<(u8, IgnoredAny)>(); + let (channel, _) = values.next()?.ok()?; + if channel != 2 { + return None; + } + let end = values.byte_offset(); + let remaining = &arguments[end..]; + let closing = remaining.trim_start(); + closing.strip_prefix(')')?; + Some(end + remaining.len() - closing.len() + 1) +} + /// Match a call head only in executable source, then read its original string. fn executable_payload_range(content: &str, code: &str, trimmed: bool) -> Option<(usize, usize)> { if trimmed { @@ -521,23 +642,35 @@ fn contains_qualified_flight(source: &str) -> bool { first_qualified_flight_push(source).is_some() } +fn contains_qualified_flight_data(source: &str) -> bool { + first_qualified_flight_head(source, &QUALIFIED_FLIGHT_DATA_HEAD).is_some() +} + fn first_qualified_flight_push(source: &str) -> Option> { - QUALIFIED_FLIGHT_PUSH_PATTERN - .find_iter(source) - .find(|call| { - let identifier = call.start() - + call - .as_str() - .find("__next_f") - .expect("should find identifier in qualified push pattern"); - receiver_context_is_flight_push(&source[..identifier]) - }) + first_qualified_flight_head(source, &QUALIFIED_FLIGHT_PUSH_PATTERN) +} + +fn first_qualified_flight_head<'a>(source: &'a str, pattern: &Regex) -> Option> { + pattern.find_iter(source).find(|call| { + let identifier = call.start() + + call + .as_str() + .find("__next_f") + .expect("should find identifier in qualified push pattern"); + receiver_context_is_flight_push(&source[..identifier]) + }) } /// Observe a bounded whitespace-normalized call head without buffering script /// source, including initializer heads split after document bypass. Return its /// original fragment byte-end so raw and masked observations can be compared. -fn observe_flight_qualifier(content: &str, tail: &mut String, seen: &mut bool) -> Option { +fn observe_flight_qualifier( + content: &str, + tail: &mut String, + seen: &mut bool, + pattern: &Regex, + trigger: char, +) -> Option { if *seen { return None; } @@ -558,10 +691,10 @@ fn observe_flight_qualifier(content: &str, tail: &mut String, seen: &mut bool) - .expect("should find bounded qualifier character boundary"); tail.drain(..start); } - if character == '(' && contains_qualified_flight(tail) { + if character == trigger && first_qualified_flight_head(tail, pattern).is_some() { *seen = true; tail.clear(); - return Some(offset + 1); + return Some(offset + character.len_utf8()); } } None @@ -644,6 +777,97 @@ mod tests { ); } + #[test] + fn serialized_form_state_requires_complete_arguments() { + for arguments in [ + "[2,42])tail", + "[2,true])tail", + r#"[2,"quoted \" ) ] é😀"] )tail"#, + r#"[2,[{"message":"Saved"},"k1","7f3a",0]])tail"#, + r#"[2,{"nested":[null,false,{"url":"https://origin.example.com/form"}]}])tail"#, + ] { + let end = inert_flight_control_end(arguments) + .expect("should recognize complete serialized form state"); + assert_eq!( + &arguments[end..], + "tail", + "should consume exactly the control arguments" + ); + } + for arguments in [ + "[2] )", + "[2,null,null])", + "[2,null]", + "[2,computeState()])", + "[2,{'message':'Saved'}])", + r#"[2,{"message":"Saved"})"#, + r#"[3,"AAECAw=="])"#, + ] { + assert!( + inert_flight_control_end(arguments).is_none(), + "should preserve fallback for malformed, dynamic, incomplete and binary arguments" + ); + } + } + + #[test] + fn raw_data_observation_is_bounded_and_distinguishes_controls_from_data() { + let whitespace = " ".repeat(128); + for head in [ + "self.__next_f.push([1,".to_owned(), + "window.__next_f . push ( [ 1 ,".to_owned(), + "(self.__next_f=self.__next_f||[]).push([1,".to_owned(), + format!( + "(window.__next_f{whitespace}={whitespace}window.__next_f{whitespace}||{whitespace}[]{whitespace}){whitespace}.{whitespace}push{whitespace}({whitespace}[{whitespace}1{whitespace}," + ), + ] { + for split in 0..=head.len() { + let mut tail = String::new(); + let mut seen = false; + for fragment in [&head[..split], &head[split..]] { + let _ = observe_flight_qualifier( + fragment, + &mut tail, + &mut seen, + &QUALIFIED_FLIGHT_DATA_HEAD, + ',', + ); + assert!( + tail.len() <= FLIGHT_QUALIFIER_TAIL_BYTES, + "should bound the data observer" + ); + } + assert!( + seen, + "should detect a qualified channel-1 head at split {split}" + ); + } + } + for source in [ + "self.__next_f.push(", + "self.__next_f.push([0])", + "self.__next_f.push([2,null])", + "self.__next_f.push([3,", + "self.__next_f.push([10,", + "myself.__next_f.push([1,", + "foreign.__next_f.push([1,", + ] { + let mut tail = String::new(); + let mut seen = false; + let _ = observe_flight_qualifier( + source, + &mut tail, + &mut seen, + &QUALIFIED_FLIGHT_DATA_HEAD, + ',', + ); + assert!( + !seen, + "should not treat generic, control or unqualified heads as data" + ); + } + } + #[test] fn raw_qualifier_observation_is_bounded_and_survives_every_split() { for head in [ @@ -656,14 +880,32 @@ mod tests { for split in 0..=head.len() { let mut tail = String::new(); let mut seen = false; - let _ = observe_flight_qualifier(&"é😀".repeat(1000), &mut tail, &mut seen); + let _ = observe_flight_qualifier( + &"é😀".repeat(1000), + &mut tail, + &mut seen, + &QUALIFIED_FLIGHT_PUSH_PATTERN, + '(', + ); assert!( tail.len() <= FLIGHT_QUALIFIER_TAIL_BYTES, "should bound raw tail" ); assert!(!seen, "ordinary source must not protect Flight"); - let _ = observe_flight_qualifier(&head[..split], &mut tail, &mut seen); - let _ = observe_flight_qualifier(&head[split..], &mut tail, &mut seen); + let _ = observe_flight_qualifier( + &head[..split], + &mut tail, + &mut seen, + &QUALIFIED_FLIGHT_PUSH_PATTERN, + '(', + ); + let _ = observe_flight_qualifier( + &head[split..], + &mut tail, + &mut seen, + &QUALIFIED_FLIGHT_PUSH_PATTERN, + '(', + ); assert!( seen, "should recognize normalized raw head at split {split}" @@ -680,7 +922,7 @@ mod tests { fn unrecognized_qualified_prefix_makes_the_whole_batch_raw_and_protected() { let state = IntegrationDocumentState::default(); let rewriter = NextJsRscPlaceholderRewriter::new(test_config()); - let script = r#"self.__next_f.push([2,"1:T40,'http://www.googletagmanager.com/gtm.js'"]);self.__next_f.push([1,"later"])"#; + let script = r#"self.__next_f.push([3,"AAECAw=="]);self.__next_f.push([1,"later"])"#; assert_eq!( rewriter.rewrite(script, &ctx(true, &state)), ScriptRewriteAction::Keep, @@ -688,14 +930,20 @@ mod tests { ); let shared = document_state(&state); let state = shared.lock().expect("should lock state"); - assert!(state.captured_payloads.is_empty()); - assert!(state.current_fragment_protected); + assert!( + state.captured_payloads.is_empty(), + "should not capture any payload from an unsupported batch" + ); + assert!( + state.current_fragment_protected, + "should protect the unsupported batch from ordinary GTM rewriting" + ); } #[test] fn inert_controls_do_not_make_unsafe_batches_partially_capturable() { for unsafe_call in [ - r#"self.__next_f.push([2,"1:T40,'http://www.googletagmanager.com/gtm.js'"])"#, + r#"self.__next_f.push([3,"AAECAw=="])"#, "self.__next_f.push([0,1])", "self.__next_f.push([2,null", "self.__next_f.push([0]", @@ -717,9 +965,18 @@ mod tests { ); let shared = document_state(&state); let state = shared.lock().expect("should lock state"); - assert!(state.captured_payloads.is_empty()); - assert!(state.current_fragment_protected); - assert!(state.bypass_rsc); + assert!( + state.captured_payloads.is_empty(), + "should not capture any payload from an unsafe batch" + ); + assert!( + state.current_fragment_protected, + "should protect the unsafe batch from ordinary GTM rewriting" + ); + assert!( + state.bypass_rsc, + "should bypass later Flight after an unsupported or incomplete push" + ); } } } @@ -729,18 +986,30 @@ mod tests { for script in [ "(self.__next_f=self.__next_f||[]).push([0]);self.__next_f.push([2,null])", "window.__next_f . push ( [ 0 ] );window.__next_f.push( [ 2 , null ] )", + r#"self.__next_f.push([2,[{"message":"Saved","hint":"self.__next_f.push([1,"},"k1","7f3a",0]])"#, + r#"self.__next_f.push([2,"1:T40,'http://www.googletagmanager.com/gtm.js'"])"#, ] { let state = IntegrationDocumentState::default(); let rewriter = NextJsRscPlaceholderRewriter::new(test_config()); assert_eq!( rewriter.rewrite(script, &ctx(true, &state)), - ScriptRewriteAction::Keep + ScriptRewriteAction::Keep, + "should preserve completed control arguments verbatim" ); let shared = document_state(&state); let state = shared.lock().expect("should lock state"); - assert!(state.captured_payloads.is_empty()); - assert!(state.current_fragment_protected); - assert!(!state.bypass_rsc); + assert!( + state.captured_payloads.is_empty(), + "should keep completed controls out of the Flight payload queue" + ); + assert!( + state.current_fragment_protected, + "should protect completed controls from ordinary GTM rewriting" + ); + assert!( + !state.bypass_rsc, + "should allow later Flight rewriting after completed controls" + ); } } @@ -768,10 +1037,15 @@ mod tests { captured, "should commit every push or none" ); - assert_eq!(state.captured_payload_bytes, captured * payload.len()); + assert_eq!( + state.captured_payload_bytes, + captured * payload.len(), + "should account for exactly the captured payload bytes" + ); assert_eq!( matches!(action, ScriptRewriteAction::Replace(_)), - captured > 0 + captured > 0, + "should replace source only when a qualified payload was captured" ); assert_eq!( state.current_fragment_protected, @@ -792,7 +1066,8 @@ mod tests { .lock() .expect("should lock state") .captured_payloads - .is_empty() + .is_empty(), + "should discard every capture when a later push is incomplete" ); } @@ -843,7 +1118,8 @@ mod tests { let final_fragment = fragment.is_empty(); assert_eq!( rewriter.rewrite(fragment, &ctx(final_fragment, &state)), - ScriptRewriteAction::Keep + ScriptRewriteAction::Keep, + "should preserve every raw fragment during document bypass" ); assert_eq!( IntegrationScriptRewriter::rewrite( @@ -867,10 +1143,12 @@ mod tests { let ordinary = "var ordinary='http://www.googletagmanager.com/gtm.js';"; assert_eq!( rewriter.rewrite(ordinary, &ctx(true, &state)), - ScriptRewriteAction::Keep + ScriptRewriteAction::Keep, + "should leave ordinary script source to the GTM stage" ); assert!( - matches!(IntegrationScriptRewriter::rewrite(&*gtm, ordinary, &ctx(true, &state)), ScriptRewriteAction::Replace(ref text) if text.contains("/integrations/google_tag_manager/gtm.js")) + matches!(IntegrationScriptRewriter::rewrite(&*gtm, ordinary, &ctx(true, &state)), ScriptRewriteAction::Replace(ref text) if text.contains("/integrations/google_tag_manager/gtm.js")), + "should permit GTM rewriting when there is no qualified Flight head" ); } } @@ -903,7 +1181,8 @@ mod tests { let raw = r#"self.__next_f.push([1,"1:T2,'http://www.googletagmanager.com/gtm.js?id=GTM-MIX1'"] )"#; assert_eq!( rewriter.rewrite(raw, &ctx(true, &state)), - ScriptRewriteAction::Keep + ScriptRewriteAction::Keep, + "should preserve raw Flight bytes on fallback" ); assert_eq!( IntegrationScriptRewriter::rewrite(&*gtm, raw, &ctx(true, &state)), @@ -918,7 +1197,8 @@ mod tests { let last = part.ends_with(")"); assert_eq!( rewriter.rewrite(part, &ctx(last, &state)), - ScriptRewriteAction::Keep + ScriptRewriteAction::Keep, + "should preserve raw Flight continuations across fragments" ); assert_eq!( IntegrationScriptRewriter::rewrite(&*gtm, part, &ctx(last, &state)), @@ -929,16 +1209,19 @@ mod tests { let prefix = "var before='google"; assert_eq!( rewriter.rewrite(prefix, &ctx(false, &state)), - ScriptRewriteAction::Keep + ScriptRewriteAction::Keep, + "should leave the ordinary prefix to the GTM stage" ); assert_eq!( IntegrationScriptRewriter::rewrite(&*gtm, prefix, &ctx(false, &state)), - ScriptRewriteAction::RemoveNode + ScriptRewriteAction::RemoveNode, + "should allow GTM to buffer the ordinary prefix" ); let flight = r#"';self.__next_f.push([1,"1:T2,'http://www.googletagmanager.com/gtm.js'"])"#; assert_eq!( rewriter.rewrite(flight, &ctx(false, &state)), - ScriptRewriteAction::Keep + ScriptRewriteAction::Keep, + "should preserve raw Flight when its stage is bypassed" ); assert_eq!( IntegrationScriptRewriter::rewrite(&*gtm, flight, &ctx(false, &state)), @@ -947,7 +1230,8 @@ mod tests { ); assert_eq!( rewriter.rewrite("", &ctx(true, &state)), - ScriptRewriteAction::Keep + ScriptRewriteAction::Keep, + "should complete the empty final Flight fragment without replacement" ); assert_eq!( IntegrationScriptRewriter::rewrite(&*gtm, "", &ctx(true, &state)), @@ -957,7 +1241,8 @@ mod tests { let ordinary = "var url='http://www.googletagmanager.com/gtm.js';"; assert_eq!( rewriter.rewrite(ordinary, &ctx(true, &state)), - ScriptRewriteAction::Keep + ScriptRewriteAction::Keep, + "should leave the later ordinary script to the GTM stage" ); assert!( matches!(IntegrationScriptRewriter::rewrite(&*gtm, ordinary, &ctx(true, &state)), ScriptRewriteAction::Replace(ref value) if value.contains("/integrations/google_tag_manager/gtm.js")), diff --git a/crates/trusted-server-core/src/integrations/nextjs/rsc_stream.rs b/crates/trusted-server-core/src/integrations/nextjs/rsc_stream.rs index 28a1cc45d5..bc70cf49bc 100644 --- a/crates/trusted-server-core/src/integrations/nextjs/rsc_stream.rs +++ b/crates/trusted-server-core/src/integrations/nextjs/rsc_stream.rs @@ -66,6 +66,11 @@ pub(super) struct NextJsDocumentState { /// Later captures cannot make its remaining source safe for ordinary GTM. pub(super) raw_flight_unclaimed: bool, pub(super) raw_qualifier_tail: String, + /// A raw channel-1 head may open a T row continued in a later script. + pub(super) raw_data_seen: bool, + /// Data observed in released source cannot be accounted for by a later buffer. + pub(super) raw_data_unclaimed: bool, + pub(super) raw_data_tail: String, /// Lexical context advances with original parser fragments, never replacements. pub(super) flight_lexical: ScriptLexer, /// Snapshot at the beginning of a claimed buffer for source-offset matching. @@ -92,6 +97,9 @@ impl Default for NextJsDocumentState { raw_flight_seen: false, raw_flight_unclaimed: false, raw_qualifier_tail: String::new(), + raw_data_seen: false, + raw_data_unclaimed: false, + raw_data_tail: String::new(), flight_lexical: ScriptLexer::default(), rsc_lexical_start: ScriptLexer::default(), } @@ -175,13 +183,13 @@ pub(super) fn capture_fragment<'a>( pub(super) struct NextJsRscStreamProcessorFactory { config: Arc, - transform: Option String>, + transform: Option String>, } impl NextJsRscStreamProcessorFactory { pub(super) fn new( config: Arc, - transform: Option String>, + transform: Option String>, ) -> Self { Self { config, transform } } @@ -220,7 +228,7 @@ pub(super) struct NextJsRscStreamProcessor { group: Vec, classifier: RscGroupClassifier, rewriter: RscUrlRewriter, - transform: Option String>, + transform: Option String>, } impl NextJsRscStreamProcessor { @@ -230,7 +238,7 @@ impl NextJsRscStreamProcessor { request_host: String, request_scheme: String, limit: usize, - transform: Option String>, + transform: Option String>, ) -> Self { Self { state, @@ -1206,7 +1214,8 @@ mod tests { processor .process_chunk(input.as_bytes(), true) .expect("should restore held-output overflow"), - format!("{incomplete}{interstitial}{body}").as_bytes() + format!("{incomplete}{interstitial}{body}").as_bytes(), + "should restore original payloads and interstitial bytes on overflow" ); let (mut processor, _) = processor_with_payloads(&[body], 1000); processor.transform = Some(crate::integrations::google_tag_manager::rewrite_gtm_rsc_span); @@ -1215,7 +1224,8 @@ mod tests { .process_chunk(&[], true) .expect_err("should reject missing captured placeholder") .to_string() - .contains("not present") + .contains("not present"), + "should report a captured placeholder missing from parser output" ); } diff --git a/crates/trusted-server-core/src/integrations/nextjs/script_lexer.rs b/crates/trusted-server-core/src/integrations/nextjs/script_lexer.rs index a4d3822cd6..b2ee7650ea 100644 --- a/crates/trusted-server-core/src/integrations/nextjs/script_lexer.rs +++ b/crates/trusted-server-core/src/integrations/nextjs/script_lexer.rs @@ -3,7 +3,9 @@ //! This is not a JavaScript parser. It masks strings, comments and template text //! without changing byte offsets, and exposes code inside template expressions. -const MAX_TEMPLATE_DEPTH: usize = 32; +const MAX_LEXICAL_DEPTH: usize = 32; +// Retain only short keyword candidates; longer words cannot guide token context. +const WORD_BYTE_THRESHOLD: usize = 10; #[derive(Clone, Debug, Default)] enum Mode { @@ -52,11 +54,15 @@ struct PendingClass { brace_depth: usize, } +/// Resumable bounded heuristics, not a JavaScript parser. Excessive nesting +/// becomes opaque and masks the remaining source instead of guessing call heads. #[derive(Clone, Debug)] pub(super) struct ScriptLexer { mode: Mode, template_braces: Vec, + /// The next slash may begin a regexp literal rather than division. expression_start: bool, + /// The next brace may open a statement block rather than an object. statement_start: bool, braces: Vec, word: String, @@ -127,7 +133,7 @@ impl ScriptLexer { self.body_pending = None; } self.operator = None; - if self.word.len() < 10 && !self.word_overflow { + if self.word.len() < WORD_BYTE_THRESHOLD && !self.word_overflow { self.word.push(character); } else { self.word_overflow = true; @@ -168,7 +174,7 @@ impl ScriptLexer { }; if self.word == "function" { self.function_pending = Some(body); - } else if self.classes.len() == MAX_TEMPLATE_DEPTH { + } else if self.classes.len() == MAX_LEXICAL_DEPTH { self.mode = Mode::Opaque; return false; } else { @@ -197,7 +203,7 @@ impl ScriptLexer { } match character { '(' => { - if self.parentheses.len() == MAX_TEMPLATE_DEPTH { + if self.parentheses.len() == MAX_LEXICAL_DEPTH { self.mode = Mode::Opaque; } else { let kind = if self.control_pending { @@ -258,7 +264,7 @@ impl ScriptLexer { } '`' => { self.statement_start = false; - if self.template_braces.len() == MAX_TEMPLATE_DEPTH { + if self.template_braces.len() == MAX_LEXICAL_DEPTH { self.mode = Mode::Opaque; } else { self.template_braces.push(0); @@ -286,7 +292,7 @@ impl ScriptLexer { } else { BraceKind::Object }; - if self.braces.len() == MAX_TEMPLATE_DEPTH { + if self.braces.len() == MAX_LEXICAL_DEPTH { self.mode = Mode::Opaque; } else { self.braces.push(kind); @@ -427,7 +433,11 @@ mod tests { let source = r#"'é self.__next_f.push('/* self.__next_f.push( */`😀 ${ {nested: `self.__next_f.push(`} } ${self.__next_f.push([1,"data"])} `"#; let expected = ScriptLexer::default().mask(source); assert_eq!(expected.len(), source.len(), "should preserve byte offsets"); - assert_eq!(expected.matches("self.__next_f.push(").count(), 1); + assert_eq!( + expected.matches("self.__next_f.push(").count(), + 1, + "should expose exactly one executable Flight head" + ); for split in (0..=source.len()).filter(|split| source.is_char_boundary(*split)) { let mut lexer = ScriptLexer::default(); let actual = format!( @@ -487,27 +497,48 @@ mod tests { #[test] fn bounds_nested_braces() { let mut lexer = ScriptLexer::default(); - lexer.mask(&"{".repeat(MAX_TEMPLATE_DEPTH + 1)); + lexer.mask(&"{".repeat(MAX_LEXICAL_DEPTH + 1)); assert!(lexer.is_opaque(), "should refuse excessive brace nesting"); - assert_eq!(lexer.braces.len(), MAX_TEMPLATE_DEPTH); - assert!(!lexer.mask("self.__next_f.push(").contains("__next_f")); + assert_eq!( + lexer.braces.len(), + MAX_LEXICAL_DEPTH, + "should bound the brace stack at the lexical depth limit" + ); + assert!( + !lexer.mask("self.__next_f.push(").contains("__next_f"), + "should mask call heads after the lexical nesting cap is exceeded" + ); } #[test] fn bounds_pending_class_heads() { let mut lexer = ScriptLexer::default(); - lexer.mask(&"class extends (".repeat(MAX_TEMPLATE_DEPTH + 1)); + lexer.mask(&"class extends (".repeat(MAX_LEXICAL_DEPTH + 1)); assert!(lexer.is_opaque(), "should refuse excessive class nesting"); - assert_eq!(lexer.classes.len(), MAX_TEMPLATE_DEPTH); - assert!(!lexer.mask("self.__next_f.push(").contains("__next_f")); + assert_eq!( + lexer.classes.len(), + MAX_LEXICAL_DEPTH, + "should bound pending classes at the lexical depth limit" + ); + assert!( + !lexer.mask("self.__next_f.push(").contains("__next_f"), + "should mask call heads after the lexical nesting cap is exceeded" + ); } #[test] fn bounds_nested_templates() { let mut lexer = ScriptLexer::default(); - lexer.mask(&"`${".repeat(MAX_TEMPLATE_DEPTH + 1)); + lexer.mask(&"`${".repeat(MAX_LEXICAL_DEPTH + 1)); assert!(lexer.is_opaque(), "should refuse excessive nesting"); - assert_eq!(lexer.template_braces.len(), MAX_TEMPLATE_DEPTH); - assert!(!lexer.mask("self.__next_f.push(").contains("__next_f")); + assert_eq!( + lexer.template_braces.len(), + MAX_LEXICAL_DEPTH, + "should bound the template stack at the lexical depth limit" + ); + assert!( + !lexer.mask("self.__next_f.push(").contains("__next_f"), + "should mask call heads after the lexical nesting cap is exceeded" + ); } } diff --git a/crates/trusted-server-core/src/publisher.rs b/crates/trusted-server-core/src/publisher.rs index 95b615b4e1..b6398deb1c 100644 --- a/crates/trusted-server-core/src/publisher.rs +++ b/crates/trusted-server-core/src/publisher.rs @@ -6982,7 +6982,6 @@ pub async fn handle_page_bids( Vec::new() }; - let request_info = crate::http_util::RequestInfo::from_request(&req, services.client_info())?; // Owned so the identity-graph snapshot can be stored back on `ec_context` // below without holding a borrow of it across the mutation. let ec_id = ec_context diff --git a/crates/trusted-server-core/src/test_support.rs b/crates/trusted-server-core/src/test_support.rs index 44889501fd..fc02c16da4 100644 --- a/crates/trusted-server-core/src/test_support.rs +++ b/crates/trusted-server-core/src/test_support.rs @@ -298,12 +298,22 @@ pub mod nextjs_auction { "should rewrite origin" ); assert_eq!( - data["tag"], - "/integrations/google_tag_manager/gtm.js?q=\"x\"&v=é" + data["tag"], "/integrations/google_tag_manager/gtm.js?q=\"x\"&v=é", + "should preserve query quotes and Unicode while routing the GTM tag" + ); + assert_eq!( + data["ga"], "/integrations/google_tag_manager/g/collect?v=2", + "should route the GA collect endpoint" + ); + assert_eq!( + data["text"], "é😀", + "should preserve Unicode and literal closing-tag text" + ); + assert_eq!( + data["padding"], + "_".repeat(11000), + "should preserve all model padding" ); - assert_eq!(data["ga"], "/integrations/google_tag_manager/g/collect?v=2"); - assert_eq!(data["text"], "é😀"); - assert_eq!(data["padding"], "_".repeat(11000)); } /// Upstream that serves [`origin_html`] and one deterministic bid. From fe456ee2cf0e1e54fed41b0c4ffd779915dafbfe Mon Sep 17 00:00:00 2001 From: ChristianPavilonis Date: Mon, 19 Oct 2026 16:05:44 -0500 Subject: [PATCH 12/15] =?UTF-8?q?Prepared=20scoped,=20uncommitted=20change?= =?UTF-8?q?s=20for=20findings=200=E2=80=936:=20foreign-reference=20capture?= =?UTF-8?q?=20fix,=20conservative=20fallback=20regressions,=20once-only=20?= =?UTF-8?q?warnings,=20span-site=20tests,=20documentation/changelog=20upda?= =?UTF-8?q?tes,=20settings=20rollback,=20and=20routine=20cleanup.=20No=20p?= =?UTF-8?q?ublic?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CHANGELOG.md | 2 + .../tests/routes.rs | 4 - .../src/integrations/google_tag_manager.rs | 4 + .../src/integrations/nextjs/mod.rs | 194 +++++++++++++++++- .../integrations/nextjs/rsc_placeholders.rs | 156 ++++++++++++-- crates/trusted-server-core/src/settings.rs | 30 +-- docs/guide/creative-processing.md | 13 +- docs/guide/integrations/google_tag_manager.md | 24 +++ docs/guide/ts-dev-proxy.md | 19 +- 9 files changed, 381 insertions(+), 65 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7a0eb1ecd..3813f054c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- Next.js and Google Tag Manager script rewrites now compose in streamed HTML instead of overwriting each other, preserving Pages Router JSON and App Router Flight payloads with recomputed decoded `T`-record lengths. GTM/GA Flight rewrites require both integrations to be enabled. Buffering is bounded and per document; an uncaptured data head (including inert text quoting a Flight push) or binary push keeps the remaining Flight stream raw and restores queued originals, but cannot undo already-streamed rewrites. Complete serialized form state does not disable later rewriting, foreign property references no longer hide a later qualified Flight push, and placeholder-stage document fallback emits a reason-bearing warning without source data. +- Publisher and page-bids processing now reject malformed effective request hosts with 400 before inserting request-derived bytes into rewrite output. Host selection remains `Forwarded` (`host=`), then `X-Forwarded-Host`, then `Host`; a malformed or explicitly empty selected host is rejected without fallback. Valid local hostnames, underscore labels, ports, and bracketed IPv6 remain supported. Forwarded-header trust policy is unchanged. - Image Optimizer configuration now rejects profile and profile-set names within the same map that collide after trimming, even when their values match, instead of silently overwriting an entry. Empty names continue to be discarded. - Browser integration bundles now share context providers and the configured log level with TSJS core (#1196). With Permutive enabled and cohorts available, `tsjs.requestAds()` now sends `config.permutive_segments` to `/auction`; the server retains that key only when `auction.allowed_context_keys` includes `"permutive_segments"`. This restores a previously inactive data flow, not a consent gate: review publisher consent handling before relying on it. Integration logs now follow `tsjs.setConfig` and `tsjs.log.setLevel`, and creative no longer raises the default `warn` level to `info`. The `?tsdebug=1` query parameter or `localStorage['tsdebug'] = '1'` intentionally enables `debug` logging across all bundles when creative installs, overriding any log level set earlier through `tsjs.setConfig` or `tsjs.log.setLevel`. Later calls to either API can change the level again. - `[auction].allowed_context_keys` now serializes in sorted, deduplicated order, so ESI template-cache fingerprints and `ts config diff`/`push` envelope hashes are stable across loads. Template fingerprints also sort object keys independently of `serde_json/preserve_order`. Existing envelopes may show a one-time allowlist reorder after upgrading; push once to settle it. The updated fingerprint format causes one template-cache miss per cached page after deployment. diff --git a/crates/trusted-server-adapter-axum/tests/routes.rs b/crates/trusted-server-adapter-axum/tests/routes.rs index e16b107063..8e76879e98 100644 --- a/crates/trusted-server-adapter-axum/tests/routes.rs +++ b/crates/trusted-server-adapter-axum/tests/routes.rs @@ -880,10 +880,6 @@ async fn first_party_proxy_rebuild_is_routed() { /// Fastly adapter alone and not by this test. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn nextjs_auction_output_holds_until_the_structural_body_close() { - use std::sync::Arc; - - use trusted_server_core::test_support::nextjs_auction; - let client = Arc::new(nextjs_auction::NextJsAuctionOrigin::default()); let router = TrustedServerApp::routes_with_settings_and_services( nextjs_auction::settings(), diff --git a/crates/trusted-server-core/src/integrations/google_tag_manager.rs b/crates/trusted-server-core/src/integrations/google_tag_manager.rs index 4971729176..74a21a64fe 100644 --- a/crates/trusted-server-core/src/integrations/google_tag_manager.rs +++ b/crates/trusted-server-core/src/integrations/google_tag_manager.rs @@ -193,6 +193,8 @@ fn flight_url_token_end( /// recomputed. Originals used for fallback must never pass through this function. /// `record_text` marks exactly one complete T-record body. Other spans can start /// mid-URL at a payload-script boundary and must not be routed as whole URLs. +/// A record body may still span payloads: a boundary marker inside the routed +/// prefix prevents routing, and one in the suffix is copied verbatim. pub(crate) fn rewrite_gtm_rsc_span(content: &str, record_text: bool) -> String { fn route(token: &str) -> Option { let captures = GTM_RSC_WHOLE_URL_PATTERN.captures(token)?; @@ -1132,6 +1134,8 @@ impl IntegrationScriptRewriter for GoogleTagManagerIntegration { } fn rewrite(&self, content: &str, ctx: &IntegrationScriptContext<'_>) -> ScriptRewriteAction { + // Per document, never per registry: a registry-lifetime buffer would + // carry one document's partial script into the next. let state = ctx .document_state .get_or_insert_with(GTM_INTEGRATION_ID, || Mutex::new(GtmScriptState::default())); diff --git a/crates/trusted-server-core/src/integrations/nextjs/mod.rs b/crates/trusted-server-core/src/integrations/nextjs/mod.rs index dce3b0f96c..4de7662d3a 100644 --- a/crates/trusted-server-core/src/integrations/nextjs/mod.rs +++ b/crates/trusted-server-core/src/integrations/nextjs/mod.rs @@ -264,6 +264,115 @@ mod tests { String::from_utf8(decoded).expect("should emit UTF-8") } + #[test] + fn json_ld_data_head_preserves_later_flight_and_queued_originals() { + let data = json!({ + "href": "https://origin.example.com/page", + "tag": "http://www.googletagmanager.com/gtm.js?id=GTM-MIX1", + "text": "é😀", + }); + let body = data.to_string(); + let row = format!("1:T{:x},{body}", body.len()); + let flight = flight_html(&[&row]); + let hint = json!({"articleBody": "Example self.__next_f.push([1, is inert"}); + let html = format!("{flight}{flight}"); + let mut rewritten = data.clone(); + rewritten["href"] = json!("https://test.example.com/page"); + rewritten["tag"] = json!("/integrations/google_tag_manager/gtm.js?id=GTM-MIX1"); + for compression in [Compression::None, Compression::Gzip] { + for chunk_size in [16, 8192] { + let output = mixed_output_with_compression(&html, compression, chunk_size); + let payloads = flight_payloads(&output); + assert_eq!(payloads.len(), 2, "should retain both real Flight pushes"); + assert_eq!(payloads[1], row, "should leave every later payload raw"); + assert_mixed_t_model(&payloads[1], &data); + if chunk_size == 8192 { + assert_eq!(payloads[0], row, "should restore queued earlier payloads"); + assert_mixed_t_model(&payloads[0], &data); + } else { + assert_mixed_t_model(&payloads[0], &rewritten); + } + assert!( + output.contains(&hint.to_string()), + "should preserve JSON-LD source" + ); + assert!( + !output.contains("__ts_rsc_"), + "should resolve every placeholder" + ); + } + } + } + + #[test] + fn foreign_references_before_flight_preserve_capture_at_every_split() { + let data = json!({ + "href": "https://origin.example.com/page", + "tag": "http://www.googletagmanager.com/gtm.js?id=GTM-MIX1", + }); + let body = data.to_string(); + let row = format!("1:T{:x},{body}", body.len()); + let encoded = serde_json::to_string(&row).expect("should encode Flight row"); + let mut expected = data; + expected["href"] = json!("https://test.example.com/page"); + expected["tag"] = json!("/integrations/google_tag_manager/gtm.js?id=GTM-MIX1"); + for prefix in [ + "window.app.__next_f=1;", + "foreign.__next_f=1;obj.window.__next_f=2;", + "const label='é😀';window.app.__next_f=1;", + ] { + for receiver in [ + "self.__next_f", + "window.__next_f", + "(self.__next_f=self.__next_f||[])", + ] { + let source = format!("{prefix}{receiver}.push([1,{encoded}]);"); + let later = flight_html(&[&row]); + for split in (0..=source.len()).filter(|split| source.is_char_boundary(*split)) { + let mut processor = create_html_processor(mixed_config(10000, 10000)); + let first = format!("{later}", &source[split..]); + let mut output = processor + .process_chunk(first.as_bytes(), false) + .expect("should accept foreign references before Flight"); + output.extend( + processor + .process_chunk(last.as_bytes(), true) + .expect("should complete qualified Flight after foreign references"), + ); + let output = String::from_utf8(output).expect("should emit UTF-8"); + let payloads = flight_payloads(&output); + assert_eq!(payloads.len(), 2, "should retain current and later Flight"); + for payload in payloads { + assert_mixed_t_model(&payload, &expected); + } + assert!( + output.starts_with(&format!("{later}"); + for chunk_size in [16, 8192] { + let output = + mixed_output_with_compression(&html, Compression::Gzip, chunk_size); + let payloads = flight_payloads(&output); + assert_eq!(payloads.len(), 2, "should retain both gzip Flight pushes"); + for payload in payloads { + assert_mixed_t_model(&payload, &expected); + } + assert!( + !output.contains("__ts_rsc_"), + "should resolve every gzip placeholder" + ); + } + } + } + } + #[test] fn non_code_flight_hints_preserve_source_and_data_heads_bypass_later_flight() { let payload = "0:{\"url\":\"https://origin.example.com/page\"}\n"; @@ -665,8 +774,13 @@ mod tests { ); } } - // A qualified head can be observed but not buffered when an earlier - // unrelated property makes the tentative receiver claim fail. + // An earlier foreign property no longer prevents the following qualified + // push from being buffered. With no lexer trap, all three rows are safe + // to capture and rewrite, even when the first T body arrives later. + let routed_body = "`/integrations/google_tag_manager/gtm.js`"; + let routed_payload = format!("1:T{:x},{routed_body}", routed_body.len()); + let rewritten_body = json!({"url": "https://test.example.com/page", "tag": "/integrations/google_tag_manager/gtm.js?id=GTM-MIX1"}).to_string(); + let rewritten_payload = format!("1:T{:x},{rewritten_body}", rewritten_body.len()); let body_start = missed_push.find('`').expect("should locate raw T body"); let first = format!( " ``` +#### Next.js App Router Flight data + +With both `google_tag_manager` and `nextjs` enabled, Trusted Server also rewrites +supported GTM/Google Analytics URLs inside captured App Router Flight payloads. +Next.js origin-host rewrites and GTM rewrites compose before the decoded byte +length of each `T` record is recomputed. This preserves the payload's record +boundaries rather than applying ordinary script substitutions to raw Flight. +Complete serialized channel-2 form state is preserved without disabling later +Flight rewrites. + +Buffering is bounded and per document. Unsafe or unsupported payloads retain their +original bytes. An uncaptured channel-1 data head or binary channel-3 push disables +Flight rewriting for the rest of the document, including restoration of queued +payloads; already-streamed complete groups cannot be restored. The conservative +raw-head guard also applies to inert `self.__next_f.push([1,` text in strings, +comments, or JSON-LD, so pages quoting Flight source may lose later origin/GTM +rewrites. A warning identifies the first placeholder-stage document fallback +without including the source text. + +Flight rewriting operates on supported quoted URL tokens and complete-record +URLs, including bare origins; it is not limited to script attributes. This can +also change rendered content and differ from SSR HTML rewriting. Restricting this +policy and optimizing its inline-script cost are separate follow-ups. + ### 2. Script Proxying The proxy intercepts requests for the GTM library and modifies it on-the-fly. This is critical for First-Party context. diff --git a/docs/guide/ts-dev-proxy.md b/docs/guide/ts-dev-proxy.md index ebd449bb9a..02f72e5c88 100644 --- a/docs/guide/ts-dev-proxy.md +++ b/docs/guide/ts-dev-proxy.md @@ -287,12 +287,23 @@ certificate is imported until the subject check succeeds. ## Host header behavior The proxy always sends `X-Forwarded-Host: ` (the production hostname) — the -standard "original host" header for a forward proxy. Trusted Server core anchors -all HTML/URL rewriting to it (it prefers `X-Forwarded-Host`, then `Host`), so +standard "original host" header for a forward proxy. Trusted Server core selects +its effective host from `Forwarded`'s `host=` parameter, then `X-Forwarded-Host`, +then `Host`. Without a higher-priority `Forwarded` host, **first-party URLs stay on the production domain regardless of the `Host` header — as long as the upstream preserves `X-Forwarded-Host`**. That decouples routing -(`Host`) from the first-party host. (Real Trusted Server adapters strip inbound -`X-Forwarded-Host`; the caveat below covers what that means with `--rewrite-host`.) +(`Host`) from the first-party host. (Fastly and Spin strip inbound forwarded +headers; the caveat below covers what that means with `--rewrite-host`.) + +The selected host is syntax-validated before publisher processing, including +requests for assets that never reach a script rewriter. A malformed or explicitly +empty selected host returns **400**, without falling back to a lower-priority +header or reflecting the rejected value. A non-text host header, including a +non-text `Forwarded` value, is also rejected. Requests with no host metadata retain +the existing empty-host behavior. Valid ports, bracketed IPv6, local hostnames, +and ASCII underscores in host labels are retained. This validation does not +establish whether a forwarded header came from a trusted proxy; configure that +trust boundary in the adapter/deployment. By default `Host: ` too. Fastly routes by SNI (`= TO`) and passes `Host` through unchanged, so `Host: ` reaches the upstream — but the upstream still From 380909da4a1c14c4a1c80d65f3d5b76d60fd8ec9 Mon Sep 17 00:00:00 2001 From: ChristianPavilonis Date: Thu, 22 Oct 2026 19:11:06 -0500 Subject: [PATCH 13/15] Deduplicate Flight fallback warnings, enable WASM regression coverage, clarify fallback/security documentation, and apply accepted routine corrections. --- CHANGELOG.md | 4 +- crates/trusted-server-core/src/http_util.rs | 6 +- .../src/integrations/nextjs/mod.rs | 4 +- .../integrations/nextjs/rsc_placeholders.rs | 105 +++++++++++++++++- .../src/integrations/nextjs/rsc_stream.rs | 8 +- docs/guide/integrations/google_tag_manager.md | 19 ++-- docs/guide/integrations/nextjs.md | 4 + 7 files changed, 130 insertions(+), 20 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dc0f2abecd..7d76b36428 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,14 +28,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Security +- Publisher and page-bids processing now reject malformed effective request hosts with 400 before inserting request-derived bytes into rewrite output. `/first-party/sign` also returns 400 for a malformed effective host when the request URI has no scheme and request metadata is needed to determine it. Host selection remains `Forwarded` (`host=`), then `X-Forwarded-Host`, then `Host`; a malformed or explicitly empty selected host is rejected without fallback. Valid local hostnames, underscore labels, ports, and bracketed IPv6 remain supported. Forwarded-header trust policy is unchanged (#1206, #1207, #1208). - `/first-party/sign` now rejects valid targets outside `proxy.allowed_domains` before minting a proxy token. The creative runtime keeps image and iframe assignments blocked after this `403` policy response instead of loading the rejected URL directly; fetch-time checks still cover the initial target and every redirect. - Reserved the complete admin namespace at the publisher-fallback boundary. Percent-encoded separators (`/_ts/admin%2Fec`, `%2f`, and double-encoded forms) matched the `^/_ts/admin` Basic-auth handler but escaped the literal-slash namespace check, so an authenticated request fell through to publisher fallback and forwarded its `Authorization` header and body to the publisher origin. The reservation now spans the whole `/_ts/admin` prefix plus the retired `/admin/keys` aliases — including trailing, descendant, and encoded-separator forms — evaluated on the raw path and on each of its bounded percent-decodings, so multi-encoded separators such as `/admin%252Fkeys/rotate` cannot survive to fallback for a proxy or origin to decode again, and applies to every adapter. - Validate synthetic ID format on inbound values from the `x-synthetic-id` header and `synthetic_id` cookie; values that do not match the expected format (`64-hex-hmac.6-alphanumeric-suffix`) are discarded and a fresh ID is generated rather than forwarded to response headers, cookies, or third-party APIs ### Fixed -- Next.js and Google Tag Manager script rewrites now compose in streamed HTML instead of overwriting each other, preserving Pages Router JSON and App Router Flight payloads with recomputed decoded `T`-record lengths. GTM/GA Flight rewrites require both integrations to be enabled. Buffering is bounded and per document; an uncaptured data head (including inert text quoting a Flight push) or binary push keeps the remaining Flight stream raw and restores queued originals, but cannot undo already-streamed rewrites. Complete serialized form state does not disable later rewriting, foreign property references no longer hide a later qualified Flight push, and placeholder-stage document fallback emits a reason-bearing warning without source data. -- Publisher and page-bids processing now reject malformed effective request hosts with 400 before inserting request-derived bytes into rewrite output. Host selection remains `Forwarded` (`host=`), then `X-Forwarded-Host`, then `Host`; a malformed or explicitly empty selected host is rejected without fallback. Valid local hostnames, underscore labels, ports, and bracketed IPv6 remain supported. Forwarded-header trust policy is unchanged. +- Next.js and Google Tag Manager script rewrites now compose in streamed HTML instead of overwriting each other, preserving Pages Router JSON and App Router Flight payloads with recomputed decoded `T`-record lengths. GTM/GA Flight rewrites require both integrations to be enabled. Buffering is bounded and per document; an uncaptured data head (including inert text quoting a Flight push), an unsupported or binary push, or a size limit keeps the remaining Flight stream raw and restores queued originals, but cannot undo already-streamed rewrites. Complete serialized form state does not disable later rewriting, foreign property references no longer hide a later qualified Flight push, and placeholder-stage document fallback emits a reason-bearing warning without source data (#1206, #1207, #1208). - Publisher responses now map origin URLs in `Location`, `Content-Location`, `Refresh`, and `Link` headers to the serving host, and each CSP source naming the origin gains a serving-host source beside it (#1128). A redirect to the current URL under a different scheme, or a `GET`/`HEAD` redirect to the current URL that sets no cookie (or any such redirect under a `Host` override), is left unchanged to avoid a loop. `` refresh and CSP tags in the page body are not rewritten. See "Origin URLs in proxied response headers" in the configuration guide. - Image Optimizer configuration now rejects profile and profile-set names within the same map that collide after trimming, even when their values match, instead of silently overwriting an entry. Empty names continue to be discarded. - Browser integration bundles now share context providers and the configured log level with TSJS core (#1196). With Permutive enabled and cohorts available, `tsjs.requestAds()` now sends `config.permutive_segments` to `/auction`; the server retains that key only when `auction.allowed_context_keys` includes `"permutive_segments"`. This restores a previously inactive data flow, not a consent gate: review publisher consent handling before relying on it. Integration logs now follow `tsjs.setConfig` and `tsjs.log.setLevel`, and creative no longer raises the default `warn` level to `info`. The `?tsdebug=1` query parameter or `localStorage['tsdebug'] = '1'` intentionally enables `debug` logging across all bundles when creative installs, overriding any log level set earlier through `tsjs.setConfig` or `tsjs.log.setLevel`. Later calls to either API can change the level again. diff --git a/crates/trusted-server-core/src/http_util.rs b/crates/trusted-server-core/src/http_util.rs index 52eb1abb8b..2cde9b3bc5 100644 --- a/crates/trusted-server-core/src/http_util.rs +++ b/crates/trusted-server-core/src/http_util.rs @@ -149,9 +149,9 @@ impl RequestInfo { /// 4. `Fastly-SSL` /// 5. Default `http` /// - /// Fastly and Spin strip forwarded headers with [`sanitize_forwarded_headers`] - /// before routing. On those paths, `Host` and [`ClientInfo`] TLS detection - /// are the effective sources. Other adapters retain their existing policy. + /// Fastly and Spin strip [`SPOOFABLE_FORWARDED_HEADERS`] before routing. + /// On those paths, `Host` and [`ClientInfo`] TLS detection are the + /// effective sources. Other adapters retain their existing policy. /// /// The selected host must be an ASCII hostname (including underscores for /// local service names), IPv4 address, or bracketed IPv6 address, optionally diff --git a/crates/trusted-server-core/src/integrations/nextjs/mod.rs b/crates/trusted-server-core/src/integrations/nextjs/mod.rs index 4de7662d3a..ea5fb08ad4 100644 --- a/crates/trusted-server-core/src/integrations/nextjs/mod.rs +++ b/crates/trusted-server-core/src/integrations/nextjs/mod.rs @@ -791,11 +791,11 @@ mod tests { let mut processor = create_html_processor(mixed_config(10000, 10000)); let mut bytes = processor .process_chunk(first.as_bytes(), false) - .expect("should release qualified but unbuffered Flight"); + .expect("should accept a foreign reference before qualified Flight"); bytes.extend( processor .process_chunk(&continuation.as_bytes()[..split], false) - .expect("should preserve unbuffered continuation"), + .expect("should buffer the qualified continuation"), ); let last = format!("{}", &continuation[split..]); bytes.extend( diff --git a/crates/trusted-server-core/src/integrations/nextjs/rsc_placeholders.rs b/crates/trusted-server-core/src/integrations/nextjs/rsc_placeholders.rs index eeaaf29fb0..20c04b2282 100644 --- a/crates/trusted-server-core/src/integrations/nextjs/rsc_placeholders.rs +++ b/crates/trusted-server-core/src/integrations/nextjs/rsc_placeholders.rs @@ -803,10 +803,15 @@ mod tests { }) } - #[cfg(not(target_arch = "wasm32"))] mod fallback_logging { use super::*; + use crate::html_processor::{HtmlProcessorConfig, create_html_processor}; + use crate::integrations::IntegrationRegistry; + use crate::streaming_processor::{Compression, PipelineConfig, StreamingPipeline}; + use crate::test_support::nextjs_auction; + use serde_json::json; use std::cell::RefCell; + use std::io::Cursor; struct FallbackLogger; @@ -822,8 +827,11 @@ mod tests { fn log(&self, record: &log::Record<'_>) { if self.enabled(record.metadata()) - && record.target() - == "trusted_server_core::integrations::nextjs::rsc_placeholders" + && matches!( + record.target(), + "trusted_server_core::integrations::nextjs::rsc_placeholders" + | "trusted_server_core::integrations::nextjs::rsc_stream" + ) { MESSAGES.with(|messages| { if let Some(messages) = messages.borrow_mut().as_mut() { @@ -858,12 +866,101 @@ mod tests { let messages = MESSAGES .with(|messages| messages.borrow_mut().take()) .expect("should capture this thread's fallback warning"); - log::set_max_level(previous_level); assert_eq!( messages, ["Next.js RSC fallback: uncaptured data head; remaining Flight payloads stay raw"], "should log only the first transition and its fixed reason, never source text" ); + + let complete = "1:T3,abc"; + let article = json!({"articleBody": "self.__next_f.push([1, publisher-secret"}); + for (first, between, last, warning) in [ + ( + complete, + format!(""), + complete, + "Next.js RSC fallback: uncaptured data head; remaining Flight payloads stay raw", + ), + ( + complete, + format!( + "", + json!([3, "AAECAw=="]) + ), + complete, + "Next.js RSC fallback: unsupported qualified push; remaining Flight payloads stay raw", + ), + ( + "1:Tffff,abc", + format!("

{}

", "x".repeat(1024)), + complete, + "Next.js RSC fallback: capture or output limit, 1 held payloads", + ), + ( + "1:T1,é", + String::new(), + complete, + "Next.js RSC fallback: invalid group, 1 payloads", + ), + ] { + for chunk_size in [16, 8192] { + MESSAGES.with(|messages| *messages.borrow_mut() = Some(Vec::new())); + let mut settings = nextjs_auction::script_composition_settings(); + settings + .integrations + .insert_config( + "nextjs", + &json!({"enabled": true, "max_combined_payload_bytes": 512}), + ) + .expect("should configure the fallback limit"); + let registry = IntegrationRegistry::with_plan( + &settings, + Arc::new( + crate::auction::compile_auction_plan(&settings) + .expect("should compile the fallback fixture plan"), + ), + ) + .expect("should build the fallback fixture registry"); + let config = HtmlProcessorConfig::from_settings( + &settings, + ®istry, + nextjs_auction::ORIGIN_HOST, + nextjs_auction::PUBLISHER_HOST, + "https", + ); + let html = format!( + "{between}", + json!([1, first]), + json!([1, last]), + ); + let mut pipeline = StreamingPipeline::new( + PipelineConfig { + input_compression: Compression::None, + output_compression: Compression::None, + chunk_size, + }, + create_html_processor(config), + ); + let mut output = Vec::new(); + pipeline + .process(Cursor::new(html.as_bytes()), &mut output) + .expect("should restore the complete fallback response"); + assert_eq!( + output, + html.as_bytes(), + "should preserve original payloads and interstitial bytes at chunk {chunk_size}" + ); + let messages = MESSAGES + .with(|messages| messages.borrow_mut().take()) + .expect("should capture both fallback stages on this thread"); + assert_eq!( + messages, + [warning], + "should warn once with the actual fallback reason at chunk {chunk_size}" + ); + } + } + log::set_max_level(previous_level); } } diff --git a/crates/trusted-server-core/src/integrations/nextjs/rsc_stream.rs b/crates/trusted-server-core/src/integrations/nextjs/rsc_stream.rs index bc70cf49bc..e4052a1d04 100644 --- a/crates/trusted-server-core/src/integrations/nextjs/rsc_stream.rs +++ b/crates/trusted-server-core/src/integrations/nextjs/rsc_stream.rs @@ -420,7 +420,13 @@ impl NextJsRscStreamProcessor { /// in `current`. A queued payload whose placeholder had not yet been emitted /// would fail substitution rather than degrade to unchanged bytes. fn release_bypass(&mut self, current: &[u8]) -> io::Result> { - if !self.group.is_empty() || self.next_captured().is_some() { + // The rewriter and resolve_group warn when they enter fallback themselves. + let entering = !self + .state + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .bypass_rsc; + if entering && (!self.group.is_empty() || self.next_captured().is_some()) { log::warn!( "Next.js RSC fallback: capture or output limit, {} held payloads", self.group.len() diff --git a/docs/guide/integrations/google_tag_manager.md b/docs/guide/integrations/google_tag_manager.md index 8cbbbe1a24..fcada96646 100644 --- a/docs/guide/integrations/google_tag_manager.md +++ b/docs/guide/integrations/google_tag_manager.md @@ -153,14 +153,17 @@ boundaries rather than applying ordinary script substitutions to raw Flight. Complete serialized channel-2 form state is preserved without disabling later Flight rewrites. -Buffering is bounded and per document. Unsafe or unsupported payloads retain their -original bytes. An uncaptured channel-1 data head or binary channel-3 push disables -Flight rewriting for the rest of the document, including restoration of queued -payloads; already-streamed complete groups cannot be restored. The conservative -raw-head guard also applies to inert `self.__next_f.push([1,` text in strings, -comments, or JSON-LD, so pages quoting Flight source may lose later origin/GTM -rewrites. A warning identifies the first placeholder-stage document fallback -without including the source text. +Buffering is bounded and per document. Payloads that cannot be captured or +rewritten safely keep their original bytes, and some of these cases turn off +Flight rewriting for the rest of the document: an uncaptured channel-1 data +head, a binary channel-3 or other unsupported push, or a payload, script, or +held-output size limit. Queued payloads that have not yet streamed are then +restored to their original bytes, but complete groups that already streamed +cannot be restored. The conservative raw-head guard also applies to inert +`self.__next_f.push([1,` text in strings, comments, or JSON-LD, so pages quoting +Flight source may lose later origin/GTM rewrites, even with only `nextjs` +enabled. The first placeholder-stage fallback logs a warning that names the +reason without including source text. Flight rewriting operates on supported quoted URL tokens and complete-record URLs, including bare origins; it is not limited to script attributes. This can diff --git a/docs/guide/integrations/nextjs.md b/docs/guide/integrations/nextjs.md index 7c6cc7c509..847879d6d9 100644 --- a/docs/guide/integrations/nextjs.md +++ b/docs/guide/integrations/nextjs.md @@ -115,6 +115,10 @@ Targets the Next.js data script for rewriting. - Retains only an unresolved cross-script `T` chunk group - Rewrites URLs and recalculates `T` chunk byte lengths before releasing a complete group - Restores invalid, incomplete, or over-limit groups unchanged +- Some unsafe cases disable Flight rewriting for the rest of the document: an uncaptured channel-1 data head, a binary channel-3 or other unsupported push, or a payload, script, or held-output size limit +- Document-wide fallback restores queued payloads that have not yet streamed to their original bytes; already-streamed complete groups cannot be restored +- The conservative raw-head guard also applies to inert `self.__next_f.push([1,` text in strings, comments, or JSON-LD, so later origin rewrites can be lost even with only `nextjs` enabled +- The first placeholder-stage fallback logs a reason-bearing warning without including source text - Applies `max_combined_payload_bytes` independently to captured payloads and held output - Preserves script order and React hydration data From bb5b551ae18f4341d06d57aa20c676bae6230f49 Mon Sep 17 00:00:00 2001 From: Christian Date: Fri, 23 Oct 2026 06:50:08 -0500 Subject: [PATCH 14/15] Remove unused script text accumulator --- .../src/integrations/mod.rs | 1 - .../src/integrations/registry.rs | 34 +------------------ 2 files changed, 1 insertion(+), 34 deletions(-) diff --git a/crates/trusted-server-core/src/integrations/mod.rs b/crates/trusted-server-core/src/integrations/mod.rs index 00f0a8a02c..2a48e79d47 100644 --- a/crates/trusted-server-core/src/integrations/mod.rs +++ b/crates/trusted-server-core/src/integrations/mod.rs @@ -37,7 +37,6 @@ pub use registry::{ IntegrationRequestFilter, IntegrationScriptContext, IntegrationScriptRewriter, ProxyDispatchInput, RequestFilterDecision, RequestFilterEffects, RequestFilterInput, RequestFilterRegistryInput, RequestFilterRegistryOutcome, ScriptRewriteAction, - ScriptTextAccumulator, }; /// Registers or retrieves a platform backend for the given URL. diff --git a/crates/trusted-server-core/src/integrations/registry.rs b/crates/trusted-server-core/src/integrations/registry.rs index 67bc5f14ec..e9001afec0 100644 --- a/crates/trusted-server-core/src/integrations/registry.rs +++ b/crates/trusted-server-core/src/integrations/registry.rs @@ -1,6 +1,6 @@ use std::any::{Any, TypeId}; use std::collections::BTreeMap; -use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; +use std::sync::{Arc, Mutex}; use async_trait::async_trait; use edgezero_core::body::Body as EdgeBody; @@ -199,38 +199,6 @@ impl IntegrationDocumentState { } } -/// Per-document buffer for script text fragments split across chunks. -/// -/// `lol_html` can deliver one text node as several chunks, so a rewriter that -/// needs the whole script must accumulate until `is_last_in_text_node`. -/// -/// This lives in [`IntegrationDocumentState`] rather than on the rewriter. -/// Rewriters are registered once as `Arc` and -/// live as long as the [`IntegrationRegistry`], so a buffer owned by a -/// rewriter is shared by every document that registry serves. A document whose -/// stream ends before the final fragment — client disconnect, origin error, -/// truncated body — leaves its partial script in that buffer, and the next -/// document prepends the residue to its own accumulation. That corrupts the -/// response and can disclose the previous document's content. -/// -/// Keyed per integration id, so each integration gets its own buffer, and -/// dropped with the document state at end of document. -#[derive(Debug, Default)] -pub struct ScriptTextAccumulator { - buffer: Mutex, -} - -impl ScriptTextAccumulator { - /// Locks the buffer. - /// - /// Recovers from poisoning rather than panicking: a poisoned buffer holds - /// at worst a partial script, and the caller's `is_last_in_text_node` - /// handling already tolerates unexpected contents. - pub fn buffer(&self) -> MutexGuard<'_, String> { - self.buffer.lock().unwrap_or_else(PoisonError::into_inner) - } -} - /// Describes an HTTP endpoint exposed by an integration. #[derive(Clone, Debug)] pub struct IntegrationEndpoint { From db3d6c0d75efd732142b9be29f442f2642f1bc9b Mon Sep 17 00:00:00 2001 From: ChristianPavilonis Date: Mon, 26 Oct 2026 12:12:39 -0500 Subject: [PATCH 15/15] Document the nested-SVG script selector-state limitation without changing behavior. --- crates/trusted-server-core/src/html_processor.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/trusted-server-core/src/html_processor.rs b/crates/trusted-server-core/src/html_processor.rs index 4b534409fe..8523c3150d 100644 --- a/crates/trusted-server-core/src/html_processor.rs +++ b/crates/trusted-server-core/src/html_processor.rs @@ -700,6 +700,8 @@ pub(crate) fn create_html_processor_with_document_state( // Selector callbacks retain lol_html's CSS matching; only the composed text // callback mutates a fragment. Start each script with fresh match state. + // Match flags are flat: nested SVG scripts do not restore an outer script's + // flags when the inner script ends. // Without script rewriters, skip script text capture entirely. if !script_rewriters.is_empty() { let script_matches = Rc::new(RefCell::new(vec![false; script_rewriters.len()]));