diff --git a/docs/docs/guides/transit-engines.md b/docs/docs/guides/transit-engines.md index 88034fadb..e87293e12 100644 --- a/docs/docs/guides/transit-engines.md +++ b/docs/docs/guides/transit-engines.md @@ -192,6 +192,13 @@ source dependencies. Inspect the generated configuration and admin status when checking a sovereignty requirement; application checks do not replace firewall or DNS policy. +The feed proxy follows one redirect per realtime feed: a relative one or one to +the feed's own host, or one to another host whose name looks public, with no +port and without dropping from https to http. It answers any other redirect, and +any second redirect, with 502, which MOTIS treats as a failed fetch. Hostname +checks cannot tell where a public name resolves, so keep the egress policy in +place. + `MOTIS_ROUTE_SHAPES=missing` synthesizes absent route geometry from OSM. `all` is more expensive. Both add preparation work and storage; measure candidate imports before enabling them. diff --git a/packages/motis-feed-proxy-config/__tests__/build.test.ts b/packages/motis-feed-proxy-config/__tests__/build.test.ts index 51fcbf520..af8466059 100644 --- a/packages/motis-feed-proxy-config/__tests__/build.test.ts +++ b/packages/motis-feed-proxy-config/__tests__/build.test.ts @@ -3,7 +3,11 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { renderFeedProxyNginxConfig as renderFeedProxyNginxConfigCli } from "../../cli/src/lib/motis-feed-proxy"; -import { buildFeedProxyConfig, renderFeedProxyNginxConfig } from "../src/build.js"; +import { + buildFeedProxyConfig, + FEED_REDIRECT_MAPS, + renderFeedProxyNginxConfig, +} from "../src/build.js"; describe("buildFeedProxyConfig", () => { let workDir: string; @@ -178,3 +182,171 @@ describe("buildFeedProxyConfig", () => { expect(written).not.toContain("bad2"); }); }); + +/** Evaluate the redirect maps the way nginx does: regex keys in order, first match wins. */ +function evaluateRedirect(location: string, feedUrl: string) { + const parsed = new URL(feedUrl); + const vars: Record = { + orig_loc: location, + original_scheme: parsed.protocol.slice(0, -1), + original_authority: parsed.host, + }; + for (const map of FEED_REDIRECT_MAPS) { + const source = map.source.replace(/\$([a-z_]+)/g, (_, name: string) => vars[name] ?? ""); + let value = map.defaultValue; + for (const rule of map.rules) { + const match = new RegExp(rule.pattern, rule.caseInsensitive ? "i" : "").exec(source); + if (!match) continue; + value = rule.value.replace(/\$(\d)/g, (_, index: string) => match[Number(index)] ?? ""); + break; + } + vars[map.variable] = value; + } + return { allowed: vars.feed_redirect_allowed === "1", url: vars.feed_redirect_url }; +} + +describe("feed redirect handling", () => { + const feed = "https://feeds.example.org/rt/trip-updates.pb"; + const config = renderFeedProxyNginxConfig({ + secret: { url: feed, headers: { Authorization: "Bearer abc" } }, + plain: { url: "http://feeds.example.org:8080/vehicles.pb" }, + }); + const namedLocation = config.slice( + config.indexOf("location @handle_redirects"), + config.indexOf("location @feed_redirect_refused"), + ); + + it("renders the redirect maps in the http context before the first server", () => { + for (const map of FEED_REDIRECT_MAPS) { + const index = config.indexOf(`$${map.variable} {`); + expect(index).toBeGreaterThan(0); + expect(index).toBeLessThan(config.indexOf("server {")); + for (const rule of map.rules) expect(rule.pattern).not.toContain('"'); + } + expect(config).toContain('"~*^([^|]+)\\|\\1$" "1";'); + }); + + it("records each feed's scheme and authority for relative redirects", () => { + expect(config).toContain('set $original_scheme "https";'); + expect(config).toContain('set $original_authority "feeds.example.org";'); + expect(config).toContain('set $original_scheme "http";'); + expect(config).toContain('set $original_authority "feeds.example.org:8080";'); + expect(config).not.toContain("$scheme://"); + }); + + it("guards the single proxy_pass and refuses a second redirect", () => { + expect(config).toContain("error_page 301 302 303 307 308 = @handle_redirects;"); + expect(config.indexOf("recursive_error_pages on;")).toBeLessThan( + config.indexOf("location @handle_redirects"), + ); + expect(namedLocation).toContain("error_page 301 302 303 307 308 = @feed_redirect_refused;"); + expect(namedLocation).toMatch( + /if \(\$feed_redirect_allowed = 0\) \{\n\s+return 502 "feed redirect target refused\\n";/, + ); + expect(namedLocation.match(/proxy_pass /g)).toEqual(["proxy_pass "]); + expect(namedLocation).toContain("proxy_pass $feed_redirect_url;"); + expect(config).not.toContain("proxy_pass $orig_loc"); + expect(config).toContain('return 502 "feed redirect chain refused\\n";'); + }); + + it("builds the upstream URL from validated parts only", () => { + const urlMap = FEED_REDIRECT_MAPS.find((map) => map.variable === "feed_redirect_url"); + expect(urlMap?.source).not.toContain("$orig_loc"); + for (const name of urlMap?.source.match(/\$[a-z_]+/g) ?? []) { + expect([ + "$feed_redirect_kind", + "$original_scheme", + "$original_authority", + "$feed_redirect_authority", + "$feed_redirect_path", + ]).toContain(name); + } + for (const rule of urlMap?.rules ?? []) expect(rule.value).toBe("$1://$2$3"); + }); + + it("sets $orig_loc before anything reads a map derived from it", () => { + const setIndex = namedLocation.indexOf('set $orig_loc "$upstream_http_location";'); + expect(setIndex).toBeGreaterThan(0); + expect(setIndex).toBeLessThan(namedLocation.indexOf("$feed_redirect_")); + }); + + it("does not forward per-feed headers to redirect targets", () => { + expect(config).toContain('proxy_set_header "Authorization" "Bearer abc";'); + expect(namedLocation).not.toContain("proxy_set_header"); + }); + + it.each([ + ["/v2/trip-updates.pb", "https://feeds.example.org/v2/trip-updates.pb"], + ["/", "https://feeds.example.org/"], + ["/x?u=http://redis/y", "https://feeds.example.org/x?u=http://redis/y"], + ["https://feeds.example.org/v2.pb", "https://feeds.example.org/v2.pb"], + ["HTTPS://FEEDS.example.org/v2.pb", "https://FEEDS.example.org/v2.pb"], + ["https://cdn.example-cdn.net/rt.pb?sig=1", "https://cdn.example-cdn.net/rt.pb?sig=1"], + [ + "https://bucket.s3.eu-central-1.amazonaws.com", + "https://bucket.s3.eu-central-1.amazonaws.com/", + ], + ["https://cdn.example-cdn.net?feed=rt", "https://cdn.example-cdn.net/?feed=rt"], + ["https://cdn.example-cdn.net/a@b/c%20d", "https://cdn.example-cdn.net/a@b/c%20d"], + ])("follows %s", (location, expected) => { + expect(evaluateRedirect(location, feed)).toEqual({ allowed: true, url: expected }); + }); + + it.each([ + "http://feeds.example.org/v2.pb", + "http://cdn.example-cdn.net/rt.pb", + "https://redis/x", + "https://localhost/x", + "https://ops-agent:8080/x", + "https://10.0.0.5/x", + "https://93.184.216.34/x", + "https://0x7f.1/x", + "https://[::1]/x", + "https://rt.openmapx.local/x", + "https://api.internal/x", + "https://box.lan/x", + "https://nas.home.arpa/x", + "https://motis.docker/x", + "https://user@cdn.example-cdn.net/x", + "https://cdn.example-cdn.net:8443/x", + "https://cdn.example-cdn.net./x", + "//cdn.example-cdn.net/x", + "ftp://cdn.example-cdn.net/x", + "rt.pb", + "", + "https://cdn.example-cdn.net#redis:6379/x", + "https://cdn.example-cdn.net#.internal.example:8080/x", + "https://cdn.example-cdn.net#:6379/", + "https://feeds.example.org#redis:6379/x", + "https://cdn.example-cdn.net/x#part", + "/x#part", + "https://redis.openmapx/x", + "https://REDIS.OPENMAPX/x", + "https://dawarich-app.openmapx-private-dawarich/x", + "https://svc.openmapx-community-example/x", + "https://redis.openmapx_openmapx/x", + "https://cdn.example-cdn.net\\@redis/x", + "https://cdn.example-cdn.net/x\\y", + "/x\\y", + "\\\\redis/x", + "https://cdn.example-cdn.net /x", + "https://cdn.example-cdn.net/x y", + "/x y", + "/x\ty", + "/x\u0001y", + "https://cdn.example-cdn.net/x|y", + ])("refuses %j", (location) => { + expect(evaluateRedirect(location, feed).allowed).toBe(false); + }); + + it("lets an http feed follow http redirects and keeps its port on relative ones", () => { + const httpFeed = "http://10.1.2.3:8080/vehicles.pb"; + expect(evaluateRedirect("/v2.pb", httpFeed)).toEqual({ + allowed: true, + url: "http://10.1.2.3:8080/v2.pb", + }); + expect(evaluateRedirect("http://10.1.2.3:8080/v2.pb", httpFeed).allowed).toBe(true); + expect(evaluateRedirect("http://cdn.example-cdn.net/v2.pb", httpFeed).allowed).toBe(true); + expect(evaluateRedirect("http://10.1.2.4:8080/v2.pb", httpFeed).allowed).toBe(false); + }); +}); diff --git a/packages/motis-feed-proxy-config/src/build.ts b/packages/motis-feed-proxy-config/src/build.ts index d3c28a11c..664ed37d2 100644 --- a/packages/motis-feed-proxy-config/src/build.ts +++ b/packages/motis-feed-proxy-config/src/build.ts @@ -162,6 +162,167 @@ export function writeFeedProxyVarsFile(outputPath: string, raw: unknown): FeedPr return sorted; } +/** One rule of an nginx `map`: a regex key and the value it yields. */ +export interface FeedRedirectMapRule { + /** PCRE source, also valid as a JavaScript RegExp so tests can evaluate it. */ + pattern: string; + caseInsensitive?: boolean; + value: string; +} + +/** An nginx `map` block rendered into the http context. */ +export interface FeedRedirectMap { + /** Map source, a string of nginx variables. */ + source: string; + /** Variable the map defines, without the leading `$`. */ + variable: string; + defaultValue: string; + /** Checked in order; the first match wins, as nginx does for regex keys. */ + rules: FeedRedirectMapRule[]; +} + +/** + * Redirect policy for the primary feed server, as a chain of nginx maps over + * the upstream `Location` header (`$orig_loc`) and the feed's own scheme and + * authority (`$original_scheme`, `$original_authority`). + * + * A redirect is followed when it is relative, or absolute to the feed's own + * authority, or absolute to another host whose name looks public: dotted, + * letters in the last label, no port, no user info, and none of the local, + * internal or Compose network suffixes below. An https feed is never followed + * onto http. Everything else is answered with 502. Name checks cannot see + * where a public name resolves to, so network placement and egress policy + * still matter. + * + * The whole Location must match a strict shape: plain host and optional port, + * then nothing, a path or a query, using only URL characters (no fragment, + * backslash, whitespace or control characters). The upstream URL is rebuilt + * from the matched scheme, authority and path, never from the raw header, so + * nginx cannot read a different host out of it than the one checked here. + */ +const REDIRECT_PATH_CHARS = "[A-Za-z0-9._~!$&'()*+,;=:@%/?-]"; +const REDIRECT_PATH_FIRST_CHARS = "[A-Za-z0-9._~!$&'()*+,;=:@%?-]"; +const REDIRECT_AUTHORITY = "[a-z0-9.-]+(?::[0-9]+)?"; +// A single leading slash; `//host` would change host without a scheme. +const RELATIVE_REDIRECT = `/(?:${REDIRECT_PATH_FIRST_CHARS}${REDIRECT_PATH_CHARS}*)?`; +const ABSOLUTE_REDIRECT_REST = `(?:[/?]${REDIRECT_PATH_CHARS}*)?`; + +/** + * Last labels that name this stack's Compose networks: the shared `openmapx` + * network and the `openmapx-private-*` and `openmapx-community-*` networks. + * Docker's DNS also answers `.`. The project-prefixed names + * (`openmapx_…`) contain an underscore, which the host shape already refuses. + */ +const INTERNAL_NAME_SUFFIXES = + "localhost|localdomain|local|internal|lan|home\\.arpa|docker|openmapx(?:-[a-z0-9-]*)?"; + +export const FEED_REDIRECT_MAPS: readonly FeedRedirectMap[] = [ + { + source: "$orig_loc", + variable: "feed_redirect_kind", + defaultValue: "refuse", + rules: [ + { pattern: `^${RELATIVE_REDIRECT}$`, value: "relative" }, + { + pattern: `^https://${REDIRECT_AUTHORITY}${ABSOLUTE_REDIRECT_REST}$`, + caseInsensitive: true, + value: "https", + }, + { + pattern: `^http://${REDIRECT_AUTHORITY}${ABSOLUTE_REDIRECT_REST}$`, + caseInsensitive: true, + value: "http", + }, + ], + }, + { + source: "$orig_loc", + variable: "feed_redirect_authority", + defaultValue: "", + rules: [ + { + pattern: `^https?://(${REDIRECT_AUTHORITY})${ABSOLUTE_REDIRECT_REST}$`, + caseInsensitive: true, + value: "$1", + }, + ], + }, + { + source: "$orig_loc", + variable: "feed_redirect_path", + defaultValue: "", + rules: [ + { pattern: `^(${RELATIVE_REDIRECT})$`, value: "$1" }, + { + pattern: `^https?://${REDIRECT_AUTHORITY}(/${REDIRECT_PATH_CHARS}*)$`, + caseInsensitive: true, + value: "$1", + }, + // No path: send `/`, since a proxy_pass URL without one would forward + // this proxy's own request path instead. + { + pattern: `^https?://${REDIRECT_AUTHORITY}(\\?${REDIRECT_PATH_CHARS}*)?$`, + caseInsensitive: true, + value: "/$1", + }, + ], + }, + { + source: "$feed_redirect_authority|$original_authority", + variable: "feed_redirect_same_host", + defaultValue: "0", + rules: [{ pattern: "^([^|]+)\\|\\1$", caseInsensitive: true, value: "1" }], + }, + { + source: "$feed_redirect_authority", + variable: "feed_redirect_host_public", + defaultValue: "0", + rules: [ + { pattern: `(?:^|\\.)(?:${INTERNAL_NAME_SUFFIXES})$`, caseInsensitive: true, value: "0" }, + // At least two labels, and a last label starting with a letter, which + // rules out IP literals and numeric host forms. No port allowed. + { pattern: "^(?:[a-z0-9-]+\\.)+[a-z][a-z0-9-]*$", caseInsensitive: true, value: "1" }, + ], + }, + { + source: + "$feed_redirect_kind:$feed_redirect_same_host:$feed_redirect_host_public:$original_scheme", + variable: "feed_redirect_allowed", + defaultValue: "0", + rules: [ + { pattern: "^relative:", value: "1" }, + { pattern: "^https:1:", value: "1" }, + { pattern: "^https:0:1:", value: "1" }, + { pattern: "^http:1:[01]:http$", value: "1" }, + { pattern: "^http:0:1:http$", value: "1" }, + ], + }, + { + // Built only from validated parts; none of them can contain `|`. + source: + "$feed_redirect_kind|$original_scheme|$original_authority|$feed_redirect_authority|$feed_redirect_path", + variable: "feed_redirect_url", + defaultValue: "", + rules: [ + { pattern: "^relative\\|([a-z]+)\\|([^|]+)\\|\\|(/[^|]*)$", value: "$1://$2$3" }, + { pattern: "^(https?)\\|[^|]*\\|[^|]*\\|([^|]+)\\|(/[^|]*)$", value: "$1://$2$3" }, + ], + }, +]; + +function renderFeedRedirectMaps(): string[] { + const out: string[] = []; + for (const map of FEED_REDIRECT_MAPS) { + out.push(`map "${map.source}" $${map.variable} {`); + out.push(` default "${map.defaultValue}";`); + for (const rule of map.rules) { + out.push(` "~${rule.caseInsensitive ? "*" : ""}${rule.pattern}" "${rule.value}";`); + } + out.push("}"); + } + return out; +} + function renderPrimaryServer(entries: Array<[string, FeedProxyEntry]>): string[] { const out: string[] = [ "server {", @@ -183,7 +344,10 @@ function renderPrimaryServer(entries: Array<[string, FeedProxyEntry]>): string[] " proxy_ignore_headers Cache-Control;", " proxy_http_version 1.1;", " proxy_intercept_errors on;", - " error_page 301 302 307 308 = @handle_redirects;", + " error_page 301 302 303 307 308 = @handle_redirects;", + // Needed at this level so the error_page inside @handle_redirects can + // still answer a second redirect; it is not honoured there alone. + " recursive_error_pages on;", " proxy_cache_valid 200 40s;", " proxy_cache_valid 400 401 403 404 200s;", " proxy_ignore_client_abort on;", @@ -202,15 +366,21 @@ function renderPrimaryServer(entries: Array<[string, FeedProxyEntry]>): string[] const parsed = requireHttpUrl(entry.url); out.push(` location "/feed/${encodeURIComponent(key)}" {`); out.push(` set $feed_upstream "${escapeNginxQuoted(entry.url)}";`); - out.push(` set $original_host "${escapeNginxQuoted(parsed.hostname)}";`); + out.push(` set $original_scheme "${parsed.protocol.slice(0, -1)}";`); + out.push(` set $original_authority "${escapeNginxQuoted(parsed.host)}";`); out.push(" proxy_pass $feed_upstream;"); for (const header of renderHeaderDirectives(entry.headers)) out.push(header); out.push(" }", ""); } out.push( + // No proxy_set_header here: the feed location's headers, which can carry + // credentials, must not reach a redirect target. " location @handle_redirects {", ' set $original_uri "$request_uri";', + // The $feed_redirect_* maps read $orig_loc and nginx caches a map value + // the first time it is read, so this set must stay ahead of every + // directive that reads one of those variables. ' set $orig_loc "$upstream_http_location";', "", " proxy_cache feed-proxy;", @@ -218,15 +388,17 @@ function renderPrimaryServer(entries: Array<[string, FeedProxyEntry]>): string[] " proxy_cache_valid 200 40s;", " proxy_cache_valid 400 401 403 404 200s;", "", - ' if ($orig_loc ~* "://") {', - " proxy_pass $orig_loc;", - " break;", - " }", + // Follow at most one redirect; a further one is refused. + " error_page 301 302 303 307 308 = @feed_redirect_refused;", "", - ' if ($orig_loc !~* "://") {', - " proxy_pass $scheme://$original_host$orig_loc;", - " break;", + " if ($feed_redirect_allowed = 0) {", + ' return 502 "feed redirect target refused\\n";', " }", + " proxy_pass $feed_redirect_url;", + " }", + "", + " location @feed_redirect_refused {", + ' return 502 "feed redirect chain refused\\n";', " }", "}", ); @@ -298,6 +470,8 @@ export function renderFeedProxyNginxConfig( // upstream hostnames at request time (else 502 "no resolver defined"). ...(resolver.trim() ? [`resolver ${resolver.trim()};`] : []), "", + ...renderFeedRedirectMaps(), + "", ...renderPrimaryServer(entries), ]; if (includeGbfsServers) {