From 41bd031f312f43ff3c865677806f21028ab11e8c Mon Sep 17 00:00:00 2001 From: Jerry Combs Date: Fri, 23 Oct 2026 12:56:15 -0500 Subject: [PATCH] Rotate the audit log by size; read it backwards and stop early (#161) audit.log moves to audit.log.1 before a line would pass audit.maxBytes (default 10MB), older files up by one, none kept past audit.keep (default 10; 0 keeps all, maxBytes 0 never rotates). A line is never split. read() goes newest file first, each from its end in blocks, and stops at the limit or since, instead of reading the whole log per request. Also fixes since: times have 3, 6 or 9 fraction digits, so comparing them as text could end the list too early; they compare as instants now, and a partly typed time is the start of that period. Closes #161 Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 16 ++ bin/smoke-hosted.sh | 23 +- docs/hosted-install.md | 4 +- docs/hosted-mode.md | 12 +- docs/security.md | 2 +- .../java/com/pointblue/dirxml/web/Audit.java | 209 ++++++++++++++++-- .../java/com/pointblue/dirxml/web/Hosted.java | 26 +++ .../java/com/pointblue/dirxml/web/Serve.java | 2 +- .../com/pointblue/dirxml/web/AuditTest.java | 193 ++++++++++++++++ web/src/components/AuditView.tsx | 2 +- 10 files changed, 457 insertions(+), 32 deletions(-) create mode 100644 src/test/java/com/pointblue/dirxml/web/AuditTest.java diff --git a/CHANGELOG.md b/CHANGELOG.md index ae574d5..e4fce7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,22 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ## [Unreleased] +### Changed + +- **The audit log is rotated by size (#161); a change in default behaviour.** Before a line would + take `audit.log` past `audit.maxBytes` (default `10MB`), it moves to `audit.log.1` and the older + ones up by one; past `audit.keep` (default 10) the oldest is deleted, so the log takes at most + about 110 MB. `audit.keep=0` deletes nothing, `audit.maxBytes=0` never rotates (the old + behaviour). An existing `audit.log` larger than the limit is rotated on the first line after the + upgrade; nothing is deleted then. The Audit view reads the files newest first, from their ends, + and stops as soon as it has its lines, instead of reading the whole log for every request. + +### Fixed + +- The Audit view's *since* compares times as times. Times are written with 3, 6 or 9 fraction + digits, and compared as text a later line could sort before an earlier one and end the list too + soon. A partly typed time (`2026-10-07`, `2026-10-07T09:30`) is the start of that period, in UTC. + ### Removed - docs/hosted.md, the hosted-mode design note of 2026-10-05: its body described the design before it diff --git a/bin/smoke-hosted.sh b/bin/smoke-hosted.sh index ef8e6d4..9c8c9c7 100755 --- a/bin/smoke-hosted.sh +++ b/bin/smoke-hosted.sh @@ -41,6 +41,8 @@ setup.roots=$WORK agents=ci:idm-deployers;reader: agent.ci.token=ci-agent-token-0123456789 agent.reader.token=reader-agent-token-0123456789 +audit.maxBytes=2KB +audit.keep=0 ENV IDMWEB_AGENT_KIT="$HERE/hosted/agent-kit" IDM_ENVIRONMENTS="$WORK/data/environments.properties" "$HERE/bin/idmweb" serve --hosted "$WORK/hosted.properties" > "$WORK/log" 2>&1 & PID=$! @@ -389,8 +391,19 @@ try: s, r, _ = bob.call("POST", "/api/setup/directory", {"project": "outside", "path": "/etc"}) check("a deployer may attach only under setup.roots (#41)", s == 400 and "outside" in r.get("error", "")) # #42: the audit log, deployers only + import datetime + def when(t): # 3, 6 or 9 fraction digits: compare as times, not text (#161) + return datetime.datetime.fromisoformat(t.replace("Z", "+00:00")) s, a, _ = bob.call("GET", "/api/audit?limit=50") - check("a deployer reads the audit log (newest first, sign-ins among them)", s == 200 and len(a["entries"]) > 0 and any(e["event"] == "sign-in" for e in a["entries"]) and a["entries"][0]["time"] >= a["entries"][-1]["time"]) + check("a deployer reads the audit log (newest first, sign-ins among them)", s == 200 and len(a["entries"]) > 0 and any(e["event"] == "sign-in" for e in a["entries"]) and when(a["entries"][0]["time"]) >= when(a["entries"][-1]["time"])) + # #161: rotated by size (2KB here, all kept); the view reads across the files, newest first + logs = sorted(f for f in os.listdir(os.path.join(work, "data")) if f.startswith("audit.log")) + current = len(open(os.path.join(work, "data", "audit.log")).read().splitlines()) + whole = all(l.startswith("{") and l.endswith("}") for f in logs for l in open(os.path.join(work, "data", f)).read().splitlines()) + check("the audit log rotates at audit.maxBytes, every line whole", "audit.log.1" in logs and whole + and all(os.path.getsize(os.path.join(work, "data", f)) <= 2 * 1024 for f in logs)) + s, a, _ = bob.call("GET", "/api/audit?limit=2000") + check("the Audit view reads past the current file into the rotated ones", s == 200 and len(a["entries"]) > current) s, a, _ = bob.call("GET", "/api/audit?event=sign-in&user=alice") check("filtered by event and user", s == 200 and a["entries"] and all(e["event"] == "sign-in" and e["user"] == "alice" for e in a["entries"])) s, a, _ = bob.call("GET", "/api/audit?event=git-token") @@ -418,8 +431,12 @@ finally: check("sessions.json is the owner's only", oct(os.stat(os.path.join(work, "data2", "sessions.json")).st_mode)[-3:] == "600") p2.terminate() -with open(os.path.join(work, "data", "audit.log")) as f: - lines = [json.loads(l) for l in f if l.strip()] +# the whole log, oldest first: audit.log.N … audit.log.1, then audit.log (rotated, #161) +rotated = sorted((int(f.rsplit(".", 1)[1]) for f in os.listdir(os.path.join(work, "data")) if f.startswith("audit.log.")), reverse=True) +lines = [] +for f in ["audit.log.%d" % n for n in rotated] + ["audit.log"]: + with open(os.path.join(work, "data", f)) as fh: + lines += [json.loads(l) for l in fh if l.strip()] events = [(l["user"], l["event"], l["outcome"]) for l in lines] check("the audit log has sign-ins, the workspace, the edit, the commit and the sign-out, per user", ("alice", "sign-in", "ok") in events and ("alice", "workspace.open", "ok") in events and ("alice", "op/gcv.set", "ok") in events and ("alice", "commit+push", "ok") in events and ("bob", "workspace.open", "ok") in events and ("alice", "sign-out", "ok") in events and ("agent:ci", "workspace.open", "ok") in events) diff --git a/docs/hosted-install.md b/docs/hosted-install.md index fef04a5..2197dbc 100644 --- a/docs/hosted-install.md +++ b/docs/hosted-install.md @@ -152,7 +152,9 @@ ssh debian@192.168.103.99 'cd ~/idmweb && sed -i "s#^IDMWEB_IMAGE=.*#IDMWEB_IMAG - The `idmweb-data` volume holds the workspaces, `audit.log`, `sessions.json`, each person's files (`users//`: vault credentials, Git tokens, traces), clone tokens (`secrets/`) and archived projects (`archive/`); the configuration is in `./config`. Back up both. -- `audit.log` is one JSON line per event; ship it where the customer keeps logs. +- `audit.log` is one JSON line per event, rotated by size to `audit.log.1` … (`audit.maxBytes`, + `audit.keep`: [hosted-mode.md](hosted-mode.md#the-audit-log)). Ship the numbered files where the + customer keeps logs; set `audit.keep=0` if nothing may be deleted. - Sessions are kept in `/sessions.json` (owner-only): a restart signs nobody out, and each person gets their last project and branch back on their next session. - `bin/smoke-container.sh` runs the image end to end on a workstation with Docker. diff --git a/docs/hosted-mode.md b/docs/hosted-mode.md index 3bb7b26..f4d936a 100644 --- a/docs/hosted-mode.md +++ b/docs/hosted-mode.md @@ -19,7 +19,7 @@ deploys come from the base branch only, as the repository holds it. The server h definitions (`environments.file`, no secrets) with the driver secrets beside them, and each person's bind DN and password in their own file under `data.dir/users//` (docs/multi-user.md); it writes an audit line per sign-in, write, deploy, driver action, trace -change, commit and push (`data.dir/audit.log`, not rotated yet: issue #161). Roles come from groups: *readers* look; *editors* read, edit and commit; +change, commit and push (`data.dir/audit.log`, rotated by size: [the audit log](#the-audit-log)). Roles come from groups: *readers* look; *editors* read, edit and commit; *deployers* also deploy, control drivers, set trace levels and read the engine's trace file, and change the shared environments and driver secrets. Watching a live trace is open to anyone signed in. @@ -51,8 +51,18 @@ roles.deployer=idm-deployers session.hours=8 # 0: never (the one-person install); sessions survive a restart release.check=true # once a day: is a newer release out? (the top bar says so; false: never ask) setup.roots=/srv,/var/lib/idmweb # where path-based setup may point (default: data.dir and /srv) +audit.maxBytes=10MB # audit.log moves to audit.log.1 before a line would take it past this (KB, MB, GB; 0: never) +audit.keep=10 # rotated files kept (audit.log.1 … .10), the oldest deleted (0: keep them all) ``` +**The audit log.** One JSON line per sign-in, write, deploy, driver action, trace change, commit +and push, in `data.dir/audit.log`. Before a line would take it past `audit.maxBytes`, the file +moves to `audit.log.1` and the older ones up by one (`.1` to `.2` …); the ones past `audit.keep` +are deleted. With the defaults that is at most about 110 MB: the current file and ten 10 MB +ones. A line is never split between files. The Audit view (deployers) reads them newest first, and +stops as soon as it has the lines it shows. To keep everything, set `audit.keep=0` and ship the +rotated files to where your logs are kept; each one is complete once it has a number. + **Roles and the host.** An *editor* reads, edits, commits and pushes, and adds a project by uploading a Designer project or export, exporting from a vault, or cloning a remote. Attaching a path of the host — a checkout in place, a bare repository, a directory of files — typing a diff --git a/docs/security.md b/docs/security.md index d0a8750..a24dc08 100644 --- a/docs/security.md +++ b/docs/security.md @@ -14,7 +14,7 @@ configured for one person and one for a team. | Roles | none: the one person may do everything | reader, editor, deployer (`roles.*`, docs/hosted-mode.md) | | Vault credentials | the `idm` CLI's environments file (`IDM_ENVIRONMENTS`, `./environments.properties`, `~/.idm/environments.properties`), secrets included | the project's definitions in `environments.file` (no secrets; the server will not start with any), each person's bind DN and password in their own `data.dir/users//environments.properties` | | Where a typed password is kept | the macOS Keychain, the owner-only file, or a variable, command or secret-manager reference | the person's own owner-only file; a deployer may also keep a reference (below). The Keychain is macOS only, so never in the container | -| Audit log | none | `data.dir/audit.log`, readable by deployers | +| Audit log | none | `data.dir/audit.log` (owner-only), readable by deployers; rotated by size to `audit.log.1` …, the oldest past `audit.keep` deleted (`audit.keep=0` keeps all) | | Typed secrets cross | the loopback connection | the connection to the server: TLS only when `public.url` is https (a team server behind its proxy; the local container stays on the host's loopback) | The rest holds in both modes unless it names one. diff --git a/src/main/java/com/pointblue/dirxml/web/Audit.java b/src/main/java/com/pointblue/dirxml/web/Audit.java index 88a3ba3..d27ba9e 100644 --- a/src/main/java/com/pointblue/dirxml/web/Audit.java +++ b/src/main/java/com/pointblue/dirxml/web/Audit.java @@ -1,27 +1,47 @@ package com.pointblue.dirxml.web; import java.io.IOException; +import java.io.RandomAccessFile; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.time.Instant; import java.util.LinkedHashMap; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.TreeSet; /** * One JSON line per thing a person did: when, who, which workspace and branch, what, with * what arguments (never file contents or secret values), and how it went. Hosted mode writes * {@code audit.log} in its data directory; local mode has no file and the calls are no-ops. + * + *

Rotated by size (#161): a line that would take {@code audit.log} past {@code maxBytes} first moves it to + * {@code audit.log.1}, the older ones up by one ({@code .1} to {@code .2} …), and the ones past {@code keep} are + * deleted; {@code keep} 0 deletes none, {@code maxBytes} 0 never rotates. A line is never split between files. + * Reading goes from the newest line backwards, file by file, and stops as soon as it has enough. */ public final class Audit { + /** {@code audit.maxBytes} and {@code audit.keep} when the configuration doesn't say: about 110 MB in all. */ + public static final long DEFAULT_MAX_BYTES = 10L * 1024 * 1024; + public static final int DEFAULT_KEEP = 10; + private final Path file; + private final long maxBytes; + private final int keep; public Audit(Path file) { + this(file, DEFAULT_MAX_BYTES, DEFAULT_KEEP); + } + + public Audit(Path file, long maxBytes, int keep) { this.file = file; + this.maxBytes = maxBytes; + this.keep = keep; } public static Audit none() { @@ -35,34 +55,51 @@ public static Audit none() { */ public synchronized List> read(String since, String user, String project, String event, int limit) throws IOException { List> out = new ArrayList<>(); - if (file == null || !Files.exists(file)) { + if (file == null) { return out; } - List lines = Files.readAllLines(file, StandardCharsets.UTF_8); - for (int i = lines.size() - 1; i >= 0 && out.size() < Math.max(1, limit); i--) { - String line = lines.get(i).trim(); - if (line.isEmpty()) { - continue; - } - Map m; - try { - m = com.pointblue.dirxml.dev.json.Json.asMap(com.pointblue.dirxml.dev.json.Json.parse(line)); - } catch (RuntimeException e) { - continue; - } - if (since != null && !since.isBlank() && String.valueOf(m.get("time")).compareTo(since) < 0) { - break; // older than asked: the file is in time order - } - if (user != null && !user.isBlank() && !user.equals(m.get("user"))) { - continue; - } - if (project != null && !project.isBlank() && !project.equals(m.get("project"))) { + Instant from = since == null || since.isBlank() ? null : since(since); + // the current file, then .1, .2 …: newest to oldest, each in time order + List files = new ArrayList<>(); + files.add(file); + for (int n : rotated()) { + files.add(rotated(n)); + } + for (Path f : files) { + if (!Files.exists(f)) { continue; } - if (event != null && !event.isBlank() && !String.valueOf(m.get("event")).contains(event)) { - continue; + try (Backwards lines = new Backwards(f)) { + for (String raw = lines.next(); raw != null; raw = lines.next()) { + if (out.size() >= Math.max(1, limit)) { + return out; + } + String line = raw.trim(); + if (line.isEmpty()) { + continue; + } + Map m; + try { + m = com.pointblue.dirxml.dev.json.Json.asMap(com.pointblue.dirxml.dev.json.Json.parse(line)); + } catch (RuntimeException e) { + continue; + } + Instant at = instant(String.valueOf(m.get("time"))); + if (from != null && at != null && at.isBefore(from)) { + return out; // older than asked: the files are in time order + } + if (user != null && !user.isBlank() && !user.equals(m.get("user"))) { + continue; + } + if (project != null && !project.isBlank() && !project.equals(m.get("project"))) { + continue; + } + if (event != null && !event.isBlank() && !String.valueOf(m.get("event")).contains(event)) { + continue; + } + out.add(m); + } } - out.add(m); } return out; } @@ -83,14 +120,138 @@ public synchronized void log(Session session, Workspace ws, String event, Map 0 && Files.exists(file) && Files.size(file) > 0 && Files.size(file) + line.length > maxBytes) { + rotate(); + } + Files.write(file, line, StandardOpenOption.CREATE, StandardOpenOption.APPEND); PropertiesFile.ownerOnly(file); } catch (IOException e) { System.err.println("audit: cannot write " + file + ": " + e.getMessage()); } } + + /** + * A line's time, compared as a time: Instant.toString writes 3, 6 or 9 fraction digits as needed, so the + * strings don't sort (".123Z" after ".123456Z"). + */ + static Instant instant(String s) { + try { + return Instant.parse(s.trim()); + } catch (RuntimeException e) { + return null; + } + } + + /** + * The {@code since} filter, in UTC: a whole instant, or the start of one as the page's box is typed + * ({@code 2026-10}, {@code 2026-10-07T09:30}), completed with the earliest values; null (no filter) when it + * doesn't read as either. + */ + static Instant since(String s) { + String t = s.trim(); + Instant whole = instant(t); + if (whole != null) { + return whole; + } + String bare = t.endsWith("Z") || t.endsWith("z") ? t.substring(0, t.length() - 1) : t; + String template = "0000-01-01T00:00:00"; + if (bare.length() < 4 || bare.length() > template.length()) { + return null; + } + return instant(bare + template.substring(bare.length()) + "Z"); + } + + /** audit.log to .1, the rest up by one, and none kept past {@code keep} (when it is set). */ + private void rotate() throws IOException { + TreeSet old = rotated(); + if (keep > 0) { + for (int n : new ArrayList<>(old)) { + if (n >= keep) { + Files.deleteIfExists(rotated(n)); // it would move past keep + old.remove(n); + } + } + } + for (int n : old.descendingSet()) { + Files.move(rotated(n), rotated(n + 1), StandardCopyOption.REPLACE_EXISTING); + } + Files.move(file, rotated(1), StandardCopyOption.REPLACE_EXISTING); + } + + private Path rotated(int n) { + return file.resolveSibling(file.getFileName() + "." + n); + } + + /** The numbers of the rotated files there are (audit.log.1, .2 …), lowest first. */ + private TreeSet rotated() throws IOException { + TreeSet out = new TreeSet<>(); + Path dir = file.toAbsolutePath().getParent(); + if (dir == null || !Files.isDirectory(dir)) { + return out; + } + String prefix = file.getFileName() + "."; + try (var list = Files.list(dir)) { + list.forEach(p -> { + String name = p.getFileName().toString(); + if (name.startsWith(prefix) && name.substring(prefix.length()).matches("[1-9][0-9]{0,5}")) { + out.add(Integer.parseInt(name.substring(prefix.length()))); + } + }); + } + return out; + } + + /** A file's lines from the last to the first, read from the end in blocks: only as much as is asked for. */ + static final class Backwards implements AutoCloseable { + + private static final int BLOCK = 64 * 1024; + private final RandomAccessFile in; + private long pos; + private byte[] buf = new byte[0]; + private int end; // buf[0 .. end) is read and not yet returned + + Backwards(Path f) throws IOException { + in = new RandomAccessFile(f.toFile(), "r"); + pos = in.length(); + } + + /** The next line going backwards, without its newline; null at the start of the file. */ + String next() throws IOException { + while (true) { + for (int i = end - 1; i >= 0; i--) { + if (buf[i] == '\n') { + String line = new String(buf, i + 1, end - i - 1, StandardCharsets.UTF_8); + end = i; + return line; // after the file's last newline this is "", which readers skip + } + } + if (pos == 0) { + if (end == 0) { + return null; + } + String line = new String(buf, 0, end, StandardCharsets.UTF_8); + end = 0; + return line; + } + int n = (int) Math.min(BLOCK, pos); + pos -= n; + byte[] more = new byte[n + end]; + in.seek(pos); + in.readFully(more, 0, n); + System.arraycopy(buf, 0, more, n, end); + buf = more; + end = n + end; + } + } + + @Override + public void close() throws IOException { + in.close(); + } + } } diff --git a/src/main/java/com/pointblue/dirxml/web/Hosted.java b/src/main/java/com/pointblue/dirxml/web/Hosted.java index ccba1e7..e1a0a6a 100644 --- a/src/main/java/com/pointblue/dirxml/web/Hosted.java +++ b/src/main/java/com/pointblue/dirxml/web/Hosted.java @@ -47,6 +47,8 @@ * session.hours=8 * agents=ci:idm-deployers;claude:idm-editors # non-browser callers: name:group1|group2 * agent.ci.tokenKeychain=idmweb/ci # each agent's token (or .token, .tokenEnv, .tokenCommand) + * audit.maxBytes=10MB # audit.log moves to audit.log.1 before it would pass this (0: never) + * audit.keep=10 # rotated files kept, oldest deleted (0: keep them all) * * An agent sends {@code Authorization: Bearer } and gets a session of its own, with the * roles its groups give and its name on every audit line; no cookie, so no CSRF token either. @@ -71,6 +73,9 @@ public final class Hosted { /** Personal agent tokens ({@code agent.tokens}, default on) and their longest life ({@code agent.tokens.maxDays}, default 365). */ public final boolean agentTokens; public final int agentTokensMaxDays; + /** The audit log's rotation (#161): {@code audit.maxBytes} (default 10MB; 0 never rotates) and {@code audit.keep} (default 10; 0 keeps all). */ + public final long auditMaxBytes; + public final int auditKeep; /** * Where a person's environments come from: the definitions merged with their own credentials @@ -101,6 +106,11 @@ private Hosted(Path file, Properties p) { this.dataDir = Path.of(p.getProperty("data.dir", "data").trim()).toAbsolutePath().normalize(); this.agentTokens = !"false".equalsIgnoreCase(p.getProperty("agent.tokens", "true").trim()); this.agentTokensMaxDays = Integer.parseInt(p.getProperty("agent.tokens.maxDays", "365").trim()); + this.auditMaxBytes = bytes("audit.maxBytes", p.getProperty("audit.maxBytes", String.valueOf(Audit.DEFAULT_MAX_BYTES))); + this.auditKeep = Integer.parseInt(p.getProperty("audit.keep", String.valueOf(Audit.DEFAULT_KEEP)).trim()); + if (auditKeep < 0) { + throw new IllegalArgumentException("audit.keep is a number of files, 0 or more"); + } // projects=a,b with project..repo.*; the older single repo.* keys are the project named by repo.name for (String n : list(p.getProperty("projects", ""))) { Project pr = project(p, "project." + n + ".", n); @@ -355,4 +365,20 @@ public List noneUsers() { } return out; } + + /** A size: a number of bytes, or with KB, MB or GB (1024s), e.g. {@code 10MB}. */ + static long bytes(String key, String value) { + java.util.regex.Matcher m = java.util.regex.Pattern.compile("(?i)\\s*(\\d+)\\s*(KB|MB|GB|K|M|G|B)?\\s*").matcher(value); + if (!m.matches()) { + throw new IllegalArgumentException(key + " is a size: bytes, or a number with KB, MB or GB (got '" + value + "')"); + } + long n = Long.parseLong(m.group(1)); + String unit = m.group(2) == null ? "B" : m.group(2).toUpperCase(); + return switch (unit.charAt(0)) { + case 'K' -> n * 1024; + case 'M' -> n * 1024 * 1024; + case 'G' -> n * 1024 * 1024 * 1024; + default -> n; + }; + } } diff --git a/src/main/java/com/pointblue/dirxml/web/Serve.java b/src/main/java/com/pointblue/dirxml/web/Serve.java index 2be12d2..91a2b6d 100644 --- a/src/main/java/com/pointblue/dirxml/web/Serve.java +++ b/src/main/java/com/pointblue/dirxml/web/Serve.java @@ -101,7 +101,7 @@ static void hosted(Path config) throws Exception { // kept in the data directory: a restart signs nobody out, and everyone gets their last branch back Session.Store sessions = Session.Store.persistent(h.sessionHours <= 0 ? Session.Store.NEVER : h.sessionHours * 3600, h.dataDir.resolve("sessions.json")); Security security = Security.hosted(h.publicUrl); - Audit audit = new Audit(h.dataDir.resolve("audit.log")); + Audit audit = new Audit(h.dataDir.resolve("audit.log"), h.auditMaxBytes, h.auditKeep); Identity identity; switch (h.identity) { case "none": diff --git a/src/test/java/com/pointblue/dirxml/web/AuditTest.java b/src/test/java/com/pointblue/dirxml/web/AuditTest.java new file mode 100644 index 0000000..88d36ef --- /dev/null +++ b/src/test/java/com/pointblue/dirxml/web/AuditTest.java @@ -0,0 +1,193 @@ +package com.pointblue.dirxml.web; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +/** The audit log (#161): rotated by size, never a line split, read newest first across the files, stopping early. */ +public class AuditTest { + + @Rule + public TemporaryFolder tmp = new TemporaryFolder(); + + private static Session as(String who) { + return new Session("s-" + who, new Session.Principal(who, who, null, List.of("idm-deployers"), true, true)); + } + + private static void write(Audit a, int from, int to) { + for (int i = from; i < to; i++) { + a.log(as("alice"), null, "op", Map.of("n", i), "ok"); + } + } + + private static List numbers(List> entries) { + List out = new ArrayList<>(); + for (Map e : entries) { + out.add(((Number) ((Map) e.get("details")).get("n")).intValue()); + } + return out; + } + + private static int lines(Path f) throws Exception { + return Files.exists(f) ? Files.readAllLines(f, StandardCharsets.UTF_8).size() : 0; + } + + @Test + public void rotatesBeforeALineWouldPassTheLimitAndNeverSplitsOne() throws Exception { + Path dir = tmp.newFolder("data").toPath(); + Path log = dir.resolve("audit.log"); + new Audit(log).log(as("alice"), null, "op", Map.of("n", 0), "ok"); + long one = Files.size(log); + Files.delete(log); + Audit a = new Audit(log, one * 3, 10); // three lines a file + write(a, 0, 7); + assertEquals(1, lines(log)); + assertEquals(3, lines(dir.resolve("audit.log.1"))); + assertEquals(3, lines(dir.resolve("audit.log.2"))); + assertTrue(Files.size(log) <= one * 3 && Files.size(dir.resolve("audit.log.1")) <= one * 3); + for (Path f : List.of(log, dir.resolve("audit.log.1"), dir.resolve("audit.log.2"))) { + for (String l : Files.readAllLines(f, StandardCharsets.UTF_8)) { + assertTrue("whole JSON lines only: " + l, l.startsWith("{") && l.endsWith("}")); + } + } + assertEquals("newest first, across the files", List.of(6, 5, 4, 3, 2, 1, 0), numbers(a.read(null, null, null, null, 100))); + } + + @Test + public void keepsOnlyKeepRotatedFilesAndZeroKeepsThemAll() throws Exception { + Path dir = tmp.newFolder("data").toPath(); + Path log = dir.resolve("audit.log"); + new Audit(log).log(as("alice"), null, "op", Map.of("n", 0), "ok"); + long one = Files.size(log); + Files.delete(log); + Audit two = new Audit(log, one, 2); // one line a file, two rotated files kept + write(two, 0, 6); + assertTrue(Files.exists(dir.resolve("audit.log.2"))); + assertFalse("the oldest go", Files.exists(dir.resolve("audit.log.3"))); + assertEquals(List.of(5, 4, 3), numbers(two.read(null, null, null, null, 100))); + + Path dir2 = tmp.newFolder("all").toPath(); + Audit all = new Audit(dir2.resolve("audit.log"), one, 0); + write(all, 0, 6); + assertTrue("keep 0 deletes none", Files.exists(dir2.resolve("audit.log.5"))); + assertEquals(List.of(5, 4, 3, 2, 1, 0), numbers(all.read(null, null, null, null, 100))); + + // a lower keep later: the files past it go at the next rotation + Audit fewer = new Audit(dir2.resolve("audit.log"), one, 2); + write(fewer, 6, 7); + assertFalse(Files.exists(dir2.resolve("audit.log.3"))); + assertEquals(List.of(6, 5, 4), numbers(fewer.read(null, null, null, null, 100))); + } + + @Test + public void zeroMaxBytesNeverRotates() throws Exception { + Path dir = tmp.newFolder("data").toPath(); + Audit a = new Audit(dir.resolve("audit.log"), 0, 10); + write(a, 0, 50); + assertEquals(50, lines(dir.resolve("audit.log"))); + assertFalse(Files.exists(dir.resolve("audit.log.1"))); + } + + @Test + public void readingStopsAtSinceAndLimitAndFilters() throws Exception { + Path dir = tmp.newFolder("data").toPath(); + Path log = dir.resolve("audit.log"); + Audit a = new Audit(log, 400, 10); // a few lines a file + write(a, 0, 20); + a.log(as("bob"), null, "deploy", Map.of("n", 20), "ok"); + assertTrue("rotated", Files.exists(dir.resolve("audit.log.2"))); + assertEquals(List.of(20, 19, 18), numbers(a.read(null, null, null, null, 3))); + assertEquals(List.of(20), numbers(a.read(null, "bob", null, null, 100))); + assertEquals(List.of(20), numbers(a.read(null, null, null, "deploy", 100))); + String sinceLast = String.valueOf(a.read(null, null, null, null, 1).get(0).get("time")); + assertTrue(numbers(a.read(sinceLast, null, null, null, 100)).contains(20)); + assertTrue("since stops before older lines", a.read("2999-01-01T00:00:00Z", null, null, null, 100).isEmpty()); + // times with 3, 6 or 9 fraction digits compare as times, not as text + Path mixed = dir.resolve("mixed.log"); + Files.writeString(mixed, "{\"time\":\"2026-10-23T09:00:00.123Z\",\"event\":\"a\",\"details\":{\"n\":1}}\n" + + "{\"time\":\"2026-10-23T09:00:00.123456Z\",\"event\":\"b\",\"details\":{\"n\":2}}\n", StandardCharsets.UTF_8); + assertEquals("only the later line is at or after .1234 (as text, .123456Z sorts before .1234Z)", List.of(2), numbers(new Audit(mixed).read("2026-10-23T09:00:00.1234Z", null, null, null, 10))); + } + + @Test + public void readsBackwardsAcrossBlocksWithLongAndNonAsciiLines() throws Exception { + Path f = tmp.newFile("lines").toPath(); + StringBuilder sb = new StringBuilder(); + List want = new ArrayList<>(); + for (int i = 0; i < 300; i++) { + String l = i + " " + "é€😀".repeat(i % 7) + "x".repeat((i * 997) % 3000); + want.add(l); + sb.append(l).append('\n'); + } + Files.writeString(f, sb.toString(), StandardCharsets.UTF_8); + List got = new ArrayList<>(); + try (Audit.Backwards b = new Audit.Backwards(f)) { + for (String l = b.next(); l != null; l = b.next()) { + if (!l.isEmpty()) { + got.add(0, l); + } + } + } + assertEquals(want, got); + } + + @Test + public void concurrentWritersLoseNoLineAcrossRotations() throws Exception { + Path dir = tmp.newFolder("data").toPath(); + Audit a = new Audit(dir.resolve("audit.log"), 2000, 0); + List threads = new ArrayList<>(); + for (int t = 0; t < 8; t++) { + int base = t * 100; + threads.add(new Thread(() -> write(a, base, base + 100))); + } + threads.forEach(Thread::start); + for (Thread t : threads) { + t.join(); + } + List> all = a.read(null, null, null, null, 10_000); + Set seen = new HashSet<>(numbers(all)); + assertEquals(800, all.size()); + assertEquals(800, seen.size()); + for (int i = 1; i < all.size(); i++) { + assertFalse("newest first", Audit.instant(String.valueOf(all.get(i - 1).get("time"))).isBefore(Audit.instant(String.valueOf(all.get(i).get("time"))))); + } + } + + @Test + public void sinceTakesThePartTypedSoFar() { + assertEquals(java.time.Instant.parse("2026-01-01T00:00:00Z"), Audit.since("2026")); + assertEquals(java.time.Instant.parse("2026-10-01T00:00:00Z"), Audit.since("2026-10")); + assertEquals(java.time.Instant.parse("2026-10-07T00:00:00Z"), Audit.since("2026-10-07")); + assertEquals(java.time.Instant.parse("2026-10-07T09:30:00Z"), Audit.since("2026-10-07T09:30")); + assertEquals(java.time.Instant.parse("2026-10-07T09:30:15.5Z"), Audit.since("2026-10-07T09:30:15.5Z")); + assertEquals(null, Audit.since("yesterday")); + assertEquals(null, Audit.since("20")); + } + + @Test + public void sizesTakeUnits() { + assertEquals(10L * 1024 * 1024, Hosted.bytes("audit.maxBytes", "10MB")); + assertEquals(512 * 1024, Hosted.bytes("audit.maxBytes", "512k")); + assertEquals(2L * 1024 * 1024 * 1024, Hosted.bytes("audit.maxBytes", "2 GB")); + assertEquals(1000, Hosted.bytes("audit.maxBytes", "1000")); + assertEquals(0, Hosted.bytes("audit.maxBytes", "0")); + try { + Hosted.bytes("audit.maxBytes", "ten"); + throw new AssertionError("refused"); + } catch (IllegalArgumentException expected) { + assertTrue(expected.getMessage().contains("audit.maxBytes")); + } + } +} diff --git a/web/src/components/AuditView.tsx b/web/src/components/AuditView.tsx index 49391af..0c71dba 100644 --- a/web/src/components/AuditView.tsx +++ b/web/src/components/AuditView.tsx @@ -26,7 +26,7 @@ export function AuditView() { setF({ ...f, user: e.target.value })} placeholder="user" spellCheck={false} /> setF({ ...f, project: e.target.value })} placeholder="project" spellCheck={false} /> setF({ ...f, event: e.target.value })} placeholder="event (sign-in, write, deploy, commit…)" spellCheck={false} /> - setF({ ...f, since: e.target.value })} placeholder="since (2026-10-07T00:00)" spellCheck={false} /> + setF({ ...f, since: e.target.value })} placeholder="since, UTC (2026-10-07T00:00)" spellCheck={false} /> {error &&

{error}
}