diff --git a/CHANGELOG.md b/CHANGELOG.md index ae574d5..e4fce7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,22 @@ and this project uses [Semantic Versioning](https://semver.org/spec/v2.0.0.html) ## [Unreleased] +### Changed + +- **The audit log is rotated by size (#161); a change in default behaviour.** Before a line would + take `audit.log` past `audit.maxBytes` (default `10MB`), it moves to `audit.log.1` and the older + ones up by one; past `audit.keep` (default 10) the oldest is deleted, so the log takes at most + about 110 MB. `audit.keep=0` deletes nothing, `audit.maxBytes=0` never rotates (the old + behaviour). An existing `audit.log` larger than the limit is rotated on the first line after the + upgrade; nothing is deleted then. The Audit view reads the files newest first, from their ends, + and stops as soon as it has its lines, instead of reading the whole log for every request. + +### Fixed + +- The Audit view's *since* compares times as times. Times are written with 3, 6 or 9 fraction + digits, and compared as text a later line could sort before an earlier one and end the list too + soon. A partly typed time (`2026-10-07`, `2026-10-07T09:30`) is the start of that period, in UTC. + ### Removed - docs/hosted.md, the hosted-mode design note of 2026-10-05: its body described the design before it diff --git a/bin/smoke-hosted.sh b/bin/smoke-hosted.sh index ef8e6d4..9c8c9c7 100755 --- a/bin/smoke-hosted.sh +++ b/bin/smoke-hosted.sh @@ -41,6 +41,8 @@ setup.roots=$WORK agents=ci:idm-deployers;reader: agent.ci.token=ci-agent-token-0123456789 agent.reader.token=reader-agent-token-0123456789 +audit.maxBytes=2KB +audit.keep=0 ENV IDMWEB_AGENT_KIT="$HERE/hosted/agent-kit" IDM_ENVIRONMENTS="$WORK/data/environments.properties" "$HERE/bin/idmweb" serve --hosted "$WORK/hosted.properties" > "$WORK/log" 2>&1 & PID=$! @@ -389,8 +391,19 @@ try: s, r, _ = bob.call("POST", "/api/setup/directory", {"project": "outside", "path": "/etc"}) check("a deployer may attach only under setup.roots (#41)", s == 400 and "outside" in r.get("error", "")) # #42: the audit log, deployers only + import datetime + def when(t): # 3, 6 or 9 fraction digits: compare as times, not text (#161) + return datetime.datetime.fromisoformat(t.replace("Z", "+00:00")) s, a, _ = bob.call("GET", "/api/audit?limit=50") - check("a deployer reads the audit log (newest first, sign-ins among them)", s == 200 and len(a["entries"]) > 0 and any(e["event"] == "sign-in" for e in a["entries"]) and a["entries"][0]["time"] >= a["entries"][-1]["time"]) + check("a deployer reads the audit log (newest first, sign-ins among them)", s == 200 and len(a["entries"]) > 0 and any(e["event"] == "sign-in" for e in a["entries"]) and when(a["entries"][0]["time"]) >= when(a["entries"][-1]["time"])) + # #161: rotated by size (2KB here, all kept); the view reads across the files, newest first + logs = sorted(f for f in os.listdir(os.path.join(work, "data")) if f.startswith("audit.log")) + current = len(open(os.path.join(work, "data", "audit.log")).read().splitlines()) + whole = all(l.startswith("{") and l.endswith("}") for f in logs for l in open(os.path.join(work, "data", f)).read().splitlines()) + check("the audit log rotates at audit.maxBytes, every line whole", "audit.log.1" in logs and whole + and all(os.path.getsize(os.path.join(work, "data", f)) <= 2 * 1024 for f in logs)) + s, a, _ = bob.call("GET", "/api/audit?limit=2000") + check("the Audit view reads past the current file into the rotated ones", s == 200 and len(a["entries"]) > current) s, a, _ = bob.call("GET", "/api/audit?event=sign-in&user=alice") check("filtered by event and user", s == 200 and a["entries"] and all(e["event"] == "sign-in" and e["user"] == "alice" for e in a["entries"])) s, a, _ = bob.call("GET", "/api/audit?event=git-token") @@ -418,8 +431,12 @@ finally: check("sessions.json is the owner's only", oct(os.stat(os.path.join(work, "data2", "sessions.json")).st_mode)[-3:] == "600") p2.terminate() -with open(os.path.join(work, "data", "audit.log")) as f: - lines = [json.loads(l) for l in f if l.strip()] +# the whole log, oldest first: audit.log.N … audit.log.1, then audit.log (rotated, #161) +rotated = sorted((int(f.rsplit(".", 1)[1]) for f in os.listdir(os.path.join(work, "data")) if f.startswith("audit.log.")), reverse=True) +lines = [] +for f in ["audit.log.%d" % n for n in rotated] + ["audit.log"]: + with open(os.path.join(work, "data", f)) as fh: + lines += [json.loads(l) for l in fh if l.strip()] events = [(l["user"], l["event"], l["outcome"]) for l in lines] check("the audit log has sign-ins, the workspace, the edit, the commit and the sign-out, per user", ("alice", "sign-in", "ok") in events and ("alice", "workspace.open", "ok") in events and ("alice", "op/gcv.set", "ok") in events and ("alice", "commit+push", "ok") in events and ("bob", "workspace.open", "ok") in events and ("alice", "sign-out", "ok") in events and ("agent:ci", "workspace.open", "ok") in events) diff --git a/docs/hosted-install.md b/docs/hosted-install.md index fef04a5..2197dbc 100644 --- a/docs/hosted-install.md +++ b/docs/hosted-install.md @@ -152,7 +152,9 @@ ssh debian@192.168.103.99 'cd ~/idmweb && sed -i "s#^IDMWEB_IMAGE=.*#IDMWEB_IMAG - The `idmweb-data` volume holds the workspaces, `audit.log`, `sessions.json`, each person's files (`users//`: vault credentials, Git tokens, traces), clone tokens (`secrets/`) and archived projects (`archive/`); the configuration is in `./config`. Back up both. -- `audit.log` is one JSON line per event; ship it where the customer keeps logs. +- `audit.log` is one JSON line per event, rotated by size to `audit.log.1` … (`audit.maxBytes`, + `audit.keep`: [hosted-mode.md](hosted-mode.md#the-audit-log)). Ship the numbered files where the + customer keeps logs; set `audit.keep=0` if nothing may be deleted. - Sessions are kept in `/sessions.json` (owner-only): a restart signs nobody out, and each person gets their last project and branch back on their next session. - `bin/smoke-container.sh` runs the image end to end on a workstation with Docker. diff --git a/docs/hosted-mode.md b/docs/hosted-mode.md index 3bb7b26..f4d936a 100644 --- a/docs/hosted-mode.md +++ b/docs/hosted-mode.md @@ -19,7 +19,7 @@ deploys come from the base branch only, as the repository holds it. The server h definitions (`environments.file`, no secrets) with the driver secrets beside them, and each person's bind DN and password in their own file under `data.dir/users//` (docs/multi-user.md); it writes an audit line per sign-in, write, deploy, driver action, trace -change, commit and push (`data.dir/audit.log`, not rotated yet: issue #161). Roles come from groups: *readers* look; *editors* read, edit and commit; +change, commit and push (`data.dir/audit.log`, rotated by size: [the audit log](#the-audit-log)). Roles come from groups: *readers* look; *editors* read, edit and commit; *deployers* also deploy, control drivers, set trace levels and read the engine's trace file, and change the shared environments and driver secrets. Watching a live trace is open to anyone signed in. @@ -51,8 +51,18 @@ roles.deployer=idm-deployers session.hours=8 # 0: never (the one-person install); sessions survive a restart release.check=true # once a day: is a newer release out? (the top bar says so; false: never ask) setup.roots=/srv,/var/lib/idmweb # where path-based setup may point (default: data.dir and /srv) +audit.maxBytes=10MB # audit.log moves to audit.log.1 before a line would take it past this (KB, MB, GB; 0: never) +audit.keep=10 # rotated files kept (audit.log.1 … .10), the oldest deleted (0: keep them all) ``` +**The audit log.** One JSON line per sign-in, write, deploy, driver action, trace change, commit +and push, in `data.dir/audit.log`. Before a line would take it past `audit.maxBytes`, the file +moves to `audit.log.1` and the older ones up by one (`.1` to `.2` …); the ones past `audit.keep` +are deleted. With the defaults that is at most about 110 MB: the current file and ten 10 MB +ones. A line is never split between files. The Audit view (deployers) reads them newest first, and +stops as soon as it has the lines it shows. To keep everything, set `audit.keep=0` and ship the +rotated files to where your logs are kept; each one is complete once it has a number. + **Roles and the host.** An *editor* reads, edits, commits and pushes, and adds a project by uploading a Designer project or export, exporting from a vault, or cloning a remote. Attaching a path of the host — a checkout in place, a bare repository, a directory of files — typing a diff --git a/docs/security.md b/docs/security.md index d0a8750..a24dc08 100644 --- a/docs/security.md +++ b/docs/security.md @@ -14,7 +14,7 @@ configured for one person and one for a team. | Roles | none: the one person may do everything | reader, editor, deployer (`roles.*`, docs/hosted-mode.md) | | Vault credentials | the `idm` CLI's environments file (`IDM_ENVIRONMENTS`, `./environments.properties`, `~/.idm/environments.properties`), secrets included | the project's definitions in `environments.file` (no secrets; the server will not start with any), each person's bind DN and password in their own `data.dir/users//environments.properties` | | Where a typed password is kept | the macOS Keychain, the owner-only file, or a variable, command or secret-manager reference | the person's own owner-only file; a deployer may also keep a reference (below). The Keychain is macOS only, so never in the container | -| Audit log | none | `data.dir/audit.log`, readable by deployers | +| Audit log | none | `data.dir/audit.log` (owner-only), readable by deployers; rotated by size to `audit.log.1` …, the oldest past `audit.keep` deleted (`audit.keep=0` keeps all) | | Typed secrets cross | the loopback connection | the connection to the server: TLS only when `public.url` is https (a team server behind its proxy; the local container stays on the host's loopback) | The rest holds in both modes unless it names one. diff --git a/src/main/java/com/pointblue/dirxml/web/Audit.java b/src/main/java/com/pointblue/dirxml/web/Audit.java index 88a3ba3..d27ba9e 100644 --- a/src/main/java/com/pointblue/dirxml/web/Audit.java +++ b/src/main/java/com/pointblue/dirxml/web/Audit.java @@ -1,27 +1,47 @@ package com.pointblue.dirxml.web; import java.io.IOException; +import java.io.RandomAccessFile; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.time.Instant; import java.util.LinkedHashMap; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.TreeSet; /** * One JSON line per thing a person did: when, who, which workspace and branch, what, with * what arguments (never file contents or secret values), and how it went. Hosted mode writes * {@code audit.log} in its data directory; local mode has no file and the calls are no-ops. + * + *

Rotated by size (#161): a line that would take {@code audit.log} past {@code maxBytes} first moves it to + * {@code audit.log.1}, the older ones up by one ({@code .1} to {@code .2} …), and the ones past {@code keep} are + * deleted; {@code keep} 0 deletes none, {@code maxBytes} 0 never rotates. A line is never split between files. + * Reading goes from the newest line backwards, file by file, and stops as soon as it has enough. */ public final class Audit { + /** {@code audit.maxBytes} and {@code audit.keep} when the configuration doesn't say: about 110 MB in all. */ + public static final long DEFAULT_MAX_BYTES = 10L * 1024 * 1024; + public static final int DEFAULT_KEEP = 10; + private final Path file; + private final long maxBytes; + private final int keep; public Audit(Path file) { + this(file, DEFAULT_MAX_BYTES, DEFAULT_KEEP); + } + + public Audit(Path file, long maxBytes, int keep) { this.file = file; + this.maxBytes = maxBytes; + this.keep = keep; } public static Audit none() { @@ -35,34 +55,51 @@ public static Audit none() { */ public synchronized List> read(String since, String user, String project, String event, int limit) throws IOException { List> out = new ArrayList<>(); - if (file == null || !Files.exists(file)) { + if (file == null) { return out; } - List lines = Files.readAllLines(file, StandardCharsets.UTF_8); - for (int i = lines.size() - 1; i >= 0 && out.size() < Math.max(1, limit); i--) { - String line = lines.get(i).trim(); - if (line.isEmpty()) { - continue; - } - Map m; - try { - m = com.pointblue.dirxml.dev.json.Json.asMap(com.pointblue.dirxml.dev.json.Json.parse(line)); - } catch (RuntimeException e) { - continue; - } - if (since != null && !since.isBlank() && String.valueOf(m.get("time")).compareTo(since) < 0) { - break; // older than asked: the file is in time order - } - if (user != null && !user.isBlank() && !user.equals(m.get("user"))) { - continue; - } - if (project != null && !project.isBlank() && !project.equals(m.get("project"))) { + Instant from = since == null || since.isBlank() ? null : since(since); + // the current file, then .1, .2 …: newest to oldest, each in time order + List files = new ArrayList<>(); + files.add(file); + for (int n : rotated()) { + files.add(rotated(n)); + } + for (Path f : files) { + if (!Files.exists(f)) { continue; } - if (event != null && !event.isBlank() && !String.valueOf(m.get("event")).contains(event)) { - continue; + try (Backwards lines = new Backwards(f)) { + for (String raw = lines.next(); raw != null; raw = lines.next()) { + if (out.size() >= Math.max(1, limit)) { + return out; + } + String line = raw.trim(); + if (line.isEmpty()) { + continue; + } + Map m; + try { + m = com.pointblue.dirxml.dev.json.Json.asMap(com.pointblue.dirxml.dev.json.Json.parse(line)); + } catch (RuntimeException e) { + continue; + } + Instant at = instant(String.valueOf(m.get("time"))); + if (from != null && at != null && at.isBefore(from)) { + return out; // older than asked: the files are in time order + } + if (user != null && !user.isBlank() && !user.equals(m.get("user"))) { + continue; + } + if (project != null && !project.isBlank() && !project.equals(m.get("project"))) { + continue; + } + if (event != null && !event.isBlank() && !String.valueOf(m.get("event")).contains(event)) { + continue; + } + out.add(m); + } } - out.add(m); } return out; } @@ -83,14 +120,138 @@ public synchronized void log(Session session, Workspace ws, String event, Map 0 && Files.exists(file) && Files.size(file) > 0 && Files.size(file) + line.length > maxBytes) { + rotate(); + } + Files.write(file, line, StandardOpenOption.CREATE, StandardOpenOption.APPEND); PropertiesFile.ownerOnly(file); } catch (IOException e) { System.err.println("audit: cannot write " + file + ": " + e.getMessage()); } } + + /** + * A line's time, compared as a time: Instant.toString writes 3, 6 or 9 fraction digits as needed, so the + * strings don't sort (".123Z" after ".123456Z"). + */ + static Instant instant(String s) { + try { + return Instant.parse(s.trim()); + } catch (RuntimeException e) { + return null; + } + } + + /** + * The {@code since} filter, in UTC: a whole instant, or the start of one as the page's box is typed + * ({@code 2026-10}, {@code 2026-10-07T09:30}), completed with the earliest values; null (no filter) when it + * doesn't read as either. + */ + static Instant since(String s) { + String t = s.trim(); + Instant whole = instant(t); + if (whole != null) { + return whole; + } + String bare = t.endsWith("Z") || t.endsWith("z") ? t.substring(0, t.length() - 1) : t; + String template = "0000-01-01T00:00:00"; + if (bare.length() < 4 || bare.length() > template.length()) { + return null; + } + return instant(bare + template.substring(bare.length()) + "Z"); + } + + /** audit.log to .1, the rest up by one, and none kept past {@code keep} (when it is set). */ + private void rotate() throws IOException { + TreeSet old = rotated(); + if (keep > 0) { + for (int n : new ArrayList<>(old)) { + if (n >= keep) { + Files.deleteIfExists(rotated(n)); // it would move past keep + old.remove(n); + } + } + } + for (int n : old.descendingSet()) { + Files.move(rotated(n), rotated(n + 1), StandardCopyOption.REPLACE_EXISTING); + } + Files.move(file, rotated(1), StandardCopyOption.REPLACE_EXISTING); + } + + private Path rotated(int n) { + return file.resolveSibling(file.getFileName() + "." + n); + } + + /** The numbers of the rotated files there are (audit.log.1, .2 …), lowest first. */ + private TreeSet rotated() throws IOException { + TreeSet out = new TreeSet<>(); + Path dir = file.toAbsolutePath().getParent(); + if (dir == null || !Files.isDirectory(dir)) { + return out; + } + String prefix = file.getFileName() + "."; + try (var list = Files.list(dir)) { + list.forEach(p -> { + String name = p.getFileName().toString(); + if (name.startsWith(prefix) && name.substring(prefix.length()).matches("[1-9][0-9]{0,5}")) { + out.add(Integer.parseInt(name.substring(prefix.length()))); + } + }); + } + return out; + } + + /** A file's lines from the last to the first, read from the end in blocks: only as much as is asked for. */ + static final class Backwards implements AutoCloseable { + + private static final int BLOCK = 64 * 1024; + private final RandomAccessFile in; + private long pos; + private byte[] buf = new byte[0]; + private int end; // buf[0 .. end) is read and not yet returned + + Backwards(Path f) throws IOException { + in = new RandomAccessFile(f.toFile(), "r"); + pos = in.length(); + } + + /** The next line going backwards, without its newline; null at the start of the file. */ + String next() throws IOException { + while (true) { + for (int i = end - 1; i >= 0; i--) { + if (buf[i] == '\n') { + String line = new String(buf, i + 1, end - i - 1, StandardCharsets.UTF_8); + end = i; + return line; // after the file's last newline this is "", which readers skip + } + } + if (pos == 0) { + if (end == 0) { + return null; + } + String line = new String(buf, 0, end, StandardCharsets.UTF_8); + end = 0; + return line; + } + int n = (int) Math.min(BLOCK, pos); + pos -= n; + byte[] more = new byte[n + end]; + in.seek(pos); + in.readFully(more, 0, n); + System.arraycopy(buf, 0, more, n, end); + buf = more; + end = n + end; + } + } + + @Override + public void close() throws IOException { + in.close(); + } + } } diff --git a/src/main/java/com/pointblue/dirxml/web/Hosted.java b/src/main/java/com/pointblue/dirxml/web/Hosted.java index ccba1e7..e1a0a6a 100644 --- a/src/main/java/com/pointblue/dirxml/web/Hosted.java +++ b/src/main/java/com/pointblue/dirxml/web/Hosted.java @@ -47,6 +47,8 @@ * session.hours=8 * agents=ci:idm-deployers;claude:idm-editors # non-browser callers: name:group1|group2 * agent.ci.tokenKeychain=idmweb/ci # each agent's token (or .token, .tokenEnv, .tokenCommand) + * audit.maxBytes=10MB # audit.log moves to audit.log.1 before it would pass this (0: never) + * audit.keep=10 # rotated files kept, oldest deleted (0: keep them all) * * An agent sends {@code Authorization: Bearer } and gets a session of its own, with the * roles its groups give and its name on every audit line; no cookie, so no CSRF token either. @@ -71,6 +73,9 @@ public final class Hosted { /** Personal agent tokens ({@code agent.tokens}, default on) and their longest life ({@code agent.tokens.maxDays}, default 365). */ public final boolean agentTokens; public final int agentTokensMaxDays; + /** The audit log's rotation (#161): {@code audit.maxBytes} (default 10MB; 0 never rotates) and {@code audit.keep} (default 10; 0 keeps all). */ + public final long auditMaxBytes; + public final int auditKeep; /** * Where a person's environments come from: the definitions merged with their own credentials @@ -101,6 +106,11 @@ private Hosted(Path file, Properties p) { this.dataDir = Path.of(p.getProperty("data.dir", "data").trim()).toAbsolutePath().normalize(); this.agentTokens = !"false".equalsIgnoreCase(p.getProperty("agent.tokens", "true").trim()); this.agentTokensMaxDays = Integer.parseInt(p.getProperty("agent.tokens.maxDays", "365").trim()); + this.auditMaxBytes = bytes("audit.maxBytes", p.getProperty("audit.maxBytes", String.valueOf(Audit.DEFAULT_MAX_BYTES))); + this.auditKeep = Integer.parseInt(p.getProperty("audit.keep", String.valueOf(Audit.DEFAULT_KEEP)).trim()); + if (auditKeep < 0) { + throw new IllegalArgumentException("audit.keep is a number of files, 0 or more"); + } // projects=a,b with project..repo.*; the older single repo.* keys are the project named by repo.name for (String n : list(p.getProperty("projects", ""))) { Project pr = project(p, "project." + n + ".", n); @@ -355,4 +365,20 @@ public List noneUsers() { } return out; } + + /** A size: a number of bytes, or with KB, MB or GB (1024s), e.g. {@code 10MB}. */ + static long bytes(String key, String value) { + java.util.regex.Matcher m = java.util.regex.Pattern.compile("(?i)\\s*(\\d+)\\s*(KB|MB|GB|K|M|G|B)?\\s*").matcher(value); + if (!m.matches()) { + throw new IllegalArgumentException(key + " is a size: bytes, or a number with KB, MB or GB (got '" + value + "')"); + } + long n = Long.parseLong(m.group(1)); + String unit = m.group(2) == null ? "B" : m.group(2).toUpperCase(); + return switch (unit.charAt(0)) { + case 'K' -> n * 1024; + case 'M' -> n * 1024 * 1024; + case 'G' -> n * 1024 * 1024 * 1024; + default -> n; + }; + } } diff --git a/src/main/java/com/pointblue/dirxml/web/Serve.java b/src/main/java/com/pointblue/dirxml/web/Serve.java index 2be12d2..91a2b6d 100644 --- a/src/main/java/com/pointblue/dirxml/web/Serve.java +++ b/src/main/java/com/pointblue/dirxml/web/Serve.java @@ -101,7 +101,7 @@ static void hosted(Path config) throws Exception { // kept in the data directory: a restart signs nobody out, and everyone gets their last branch back Session.Store sessions = Session.Store.persistent(h.sessionHours <= 0 ? Session.Store.NEVER : h.sessionHours * 3600, h.dataDir.resolve("sessions.json")); Security security = Security.hosted(h.publicUrl); - Audit audit = new Audit(h.dataDir.resolve("audit.log")); + Audit audit = new Audit(h.dataDir.resolve("audit.log"), h.auditMaxBytes, h.auditKeep); Identity identity; switch (h.identity) { case "none": diff --git a/src/test/java/com/pointblue/dirxml/web/AuditTest.java b/src/test/java/com/pointblue/dirxml/web/AuditTest.java new file mode 100644 index 0000000..88d36ef --- /dev/null +++ b/src/test/java/com/pointblue/dirxml/web/AuditTest.java @@ -0,0 +1,193 @@ +package com.pointblue.dirxml.web; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +/** The audit log (#161): rotated by size, never a line split, read newest first across the files, stopping early. */ +public class AuditTest { + + @Rule + public TemporaryFolder tmp = new TemporaryFolder(); + + private static Session as(String who) { + return new Session("s-" + who, new Session.Principal(who, who, null, List.of("idm-deployers"), true, true)); + } + + private static void write(Audit a, int from, int to) { + for (int i = from; i < to; i++) { + a.log(as("alice"), null, "op", Map.of("n", i), "ok"); + } + } + + private static List numbers(List> entries) { + List out = new ArrayList<>(); + for (Map e : entries) { + out.add(((Number) ((Map) e.get("details")).get("n")).intValue()); + } + return out; + } + + private static int lines(Path f) throws Exception { + return Files.exists(f) ? Files.readAllLines(f, StandardCharsets.UTF_8).size() : 0; + } + + @Test + public void rotatesBeforeALineWouldPassTheLimitAndNeverSplitsOne() throws Exception { + Path dir = tmp.newFolder("data").toPath(); + Path log = dir.resolve("audit.log"); + new Audit(log).log(as("alice"), null, "op", Map.of("n", 0), "ok"); + long one = Files.size(log); + Files.delete(log); + Audit a = new Audit(log, one * 3, 10); // three lines a file + write(a, 0, 7); + assertEquals(1, lines(log)); + assertEquals(3, lines(dir.resolve("audit.log.1"))); + assertEquals(3, lines(dir.resolve("audit.log.2"))); + assertTrue(Files.size(log) <= one * 3 && Files.size(dir.resolve("audit.log.1")) <= one * 3); + for (Path f : List.of(log, dir.resolve("audit.log.1"), dir.resolve("audit.log.2"))) { + for (String l : Files.readAllLines(f, StandardCharsets.UTF_8)) { + assertTrue("whole JSON lines only: " + l, l.startsWith("{") && l.endsWith("}")); + } + } + assertEquals("newest first, across the files", List.of(6, 5, 4, 3, 2, 1, 0), numbers(a.read(null, null, null, null, 100))); + } + + @Test + public void keepsOnlyKeepRotatedFilesAndZeroKeepsThemAll() throws Exception { + Path dir = tmp.newFolder("data").toPath(); + Path log = dir.resolve("audit.log"); + new Audit(log).log(as("alice"), null, "op", Map.of("n", 0), "ok"); + long one = Files.size(log); + Files.delete(log); + Audit two = new Audit(log, one, 2); // one line a file, two rotated files kept + write(two, 0, 6); + assertTrue(Files.exists(dir.resolve("audit.log.2"))); + assertFalse("the oldest go", Files.exists(dir.resolve("audit.log.3"))); + assertEquals(List.of(5, 4, 3), numbers(two.read(null, null, null, null, 100))); + + Path dir2 = tmp.newFolder("all").toPath(); + Audit all = new Audit(dir2.resolve("audit.log"), one, 0); + write(all, 0, 6); + assertTrue("keep 0 deletes none", Files.exists(dir2.resolve("audit.log.5"))); + assertEquals(List.of(5, 4, 3, 2, 1, 0), numbers(all.read(null, null, null, null, 100))); + + // a lower keep later: the files past it go at the next rotation + Audit fewer = new Audit(dir2.resolve("audit.log"), one, 2); + write(fewer, 6, 7); + assertFalse(Files.exists(dir2.resolve("audit.log.3"))); + assertEquals(List.of(6, 5, 4), numbers(fewer.read(null, null, null, null, 100))); + } + + @Test + public void zeroMaxBytesNeverRotates() throws Exception { + Path dir = tmp.newFolder("data").toPath(); + Audit a = new Audit(dir.resolve("audit.log"), 0, 10); + write(a, 0, 50); + assertEquals(50, lines(dir.resolve("audit.log"))); + assertFalse(Files.exists(dir.resolve("audit.log.1"))); + } + + @Test + public void readingStopsAtSinceAndLimitAndFilters() throws Exception { + Path dir = tmp.newFolder("data").toPath(); + Path log = dir.resolve("audit.log"); + Audit a = new Audit(log, 400, 10); // a few lines a file + write(a, 0, 20); + a.log(as("bob"), null, "deploy", Map.of("n", 20), "ok"); + assertTrue("rotated", Files.exists(dir.resolve("audit.log.2"))); + assertEquals(List.of(20, 19, 18), numbers(a.read(null, null, null, null, 3))); + assertEquals(List.of(20), numbers(a.read(null, "bob", null, null, 100))); + assertEquals(List.of(20), numbers(a.read(null, null, null, "deploy", 100))); + String sinceLast = String.valueOf(a.read(null, null, null, null, 1).get(0).get("time")); + assertTrue(numbers(a.read(sinceLast, null, null, null, 100)).contains(20)); + assertTrue("since stops before older lines", a.read("2999-01-01T00:00:00Z", null, null, null, 100).isEmpty()); + // times with 3, 6 or 9 fraction digits compare as times, not as text + Path mixed = dir.resolve("mixed.log"); + Files.writeString(mixed, "{\"time\":\"2026-10-23T09:00:00.123Z\",\"event\":\"a\",\"details\":{\"n\":1}}\n" + + "{\"time\":\"2026-10-23T09:00:00.123456Z\",\"event\":\"b\",\"details\":{\"n\":2}}\n", StandardCharsets.UTF_8); + assertEquals("only the later line is at or after .1234 (as text, .123456Z sorts before .1234Z)", List.of(2), numbers(new Audit(mixed).read("2026-10-23T09:00:00.1234Z", null, null, null, 10))); + } + + @Test + public void readsBackwardsAcrossBlocksWithLongAndNonAsciiLines() throws Exception { + Path f = tmp.newFile("lines").toPath(); + StringBuilder sb = new StringBuilder(); + List want = new ArrayList<>(); + for (int i = 0; i < 300; i++) { + String l = i + " " + "é€😀".repeat(i % 7) + "x".repeat((i * 997) % 3000); + want.add(l); + sb.append(l).append('\n'); + } + Files.writeString(f, sb.toString(), StandardCharsets.UTF_8); + List got = new ArrayList<>(); + try (Audit.Backwards b = new Audit.Backwards(f)) { + for (String l = b.next(); l != null; l = b.next()) { + if (!l.isEmpty()) { + got.add(0, l); + } + } + } + assertEquals(want, got); + } + + @Test + public void concurrentWritersLoseNoLineAcrossRotations() throws Exception { + Path dir = tmp.newFolder("data").toPath(); + Audit a = new Audit(dir.resolve("audit.log"), 2000, 0); + List threads = new ArrayList<>(); + for (int t = 0; t < 8; t++) { + int base = t * 100; + threads.add(new Thread(() -> write(a, base, base + 100))); + } + threads.forEach(Thread::start); + for (Thread t : threads) { + t.join(); + } + List> all = a.read(null, null, null, null, 10_000); + Set seen = new HashSet<>(numbers(all)); + assertEquals(800, all.size()); + assertEquals(800, seen.size()); + for (int i = 1; i < all.size(); i++) { + assertFalse("newest first", Audit.instant(String.valueOf(all.get(i - 1).get("time"))).isBefore(Audit.instant(String.valueOf(all.get(i).get("time"))))); + } + } + + @Test + public void sinceTakesThePartTypedSoFar() { + assertEquals(java.time.Instant.parse("2026-01-01T00:00:00Z"), Audit.since("2026")); + assertEquals(java.time.Instant.parse("2026-10-01T00:00:00Z"), Audit.since("2026-10")); + assertEquals(java.time.Instant.parse("2026-10-07T00:00:00Z"), Audit.since("2026-10-07")); + assertEquals(java.time.Instant.parse("2026-10-07T09:30:00Z"), Audit.since("2026-10-07T09:30")); + assertEquals(java.time.Instant.parse("2026-10-07T09:30:15.5Z"), Audit.since("2026-10-07T09:30:15.5Z")); + assertEquals(null, Audit.since("yesterday")); + assertEquals(null, Audit.since("20")); + } + + @Test + public void sizesTakeUnits() { + assertEquals(10L * 1024 * 1024, Hosted.bytes("audit.maxBytes", "10MB")); + assertEquals(512 * 1024, Hosted.bytes("audit.maxBytes", "512k")); + assertEquals(2L * 1024 * 1024 * 1024, Hosted.bytes("audit.maxBytes", "2 GB")); + assertEquals(1000, Hosted.bytes("audit.maxBytes", "1000")); + assertEquals(0, Hosted.bytes("audit.maxBytes", "0")); + try { + Hosted.bytes("audit.maxBytes", "ten"); + throw new AssertionError("refused"); + } catch (IllegalArgumentException expected) { + assertTrue(expected.getMessage().contains("audit.maxBytes")); + } + } +} diff --git a/web/src/components/AuditView.tsx b/web/src/components/AuditView.tsx index 49391af..0c71dba 100644 --- a/web/src/components/AuditView.tsx +++ b/web/src/components/AuditView.tsx @@ -26,7 +26,7 @@ export function AuditView() { setF({ ...f, user: e.target.value })} placeholder="user" spellCheck={false} /> setF({ ...f, project: e.target.value })} placeholder="project" spellCheck={false} /> setF({ ...f, event: e.target.value })} placeholder="event (sign-in, write, deploy, commit…)" spellCheck={false} /> - setF({ ...f, since: e.target.value })} placeholder="since (2026-10-07T00:00)" spellCheck={false} /> + setF({ ...f, since: e.target.value })} placeholder="since, UTC (2026-10-07T00:00)" spellCheck={false} /> {error &&

{error}
}