T-21168 Add an E2E test of the example project on the oldest, newest and pre-release Go

.github/actions/cleanup-sources/action.yml

@@ -0,0 +1,33 @@
+# Deletes the run's source and stale sources of earlier runs of the same test, and never fails the job.
+# The logic is in run.sh, which reads only environment variables, so it runs the same way outside of Actions.
+name: "Clean up Better Stack sources"
+description: "Deletes this run's source and sources of earlier runs, named with the same prefix, that are older than stale-hours"
+
+inputs:
+  api-token:
+    description: "Telemetry API token of the team the sources are in"
+    required: true
+  prefix:
+    description: "Name prefix of this test's sources, only those are swept"
+    required: true
+  id:
+    description: "Id of this run's source, deleted whatever its age"
+    default: ""
+  stale-hours:
+    description: "Sources with the prefix older than this many hours are left over from earlier runs"
+    default: "3"
+  out:
+    description: "Directory for cleanup.md"
+    default: out
+
+runs:
+  using: composite
+  steps:
+    - shell: bash
+      env:
+        INPUT_API_TOKEN: ${{ inputs.api-token }}
+        INPUT_PREFIX: ${{ inputs.prefix }}
+        INPUT_ID: ${{ inputs.id }}
+        INPUT_STALE_HOURS: ${{ inputs.stale-hours }}
+        INPUT_OUT: ${{ inputs.out }}
+      run: bash "${{ github.action_path }}/run.sh"

.github/actions/cleanup-sources/run.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# Deletes this run's source, then what earlier runs of the same test left behind (a cancelled job never reaches its
+# cleanup): every source whose name starts with the prefix and that is older than stale-hours. Younger sources may
+# belong to a run still in progress and stay. Writes <out>/cleanup.md, one line per deletion, and always exits 0:
+# a failed deletion shows in the report, but never hides the test result.
+# Env: INPUT_API_TOKEN, INPUT_PREFIX, INPUT_ID (optional), INPUT_STALE_HOURS (default 3), INPUT_OUT (default out).
+set -euo pipefail
+# A native jq.exe, as on Windows runners, ends its output lines with CRLF unless given -b.
+if [ "${OS:-}" = Windows_NT ]; then jq() { command jq -b "$@"; }; fi
+
+API=https://telemetry.betterstack.com
+OUT=${INPUT_OUT:-out}
+mkdir -p "$OUT"
+: > "$OUT/cleanup.md"
+say() { echo "$1"; echo "$1" >> "$OUT/cleanup.md"; }
+trap 'status=$?; [ "$status" = 0 ] || say "- ❌ cleanup stopped on an error, see the log of this step"; exit 0' EXIT
+
+delete() {
+  local code
+  code=$(curl -sS -m 60 -o /dev/null -w '%{http_code}' -X DELETE "$API/api/v1/sources/$1" -H "Authorization: Bearer $INPUT_API_TOKEN") || code=000
+  case "$code" in
+    204) say "- deleted $2" ;;
+    404) say "- $2 was already gone" ;;
+    *) say "- ❌ could not delete $2 (HTTP $code)" ;;
+  esac
+}
+
+say "### Cleanup"
+say ""
+[ -z "${INPUT_ID:-}" ] || delete "$INPUT_ID" "this run's source $INPUT_ID"
+
+# An empty prefix would sweep every source of the team.
+[ -n "$INPUT_PREFIX" ] || { say "- ❌ no prefix given, so stale sources were not looked for"; exit 0; }
+# The name filter matches anywhere in the name, so the prefix is checked again. The next-page link the API returns
+# drops the filter, so pages are counted here instead of following it.
+STALE=""
+for ((page = 1; ; page++)); do
+  RES=$(curl -sS -m 60 -G "$API/api/v1/sources" --data-urlencode "name=$INPUT_PREFIX" -d per_page=250 -d "page=$page" -H "Authorization: Bearer $INPUT_API_TOKEN") || RES=""
+  jq -e '.data | type == "array"' <<<"$RES" >/dev/null 2>&1 || { say "- ❌ could not list sources, so stale ones were not deleted: $(head -c 300 <<<"${RES:-no response}")"; STALE=""; break; }
+  STALE+=$(jq -r --arg prefix "$INPUT_PREFIX" --arg id "${INPUT_ID:-}" --argjson hours "${INPUT_STALE_HOURS:-3}" '.data[]
+    | select((.attributes.name | startswith($prefix)) and .id != $id)
+    | select((.attributes.created_at | sub("\\.[0-9]+"; "") | fromdateiso8601) < now - $hours * 3600)
+    | "\(.id) \(.attributes.created_at) \(.attributes.name)"' <<<"$RES")$'\n'
+  [ "$(jq -r '.pagination.next' <<<"$RES")" != null ] || break
+done
+while read -r id created_at name; do
+  [ -n "$id" ] || continue
+  delete "$id" "stale source $id \`$name\` (created $created_at)"
+done <<<"$STALE"
+
+[ "$(wc -l < "$OUT/cleanup.md")" -gt 2 ] || say "- nothing to delete"

.github/actions/create-source/action.yml

@@ -0,0 +1,46 @@
+# Creates a Better Stack source through the Telemetry API and waits until ingest reliably accepts its token.
+# The logic is in run.sh, which reads only environment variables, so it runs the same way outside of Actions.
+name: "Create a Better Stack source"
+description: "Creates a source, warms it up until ingest accepts its token and outputs where to send and read logs"
+
+inputs:
+  api-token:
+    description: "Telemetry API token of the team the source is created in"
+    required: true
+  name:
+    description: "Name of the new source"
+    required: true
+  platform:
+    description: "Platform of the new source, as the Telemetry API names it"
+    required: true
+
+outputs:
+  id:
+    description: "Source id"
+    value: ${{ steps.run.outputs.id }}
+  token:
+    description: "Source token for ingest (masked in the logs)"
+    value: ${{ steps.run.outputs.token }}
+  ingesting-host:
+    description: "Host to send logs to"
+    value: ${{ steps.run.outputs.ingesting-host }}
+  data-region:
+    description: "Data region the source stores its logs in"
+    value: ${{ steps.run.outputs.data-region }}
+  collection:
+    description: "Collection name for the SQL API, t<team id>_<table name>"
+    value: ${{ steps.run.outputs.collection }}
+  connect-host:
+    description: "SQL API host that reads this source"
+    value: ${{ steps.run.outputs.connect-host }}
+
+runs:
+  using: composite
+  steps:
+    - id: run
+      shell: bash
+      env:
+        INPUT_API_TOKEN: ${{ inputs.api-token }}
+        INPUT_NAME: ${{ inputs.name }}
+        INPUT_PLATFORM: ${{ inputs.platform }}
+      run: bash "${{ github.action_path }}/run.sh"

.github/actions/create-source/run.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# Creates a source and warms it up. A fresh source drops what it receives in its first seconds, and every ingest
+# process picks up a new token on its own schedule, so one accepted line proves nothing: the source counts as ready
+# after 20 probe lines in a row were accepted, plus a short pause to settle. Probe lines carry the message
+# "e2e warm-up <source name>", which verify-logs ignores.
+# Env: INPUT_API_TOKEN, INPUT_NAME, INPUT_PLATFORM, GITHUB_OUTPUT.
+set -euo pipefail
+# A native jq.exe, as on Windows runners, ends its output lines with CRLF unless given -b.
+if [ "${OS:-}" = Windows_NT ]; then jq() { command jq -b "$@"; }; fi
+
+API=https://telemetry.betterstack.com
+
+BODY=$(jq -nc --arg name "$INPUT_NAME" --arg platform "$INPUT_PLATFORM" '{name: $name, platform: $platform}')
+RES=$(curl -sS -m 60 -X POST "$API/api/v1/sources" -H "Authorization: Bearer $INPUT_API_TOKEN" -H 'Content-Type: application/json' -d "$BODY") || true
+ID=$(jq -er '.data.id' <<<"$RES" 2>/dev/null) || { echo "::error::creating source $INPUT_NAME failed: ${RES:-no response}"; exit 1; }
+TOKEN=$(jq -r '.data.attributes.token' <<<"$RES")
+INGESTING_HOST=$(jq -r '.data.attributes.ingesting_host' <<<"$RES")
+DATA_REGION=$(jq -r '.data.attributes.data_region' <<<"$RES")
+echo "::add-mask::$TOKEN"
+{
+  echo "id=$ID"
+  echo "token=$TOKEN"
+  echo "ingesting-host=$INGESTING_HOST"
+  echo "data-region=$DATA_REGION"
+  echo "collection=t$(jq -r '.data.attributes.team_id' <<<"$RES")_$(jq -r '.data.attributes.table_name' <<<"$RES")"
+  echo "connect-host=$DATA_REGION-connect.betterstackdata.com"
+} >> "$GITHUB_OUTPUT"
+echo "created source $ID \`$INPUT_NAME\` in data region $DATA_REGION"
+
+PROBE=$(jq -nc --arg message "e2e warm-up $INPUT_NAME" '{message: $message}')
+STREAK=0
+DEADLINE=$((SECONDS + 300))
+while [ "$STREAK" -lt 20 ]; do
+  CODE=$(curl -sS -m 10 -o /dev/null -w '%{http_code}' -X POST "https://$INGESTING_HOST/" -H "Authorization: Bearer $TOKEN" -H 'Content-Type: application/json' -d "$PROBE") || true
+  if [ "$CODE" = 202 ]; then STREAK=$((STREAK + 1)); else echo "ingest answered HTTP $CODE after $STREAK accepted in a row"; STREAK=0; fi
+  [ "$SECONDS" -lt "$DEADLINE" ] || { echo "::error::ingest did not accept the token of source $ID 20 times in a row within 5 min"; exit 1; }
+  sleep 1
+done
+sleep 15
+echo "ingest accepted 20 probe lines in a row, source $ID is ready"

.github/actions/verify-logs/action.yml

@@ -0,0 +1,47 @@
+# Reads a source's logs back through the team's SQL API connection and checks them exactly against an expected file.
+# The logic is in run.sh, which reads only environment variables, so it runs the same way outside of Actions.
+name: "Verify stored logs"
+description: "Polls a source's stored logs until the expected rows arrive, then checks each one is stored exactly once"
+
+inputs:
+  connection-username:
+    description: "Username of the team's SQL API connection"
+    required: true
+  connection-password:
+    description: "Password of the team's SQL API connection"
+    required: true
+  connect-host:
+    description: "SQL API host, the connect-host output of create-source"
+    required: true
+  collection:
+    description: "Collection to read, the collection output of create-source"
+    required: true
+  expected:
+    description: >-
+      Path to a JSON array of the rows that must be stored, each {"level": ..., "message": ...} or
+      {"level": ..., "message_prefix": ...}, optionally with "contains": an object the stored row must contain
+    required: true
+  out:
+    description: "Directory for rows.json and summary.md"
+    default: out
+  deadline-seconds:
+    description: "How long to wait for the expected number of rows"
+    default: "300"
+  ignore-message-prefix:
+    description: "Rows whose message starts with this are not counted (the warm-up lines of create-source)"
+    default: "e2e warm-up"
+
+runs:
+  using: composite
+  steps:
+    - shell: bash
+      env:
+        INPUT_CONNECTION_USERNAME: ${{ inputs.connection-username }}
+        INPUT_CONNECTION_PASSWORD: ${{ inputs.connection-password }}
+        INPUT_CONNECT_HOST: ${{ inputs.connect-host }}
+        INPUT_COLLECTION: ${{ inputs.collection }}
+        INPUT_EXPECTED: ${{ inputs.expected }}
+        INPUT_OUT: ${{ inputs.out }}
+        INPUT_DEADLINE_SECONDS: ${{ inputs.deadline-seconds }}
+        INPUT_IGNORE_MESSAGE_PREFIX: ${{ inputs.ignore-message-prefix }}
+      run: bash "${{ github.action_path }}/run.sh"

.github/actions/verify-logs/run.sh

@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+# Reads a source's logs back through the team's SQL API connection and checks them against an expected file: as
+# many rows as expected, each expected entry matched by exactly one row, no row left unmatched. Rows land in hot
+# storage first and are copied to cold storage later, so both are read and a cold row counts only when hot storage
+# lacks it. Once enough rows arrived it reads again 30 s later, so a duplicate delivery fails instead of slipping
+# through. Writes <out>/rows.json and <out>/summary.md on every exit path: the summary is what a failure report shows.
+# Env: INPUT_CONNECTION_USERNAME, INPUT_CONNECTION_PASSWORD, INPUT_CONNECT_HOST, INPUT_COLLECTION, INPUT_EXPECTED,
+#      INPUT_OUT (default out), INPUT_DEADLINE_SECONDS (default 300), INPUT_IGNORE_MESSAGE_PREFIX (default "e2e warm-up").
+set -euo pipefail
+# A native jq.exe, as on Windows runners, ends its output lines with CRLF unless given -b.
+if [ "${OS:-}" = Windows_NT ]; then jq() { command jq -b "$@"; }; fi
+
+OUT=${INPUT_OUT:-out}
+IGNORE=${INPUT_IGNORE_MESSAGE_PREFIX-e2e warm-up}
+# jq: the warm-up lines of create-source, rows whose message starts with the ignore prefix.
+IGNORED='def ignored: $ignore != "" and (.raw.message | type) == "string" and (.raw.message | startswith($ignore));'
+CHECKS=""
+FAILURE=""
+COUNTS=""
+mkdir -p "$OUT"
+echo '[]' > "$OUT/rows.json"
+
+fail() { FAILURE="$1"; echo "::error::$1"; exit 1; }
+
+summarize() {
+  local status=$?
+  {
+    echo "Read back through \`$INPUT_CONNECT_HOST\` from \`$INPUT_COLLECTION\`${COUNTS:+: $COUNTS}"
+    echo
+    if [ -n "$CHECKS" ]; then printf '%s' "$CHECKS"; else echo "- ❌ ${FAILURE:-stopped on an error, see the log of this step}"; fi
+    if [ "$(jq length "$OUT/rows.json")" -gt 0 ]; then
+      echo
+      echo "<details><summary>One stored row</summary>"
+      echo
+      echo '```json'
+      jq --arg ignore "$IGNORE" "$IGNORED"' (map(select(ignored | not)) | max_by(.raw | keys | length)) // .[0]' "$OUT/rows.json"
+      echo '```'
+      echo
+      echo "</details>"
+    fi
+  } > "$OUT/summary.md"
+  exit "$status"
+}
+trap summarize EXIT
+
+jq -e 'type == "array" and length > 0 and all(.[]; type == "object" and has("level") and (has("message") or has("message_prefix")))' "$INPUT_EXPECTED" >/dev/null 2>&1 \
+  || fail "$INPUT_EXPECTED is not a JSON array of {\"level\", \"message\" or \"message_prefix\", optional \"contains\"} objects"
+WANT=$(jq length "$INPUT_EXPECTED")
+
+# Writes the rows as JSONEachRow into $2, or nothing while the collection does not exist: it is created with the
+# source's first stored row.
+sql() {
+  local code
+  code=$(curl -sS -m 60 -o "$2" -w '%{http_code}' -u "$INPUT_CONNECTION_USERNAME:$INPUT_CONNECTION_PASSWORD" \
+    -H 'Content-Type: plain/text' --data-binary "$1 FORMAT JSONEachRow" "https://$INPUT_CONNECT_HOST/") || code=000
+  if [ "$code" = 500 ] && grep -qE 'CLUSTER_DOESNT_EXIST|NAMED_COLLECTION_DOESNT_EXIST|UNKNOWN_TABLE' "$2"; then : > "$2"; return; fi
+  [ "$code" = 200 ] || fail "the SQL API on $INPUT_CONNECT_HOST answered HTTP $code: $(head -c 300 "$2" 2>/dev/null)"
+}
+
+read_rows() {
+  sql "SELECT dt, raw FROM remote(${INPUT_COLLECTION}_logs)" "$OUT/hot.ndjson"
+  sql "SELECT dt, raw FROM s3Cluster(primary, ${INPUT_COLLECTION}_s3) WHERE _row_type = 1" "$OUT/cold.ndjson"
+  jq -n --slurpfile hot "$OUT/hot.ndjson" --slurpfile cold "$OUT/cold.ndjson" \
+    '($hot | map({dt, raw})) as $h | $h + ($cold | map({dt, raw}) | map(select(IN($h[]) | not))) | map(.raw |= fromjson) | sort_by(.dt)' > "$OUT/rows.json"
+  local all hot
+  all=$(jq length "$OUT/rows.json")
+  hot=$(jq -s length "$OUT/hot.ndjson")
+  GOT=$(jq --arg ignore "$IGNORE" "$IGNORED"' map(select(ignored | not)) | length' "$OUT/rows.json")
+  COUNTS="$hot rows in hot storage, $((all - hot)) more in cold storage, $((all - GOT)) of them ignored as \"$IGNORE\" lines"
+}
+
+DEADLINE=$((SECONDS + ${INPUT_DEADLINE_SECONDS:-300}))
+while :; do
+  read_rows
+  [ "$GOT" -lt "$WANT" ] || break
+  [ "$SECONDS" -lt "$DEADLINE" ] || { echo "deadline passed with $GOT of $WANT rows"; break; }
+  echo "$GOT of $WANT rows so far, reading again in 10 s"
+  sleep 10
+done
+echo "$GOT of $WANT rows, reading once more in 30 s to catch duplicate delivery"
+sleep 30
+read_rows
+
+# One check per line: the row count, each expected entry (exactly one row matches it), each row no entry matches.
+# "contains" is jq's contains: objects and arrays may hold more than expected, strings match as substrings.
+RESULT=$(jq -c --arg ignore "$IGNORE" --slurpfile expected "$INPUT_EXPECTED" "$IGNORED"'
+  def text: tostring | gsub("\\s+"; " ") | if length > 100 then .[0:100] + "..." else . end;
+  def message_matches($e): (.message | type) == "string" and if $e | has("message") then .message == $e.message else .message | startswith($e.message_prefix) end;
+  def matches($e): .level == $e.level and message_matches($e) and (($e | has("contains") | not) or contains($e.contains));
+  def describe: "`\(.level)` " + (if has("message") then "\"\(.message | text)\"" else "message starting with \"\(.message_prefix | text)\"" end)
+    + (if has("contains") then " containing `\(.contains | tojson | text)`" else "" end);
+  def near($e): [.[] | select(message_matches($e)) | if .level != $e.level then "a row with this message has level `\(.level)`" else "a row with this message lacks part of what it should contain" end] | unique;
+  map(select(ignored | not) | .raw) as $rows | $expected[0] as $expected
+  | [{ok: (($rows | length) == ($expected | length)), text: "\($rows | length) rows stored, \($expected | length) expected"}]
+  + [$expected[] as $e | ([$rows[] | select(matches($e))] | length) as $n | ($rows | near($e)) as $near
+      | {ok: ($n == 1), text: (($e | describe) + if $n == 1 then " stored once" elif $n > 1 then " stored \($n) times"
+          elif $near == [] then " missing" else " missing (\($near | join(", ")))" end)}]
+  + [$rows[] | select(. as $r | any($expected[]; . as $e | $r | matches($e)) | not) | {ok: false, text: "unexpected row: `\(.level)` \"\(.message | text)\""}]
+' "$OUT/rows.json")
+CHECKS=$(jq -r '.[] | "- \(if .ok then "✅" else "❌" end) \(.text)"' <<<"$RESULT")$'\n'
+printf '%s' "$CHECKS"
+PROBLEMS=$(jq -r 'map(select(.ok | not) | .text) | join("; ")' <<<"$RESULT")
+[ -z "$PROBLEMS" ] || fail "the stored rows differ from $INPUT_EXPECTED: $PROBLEMS"
+echo "every expected row is stored exactly once"