From 7da68ecb48e286baa226b2ca28513aabdac263eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Juan=20V=C3=A1squez?= <juan@ombulabs.com>
Date: Wed, 23 Sep 2026 15:58:50 -0600
Subject: [PATCH 1/2] Step 1: Save every benchmark result as JSON, with where
 it ran

Results only existed as text in job logs and as ips.fastruby.io reports,
which do not record the Ruby, OS or machine. When RESULTS_DIR is set,
docker/collect_results.rb (loaded through RUBYOPT, so benchmark files
stay untouched) writes each Benchmark.ips report to
RESULTS_DIR/<label>/<benchmark>.json, with the Ruby, its flags, the run
and the environment: arch, CPU, core count, OS, glibc, compiler and its
flags. It records the PR's own commit and number (on a pull request,
github.sha is a temporary merge commit). A field that cannot be read is null, the others are kept.

CI sets RESULTS_DIR on every benchmark job and uploads the results as
a results-<label> artifact. The results site will read these.

Brought over from the parked prototype (988a455), without the site.
---
 .github/workflows/benchmarks.yml |  15 ++++
 .gitignore                       |   1 +
 CONTRIBUTING.md                  |   8 ++
 compose.yaml                     |   8 ++
 docker/collect_results.rb        | 135 +++++++++++++++++++++++++++++++
 docker/run-benchmarks.sh         |   5 ++
 6 files changed, 172 insertions(+)
 create mode 100644 docker/collect_results.rb

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index a6c9b805..edf5433f 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -42,6 +42,14 @@ jobs:
           'truffleruby_head', 'truffleruby_22'
         ]
 
+    # Read by docker/collect_results.rb.
+    env:
+      RESULTS_DIR: results
+      # On a pull request, github.sha is a temporary merge commit.
+      RESULTS_COMMIT: ${{ github.event.pull_request.head.sha || github.sha }}
+      RESULTS_PR: ${{ github.event.pull_request.number }}
+      RESULTS_LABEL: ${{ matrix.ruby }}
+
     steps:
       - name: Set Share Env
         if: github.ref_name == 'main'
@@ -55,6 +63,13 @@ jobs:
       - name: Report failed benchmarks
         if: failure()
         run: .github/scripts/report-failures.sh benchmarks.log ${{ matrix.ruby }}
+      - name: Upload results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: results-${{ env.RESULTS_LABEL }}
+          path: results/
+          if-no-files-found: warn
 
   # The check to require on main. Passes when every benchmark job passed, or
   # when there was nothing to benchmark.
diff --git a/.gitignore b/.gitignore
index 1cd9aadb..dd761168 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,3 +1,4 @@
 /.bundle/
 *.bundle
 /Gemfile.lock
+/results/
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 28ffaf55..d70352ff 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -60,6 +60,14 @@ the head builds go stale. To get the latest nightly build:
 docker compose build --no-cache ruby_head
 ```
 
+To keep the results, set `RESULTS_DIR`. Each benchmark then also writes its
+report as JSON to `results/<label>/`, with the Ruby, its flags and the machine
+it ran on:
+
+```
+RESULTS_DIR=results RESULTS_LABEL=ruby_3.4 docker compose run --rm ruby_3.4 code/your-new/entry.rb
+```
+
 ## Benchmarks that need a newer Ruby
 
 CI runs every benchmark on every Ruby in `compose.yaml`, back to Ruby 2.1, and
diff --git a/compose.yaml b/compose.yaml
index aa9ea6e8..26b7711a 100644
--- a/compose.yaml
+++ b/compose.yaml
@@ -14,6 +14,14 @@ x-benchmark: &benchmark
     # for code/array/bsearch-vs-find.rb's 100M element array: JRuby 9.1 fails
     # even on 16 GB CI runners, JRuby 10 on smaller machines.
     - JRUBY_OPTS=-J-Xmx6g
+    # Results (docker/collect_results.rb).
+    - RESULTS_DIR
+    - RESULTS_LABEL
+    - RESULTS_COMMIT
+    - RESULTS_PR
+    - GITHUB_RUN_ID
+    - GITHUB_RUN_ATTEMPT
+    - RUNNER_NAME
   entrypoint: ["/app/docker/run-benchmarks.sh"]
 
 services:
diff --git a/docker/collect_results.rb b/docker/collect_results.rb
new file mode 100644
index 00000000..00ee82e3
--- /dev/null
+++ b/docker/collect_results.rb
@@ -0,0 +1,135 @@
+# Loaded with `ruby -r` (through RUBYOPT) when RESULTS_DIR is set. Every
+# Benchmark.ips call in a benchmark file also writes its report as JSON to
+# RESULTS_DIR/<label>/<benchmark>.json, so the benchmark files stay untouched.
+# Each result also records where it ran, so only comparable numbers get
+# compared.
+#
+# Keep this file compatible with the oldest Ruby in the CI matrix (2.1).
+#
+# JRuby 9.1 can load this file before bundler/setup from RUBYOPT, so set up
+# the bundle here to find benchmark-ips either way.
+require "bundler/setup"
+require "benchmark/ips"
+require "fileutils"
+require "json"
+require "rbconfig"
+
+module CollectResults
+  @calls = 0
+
+  class << self
+    attr_accessor :calls
+
+    def label
+      env("RESULTS_LABEL") || ["#{RUBY_ENGINE}-#{engine_version}", env("RUBY_VARIANT")].compact.join("+")
+    end
+
+    def engine_version
+      defined?(RUBY_ENGINE_VERSION) ? RUBY_ENGINE_VERSION : RUBY_VERSION
+    end
+
+    def write(report)
+      self.calls += 1
+      benchmark = $0.sub(%r{\A\./}, "")
+      name = benchmark.sub(%r{\Acode/}, "").sub(/\.rb\z/, "").gsub("/", "--")
+      name += "--#{calls}" if calls > 1
+
+      path = File.join(ENV["RESULTS_DIR"], label, "#{name}.json")
+      FileUtils.mkdir_p(File.dirname(path))
+      File.write(path, JSON.pretty_generate({
+        "benchmark" => benchmark,
+        "part" => calls,
+        "label" => label,
+        "ruby_description" => RUBY_DESCRIPTION,
+        "engine" => RUBY_ENGINE,
+        "engine_version" => engine_version,
+        "ruby_version" => RUBY_VERSION,
+        "variant" => env("RUBY_VARIANT"),
+        "flags" => env("RUBY_VARIANT_FLAGS"),
+        "jit" => jit,
+        "commit" => env("RESULTS_COMMIT"),
+        "pr" => env("RESULTS_PR") && env("RESULTS_PR").to_i,
+        "run_id" => env("GITHUB_RUN_ID"),
+        "run_attempt" => env("GITHUB_RUN_ATTEMPT"),
+        "runner" => env("RUNNER_NAME"),
+        "environment" => environment,
+        "entries" => report.data
+      }))
+    end
+
+    # An empty value counts as unset: CI passes an empty matrix field (no
+    # variant) as an empty string.
+    def env(name)
+      value = ENV[name]
+      value unless value.nil? || value.empty?
+    end
+
+    def jit
+      if defined?(RubyVM::YJIT) && RubyVM::YJIT.enabled?
+        "yjit"
+      elsif defined?(RubyVM::ZJIT) && RubyVM::ZJIT.respond_to?(:enabled?) && RubyVM::ZJIT.enabled?
+        "zjit"
+      end
+    end
+
+    # The same for every benchmark in a run, so work it out once. A field that
+    # cannot be read (no /etc/os-release or ldd outside Docker) is nil; the
+    # others are kept.
+    def environment
+      @environment ||= {
+        "arch" => RbConfig::CONFIG["host_cpu"],
+        "cpu" => safely { cpu },
+        "nproc" => safely { nproc },
+        "os" => safely { File.read("/etc/os-release")[/^PRETTY_NAME="?([^"\n]+)/, 1] },
+        "glibc" => safely { `ldd --version 2>&1`[/\d+\.\d+/] },
+        "cc" => cc_version,
+        "optflags" => RbConfig::CONFIG["optflags"],
+        "configure_args" => RbConfig::CONFIG["configure_args"]
+      }
+    end
+
+    def safely
+      yield
+    rescue SystemCallError
+      nil
+    end
+
+    # x86 lists a model name. arm64 only lists implementer and part codes,
+    # which name the CPU design on servers; inside Docker Desktop's VM on a
+    # Mac they only say "Apple", so different Macs look the same.
+    def cpu
+      model = cpuinfo[/^model name\s*:\s*(.+)$/, 1]
+      return model if model
+
+      implementer = cpuinfo[/^CPU implementer\s*:\s*(\S+)/, 1]
+      part = cpuinfo[/^CPU part\s*:\s*(\S+)/, 1]
+      "CPU implementer #{implementer}, part #{part}" if implementer && part
+    end
+
+    def cpuinfo
+      File.exist?("/proc/cpuinfo") ? File.read("/proc/cpuinfo") : ""
+    end
+
+    # Etc.nprocessors is Ruby 2.2+.
+    def nproc
+      require "etc"
+      Etc.respond_to?(:nprocessors) ? Etc.nprocessors : cpuinfo.scan(/^processor\s*:/).size
+    end
+
+    # CC_VERSION_MESSAGE is only set by newer CRubies; nil elsewhere.
+    def cc_version
+      message = RbConfig::CONFIG["CC_VERSION_MESSAGE"]
+      message && message.lines.first.strip
+    end
+  end
+
+  def ips(*args, &block)
+    report = super
+    CollectResults.write(report) if report.respond_to?(:data)
+    report
+  end
+end
+
+# Benchmark extends Benchmark::IPS, so the hook goes on Benchmark's singleton
+# class. Prepending to Benchmark::IPS would not reach it before Ruby 3.0.
+Benchmark.singleton_class.send(:prepend, CollectResults)
diff --git a/docker/run-benchmarks.sh b/docker/run-benchmarks.sh
index 514efac0..45f455ac 100755
--- a/docker/run-benchmarks.sh
+++ b/docker/run-benchmarks.sh
@@ -10,6 +10,11 @@ export BUNDLE_GEMFILE=/tmp/bundle/Gemfile
 
 bundle install
 
+# Also write every result as JSON, see docker/collect_results.rb.
+if [ -n "$RESULTS_DIR" ]; then
+  export RUBYOPT="-r/app/docker/collect_results.rb $RUBYOPT"
+fi
+
 if [ "$#" -eq 0 ]; then
   exec bundle exec rake
 fi

From f4b5a55414ed8ac1a0328b29e551840510ac516e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Juan=20V=C3=A1squez?= <juan@ombulabs.com>
Date: Wed, 23 Sep 2026 16:12:03 -0600
Subject: [PATCH 2/2] Step 2: Run YJIT and ZJIT variants of CRuby in CI

Every CRuby run was interpreter-only (YJIT is built into 3.2+ but off
by default), while many production apps run YJIT. CI now also runs
ruby_3.1 to ruby_head with --yjit, and ruby_4.0 and ruby_head with
--zjit: 8 more jobs, labeled like ruby_3.4+yjit.

RUBY_VARIANT_FLAGS goes through the variable each engine reads (RUBYOPT,
JRUBY_OPTS, TRUFFLERUBYOPT), since the Rakefile starts one process per
benchmark. The run then stops if the variant is not on, instead of
quietly benchmarking the interpreter.

Only plain Rubies share to ips.fastruby.io: shared reports do not record
the Ruby or its flags, so variant reports would be indistinguishable.
---
 .github/workflows/benchmarks.yml | 28 ++++++++++++++++++++++------
 CONTRIBUTING.md                  |  8 ++++++++
 compose.yaml                     |  4 +++-
 docker/run-benchmarks.sh         | 28 ++++++++++++++++++++++++++++
 4 files changed, 61 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
index edf5433f..72bcaf36 100644
--- a/.github/workflows/benchmarks.yml
+++ b/.github/workflows/benchmarks.yml
@@ -25,6 +25,7 @@ jobs:
         run: .github/scripts/pick-benchmarks.sh "${{ github.event_name == 'pull_request' && 'HEAD^1' || github.event.before }}"
 
   rake:
+    name: rake (${{ matrix.ruby }}${{ matrix.variant && format('+{0}', matrix.variant) || '' }})
     needs: changes
     if: needs.changes.outputs.run == 'true'
     runs-on: ubuntu-latest
@@ -41,28 +42,43 @@ jobs:
           'jruby_head', 'jruby_9.1',
           'truffleruby_head', 'truffleruby_22'
         ]
+        # '' is the plain Ruby. Each include below overwrites it, so GitHub
+        # adds it as a separate job instead of merging it into the plain one.
+        variant: ['']
+        include:
+          - { ruby: ruby_3.1, variant: yjit, flags: --yjit }
+          - { ruby: ruby_3.2, variant: yjit, flags: --yjit }
+          - { ruby: ruby_3.3, variant: yjit, flags: --yjit }
+          - { ruby: ruby_3.4, variant: yjit, flags: --yjit }
+          - { ruby: ruby_4.0, variant: yjit, flags: --yjit }
+          - { ruby: ruby_head, variant: yjit, flags: --yjit }
+          - { ruby: ruby_4.0, variant: zjit, flags: --zjit }
+          - { ruby: ruby_head, variant: zjit, flags: --zjit }
 
-    # Read by docker/collect_results.rb.
+    # Read by docker/run-benchmarks.sh and docker/collect_results.rb.
     env:
+      RUBY_VARIANT: ${{ matrix.variant }}
+      RUBY_VARIANT_FLAGS: ${{ matrix.flags }}
       RESULTS_DIR: results
-      # On a pull request, github.sha is a temporary merge commit.
       RESULTS_COMMIT: ${{ github.event.pull_request.head.sha || github.sha }}
       RESULTS_PR: ${{ github.event.pull_request.number }}
-      RESULTS_LABEL: ${{ matrix.ruby }}
+      RESULTS_LABEL: ${{ matrix.ruby }}${{ matrix.variant && format('+{0}', matrix.variant) || '' }}
 
     steps:
+      # Plain Rubies only: shared reports do not record the Ruby or its flags,
+      # so variant runs would be indistinguishable on ips.fastruby.io.
       - name: Set Share Env
-        if: github.ref_name == 'main'
+        if: github.ref_name == 'main' && matrix.variant == ''
         run: |
           echo "SHARE=1" >> "$GITHUB_ENV"
       - uses: actions/checkout@v4
-      - name: Run benchmarks on ${{ matrix.ruby }}
+      - name: Run benchmarks on ${{ env.RESULTS_LABEL }}
         env:
           FILES: ${{ needs.changes.outputs.files }}
         run: .github/scripts/run-benchmarks.sh ${{ matrix.ruby }}
       - name: Report failed benchmarks
         if: failure()
-        run: .github/scripts/report-failures.sh benchmarks.log ${{ matrix.ruby }}
+        run: .github/scripts/report-failures.sh benchmarks.log ${{ env.RESULTS_LABEL }}
       - name: Upload results
         uses: actions/upload-artifact@v4
         if: always()
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index d70352ff..a880650f 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -60,6 +60,14 @@ the head builds go stale. To get the latest nightly build:
 docker compose build --no-cache ruby_head
 ```
 
+To run it with a JIT, pass the variant and its flags. The run stops if the
+Ruby does not have that JIT, instead of quietly running without it:
+
+```
+RUBY_VARIANT=yjit RUBY_VARIANT_FLAGS=--yjit docker compose run --rm ruby_3.4 code/your-new/entry.rb
+RUBY_VARIANT=zjit RUBY_VARIANT_FLAGS=--zjit docker compose run --rm ruby_4.0 code/your-new/entry.rb
+```
+
 To keep the results, set `RESULTS_DIR`. Each benchmark then also writes its
 report as JSON to `results/<label>/`, with the Ruby, its flags and the machine
 it ran on:
diff --git a/compose.yaml b/compose.yaml
index 26b7711a..224c2824 100644
--- a/compose.yaml
+++ b/compose.yaml
@@ -14,7 +14,9 @@ x-benchmark: &benchmark
     # for code/array/bsearch-vs-find.rb's 100M element array: JRuby 9.1 fails
     # even on 16 GB CI runners, JRuby 10 on smaller machines.
     - JRUBY_OPTS=-J-Xmx6g
-    # Results (docker/collect_results.rb).
+    # Variants (docker/run-benchmarks.sh) and results (docker/collect_results.rb).
+    - RUBY_VARIANT
+    - RUBY_VARIANT_FLAGS
     - RESULTS_DIR
     - RESULTS_LABEL
     - RESULTS_COMMIT
diff --git a/docker/run-benchmarks.sh b/docker/run-benchmarks.sh
index 45f455ac..9f87ef7d 100755
--- a/docker/run-benchmarks.sh
+++ b/docker/run-benchmarks.sh
@@ -10,6 +10,34 @@ export BUNDLE_GEMFILE=/tmp/bundle/Gemfile
 
 bundle install
 
+# A variant runs the same Ruby with flags, for example YJIT. The Rakefile starts
+# one process per benchmark, so the flags go through the variable each engine
+# reads, which every child process inherits.
+if [ -n "$RUBY_VARIANT" ] || [ -n "$RUBY_VARIANT_FLAGS" ]; then
+  engine=$(ruby -e 'print RUBY_ENGINE')
+  if [ -n "$RUBY_VARIANT_FLAGS" ]; then
+    case "$engine" in
+      jruby) export JRUBY_OPTS="$JRUBY_OPTS $RUBY_VARIANT_FLAGS" ;;
+      truffleruby) export TRUFFLERUBYOPT="$RUBY_VARIANT_FLAGS $TRUFFLERUBYOPT" ;;
+      *) export RUBYOPT="$RUBY_VARIANT_FLAGS $RUBYOPT" ;;
+    esac
+  fi
+
+  # Fail here rather than silently benchmark without the variant. TruffleRuby
+  # prints nothing for its flags, so only an invalid one fails there.
+  description=$(ruby -e 'print RUBY_DESCRIPTION')
+  case "$engine:$RUBY_VARIANT" in
+    ruby:yjit) expected="+YJIT" ;;
+    ruby:zjit) expected="+ZJIT" ;;
+    jruby:dev|jruby:nojit) expected="-jit" ;;
+    *) expected="" ;;
+  esac
+  case "$description" in
+    *"$expected"*) echo "Variant $RUBY_VARIANT: $description" ;;
+    *) echo "Variant $RUBY_VARIANT is not on: $description" >&2; exit 1 ;;
+  esac
+fi
+
 # Also write every result as JSON, see docker/collect_results.rb.
 if [ -n "$RESULTS_DIR" ]; then
   export RUBYOPT="-r/app/docker/collect_results.rb $RUBYOPT"