Save results as JSON and run YJIT and ZJIT variants

.github/workflows/benchmarks.yml

@@ -25,6 +25,7 @@ jobs:
         run: .github/scripts/pick-benchmarks.sh "${{ github.event_name == 'pull_request' && 'HEAD^1' || github.event.before }}"
 
   rake:
+    name: rake (${{ matrix.ruby }}${{ matrix.variant && format('+{0}', matrix.variant) || '' }})
     needs: changes
     if: needs.changes.outputs.run == 'true'
     runs-on: ubuntu-latest
@@ -41,20 +42,50 @@ jobs:
           'jruby_head', 'jruby_9.1',
           'truffleruby_head', 'truffleruby_22'
         ]
+        # '' is the plain Ruby. Each include below overwrites it, so GitHub
+        # adds it as a separate job instead of merging it into the plain one.
+        variant: ['']
+        include:
+          - { ruby: ruby_3.1, variant: yjit, flags: --yjit }
+          - { ruby: ruby_3.2, variant: yjit, flags: --yjit }
+          - { ruby: ruby_3.3, variant: yjit, flags: --yjit }
+          - { ruby: ruby_3.4, variant: yjit, flags: --yjit }
+          - { ruby: ruby_4.0, variant: yjit, flags: --yjit }
+          - { ruby: ruby_head, variant: yjit, flags: --yjit }
+          - { ruby: ruby_4.0, variant: zjit, flags: --zjit }
+          - { ruby: ruby_head, variant: zjit, flags: --zjit }
+
+    # Read by docker/run-benchmarks.sh and docker/collect_results.rb.
+    env:
+      RUBY_VARIANT: ${{ matrix.variant }}
+      RUBY_VARIANT_FLAGS: ${{ matrix.flags }}
+      RESULTS_DIR: results
+      RESULTS_COMMIT: ${{ github.event.pull_request.head.sha || github.sha }}
+      RESULTS_PR: ${{ github.event.pull_request.number }}
+      RESULTS_LABEL: ${{ matrix.ruby }}${{ matrix.variant && format('+{0}', matrix.variant) || '' }}
 
     steps:
+      # Plain Rubies only: shared reports do not record the Ruby or its flags,
+      # so variant runs would be indistinguishable on ips.fastruby.io.
       - name: Set Share Env
-        if: github.ref_name == 'main'
+        if: github.ref_name == 'main' && matrix.variant == ''
         run: |
           echo "SHARE=1" >> "$GITHUB_ENV"
       - uses: actions/checkout@v4
-      - name: Run benchmarks on ${{ matrix.ruby }}
+      - name: Run benchmarks on ${{ env.RESULTS_LABEL }}
         env:
           FILES: ${{ needs.changes.outputs.files }}
         run: .github/scripts/run-benchmarks.sh ${{ matrix.ruby }}
       - name: Report failed benchmarks
         if: failure()
-        run: .github/scripts/report-failures.sh benchmarks.log ${{ matrix.ruby }}
+        run: .github/scripts/report-failures.sh benchmarks.log ${{ env.RESULTS_LABEL }}
+      - name: Upload results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: results-${{ env.RESULTS_LABEL }}
+          path: results/
+          if-no-files-found: warn
 
   # The check to require on main. Passes when every benchmark job passed, or
   # when there was nothing to benchmark.

.gitignore

@@ -1,3 +1,4 @@
 /.bundle/
 *.bundle
 /Gemfile.lock
+/results/

CONTRIBUTING.md

@@ -60,6 +60,22 @@ the head builds go stale. To get the latest nightly build:
 docker compose build --no-cache ruby_head
 ```
 
+To run it with a JIT, pass the variant and its flags. The run stops if the
+Ruby does not have that JIT, instead of quietly running without it:
+
+```
+RUBY_VARIANT=yjit RUBY_VARIANT_FLAGS=--yjit docker compose run --rm ruby_3.4 code/your-new/entry.rb
+RUBY_VARIANT=zjit RUBY_VARIANT_FLAGS=--zjit docker compose run --rm ruby_4.0 code/your-new/entry.rb
+```
+
+To keep the results, set `RESULTS_DIR`. Each benchmark then also writes its
+report as JSON to `results/<label>/`, with the Ruby, its flags and the machine
+it ran on:
+
+```
+RESULTS_DIR=results RESULTS_LABEL=ruby_3.4 docker compose run --rm ruby_3.4 code/your-new/entry.rb
+```
+
 ## Benchmarks that need a newer Ruby
 
 CI runs every benchmark on every Ruby in `compose.yaml`, back to Ruby 2.1, and

compose.yaml

@@ -14,6 +14,16 @@ x-benchmark: &benchmark
     # for code/array/bsearch-vs-find.rb's 100M element array: JRuby 9.1 fails
     # even on 16 GB CI runners, JRuby 10 on smaller machines.
     - JRUBY_OPTS=-J-Xmx6g
+    # Variants (docker/run-benchmarks.sh) and results (docker/collect_results.rb).
+    - RUBY_VARIANT
+    - RUBY_VARIANT_FLAGS
+    - RESULTS_DIR
+    - RESULTS_LABEL
+    - RESULTS_COMMIT
+    - RESULTS_PR
+    - GITHUB_RUN_ID
+    - GITHUB_RUN_ATTEMPT
+    - RUNNER_NAME
   entrypoint: ["/app/docker/run-benchmarks.sh"]
 
 services:

docker/collect_results.rb

@@ -0,0 +1,135 @@
+# Loaded with `ruby -r` (through RUBYOPT) when RESULTS_DIR is set. Every
+# Benchmark.ips call in a benchmark file also writes its report as JSON to
+# RESULTS_DIR/<label>/<benchmark>.json, so the benchmark files stay untouched.
+# Each result also records where it ran, so only comparable numbers get
+# compared.
+#
+# Keep this file compatible with the oldest Ruby in the CI matrix (2.1).
+#
+# JRuby 9.1 can load this file before bundler/setup from RUBYOPT, so set up
+# the bundle here to find benchmark-ips either way.
+require "bundler/setup"
+require "benchmark/ips"
+require "fileutils"
+require "json"
+require "rbconfig"
+
+module CollectResults
+  @calls = 0
+
+  class << self
+    attr_accessor :calls
+
+    def label
+      env("RESULTS_LABEL") || ["#{RUBY_ENGINE}-#{engine_version}", env("RUBY_VARIANT")].compact.join("+")
+    end
+
+    def engine_version
+      defined?(RUBY_ENGINE_VERSION) ? RUBY_ENGINE_VERSION : RUBY_VERSION
+    end
+
+    def write(report)
+      self.calls += 1
+      benchmark = $0.sub(%r{\A\./}, "")
+      name = benchmark.sub(%r{\Acode/}, "").sub(/\.rb\z/, "").gsub("/", "--")
+      name += "--#{calls}" if calls > 1
+
+      path = File.join(ENV["RESULTS_DIR"], label, "#{name}.json")
+      FileUtils.mkdir_p(File.dirname(path))
+      File.write(path, JSON.pretty_generate({
+        "benchmark" => benchmark,
+        "part" => calls,
+        "label" => label,
+        "ruby_description" => RUBY_DESCRIPTION,
+        "engine" => RUBY_ENGINE,
+        "engine_version" => engine_version,
+        "ruby_version" => RUBY_VERSION,
+        "variant" => env("RUBY_VARIANT"),
+        "flags" => env("RUBY_VARIANT_FLAGS"),
+        "jit" => jit,
+        "commit" => env("RESULTS_COMMIT"),
+        "pr" => env("RESULTS_PR") && env("RESULTS_PR").to_i,
+        "run_id" => env("GITHUB_RUN_ID"),
+        "run_attempt" => env("GITHUB_RUN_ATTEMPT"),
+        "runner" => env("RUNNER_NAME"),
+        "environment" => environment,
+        "entries" => report.data
+      }))
+    end
+
+    # An empty value counts as unset: CI passes an empty matrix field (no
+    # variant) as an empty string.
+    def env(name)
+      value = ENV[name]
+      value unless value.nil? || value.empty?
+    end
+
+    def jit
+      if defined?(RubyVM::YJIT) && RubyVM::YJIT.enabled?
+        "yjit"
+      elsif defined?(RubyVM::ZJIT) && RubyVM::ZJIT.respond_to?(:enabled?) && RubyVM::ZJIT.enabled?
+        "zjit"
+      end
+    end
+
+    # The same for every benchmark in a run, so work it out once. A field that
+    # cannot be read (no /etc/os-release or ldd outside Docker) is nil; the
+    # others are kept.
+    def environment
+      @environment ||= {
+        "arch" => RbConfig::CONFIG["host_cpu"],
+        "cpu" => safely { cpu },
+        "nproc" => safely { nproc },
+        "os" => safely { File.read("/etc/os-release")[/^PRETTY_NAME="?([^"\n]+)/, 1] },
+        "glibc" => safely { `ldd --version 2>&1`[/\d+\.\d+/] },
+        "cc" => cc_version,
+        "optflags" => RbConfig::CONFIG["optflags"],
+        "configure_args" => RbConfig::CONFIG["configure_args"]
+      }
+    end
+
+    def safely
+      yield
+    rescue SystemCallError
+      nil
+    end
+
+    # x86 lists a model name. arm64 only lists implementer and part codes,
+    # which name the CPU design on servers; inside Docker Desktop's VM on a
+    # Mac they only say "Apple", so different Macs look the same.
+    def cpu
+      model = cpuinfo[/^model name\s*:\s*(.+)$/, 1]
+      return model if model
+
+      implementer = cpuinfo[/^CPU implementer\s*:\s*(\S+)/, 1]
+      part = cpuinfo[/^CPU part\s*:\s*(\S+)/, 1]
+      "CPU implementer #{implementer}, part #{part}" if implementer && part
+    end
+
+    def cpuinfo
+      File.exist?("/proc/cpuinfo") ? File.read("/proc/cpuinfo") : ""
+    end
+
+    # Etc.nprocessors is Ruby 2.2+.
+    def nproc
+      require "etc"
+      Etc.respond_to?(:nprocessors) ? Etc.nprocessors : cpuinfo.scan(/^processor\s*:/).size
+    end
+
+    # CC_VERSION_MESSAGE is only set by newer CRubies; nil elsewhere.
+    def cc_version
+      message = RbConfig::CONFIG["CC_VERSION_MESSAGE"]
+      message && message.lines.first.strip
+    end
+  end
+
+  def ips(*args, &block)
+    report = super
+    CollectResults.write(report) if report.respond_to?(:data)
+    report
+  end
+end
+
+# Benchmark extends Benchmark::IPS, so the hook goes on Benchmark's singleton
+# class. Prepending to Benchmark::IPS would not reach it before Ruby 3.0.
+Benchmark.singleton_class.send(:prepend, CollectResults)

docker/run-benchmarks.sh

@@ -10,6 +10,39 @@ export BUNDLE_GEMFILE=/tmp/bundle/Gemfile
 
 bundle install
 
+# A variant runs the same Ruby with flags, for example YJIT. The Rakefile starts
+# one process per benchmark, so the flags go through the variable each engine
+# reads, which every child process inherits.
+if [ -n "$RUBY_VARIANT" ] || [ -n "$RUBY_VARIANT_FLAGS" ]; then
+  engine=$(ruby -e 'print RUBY_ENGINE')
+  if [ -n "$RUBY_VARIANT_FLAGS" ]; then
+    case "$engine" in
+      jruby) export JRUBY_OPTS="$JRUBY_OPTS $RUBY_VARIANT_FLAGS" ;;
+      truffleruby) export TRUFFLERUBYOPT="$RUBY_VARIANT_FLAGS $TRUFFLERUBYOPT" ;;
+      *) export RUBYOPT="$RUBY_VARIANT_FLAGS $RUBYOPT" ;;
+    esac
+  fi
+
+  # Fail here rather than silently benchmark without the variant. TruffleRuby
+  # prints nothing for its flags, so only an invalid one fails there.
+  description=$(ruby -e 'print RUBY_DESCRIPTION')
+  case "$engine:$RUBY_VARIANT" in
+    ruby:yjit) expected="+YJIT" ;;
+    ruby:zjit) expected="+ZJIT" ;;
+    jruby:dev|jruby:nojit) expected="-jit" ;;
+    *) expected="" ;;
+  esac
+  case "$description" in
+    *"$expected"*) echo "Variant $RUBY_VARIANT: $description" ;;
+    *) echo "Variant $RUBY_VARIANT is not on: $description" >&2; exit 1 ;;
+  esac
+fi
+
+# Also write every result as JSON, see docker/collect_results.rb.
+if [ -n "$RESULTS_DIR" ]; then
+  export RUBYOPT="-r/app/docker/collect_results.rb $RUBYOPT"
+fi
+
 if [ "$#" -eq 0 ]; then
   exec bundle exec rake
 fi