Fix crashing benchmarks and fail CI on crashes

.github/workflows/benchmarks.yml

@@ -1,10 +1,19 @@
 name: Benchmarks
 
+# Only changes that can affect a benchmark run it: README-only changes do not.
 on:
   push:
     branches: [ main ]
+    paths: &benchmark_paths
+      - 'code/**/*.rb'
+      - 'Gemfile'
+      - 'Rakefile'
+      - 'compose.yaml'
+      - 'docker/**'
+      - '.github/workflows/benchmarks.yml'
   pull_request:
     branches: [ main ]
+    paths: *benchmark_paths
 
 jobs:
   rake:
@@ -27,7 +36,38 @@ jobs:
       - name: Set Share Env
         if: github.ref_name == 'main'
         run: |
-          echo "SHARE=1" >> $GITHUB_ENV
+          echo "SHARE=1" >> "$GITHUB_ENV"
       - uses: actions/checkout@v4
+        with:
+          # A pull_request checks out a merge commit: HEAD^1 is the base.
+          fetch-depth: 2
+      # On a pull request, run only the benchmark files it changes, unless it
+      # changes something every benchmark depends on. Pushes to main run all.
+      - name: Pick benchmarks
+        id: pick
+        if: github.event_name == 'pull_request'
+        run: |
+          changed=$(git diff --name-only HEAD^1 HEAD)
+          if echo "$changed" | grep -qE '^(Gemfile|Rakefile|compose\.yaml|docker/|\.github/workflows/benchmarks\.yml)'; then
+            echo "Shared files changed, running every benchmark"
+          else
+            files=$(git diff --name-only --diff-filter=d HEAD^1 HEAD -- 'code/**/*.rb' | tr '\n' ' ')
+            if [ -z "$files" ]; then
+              # For example a PR that only deletes a benchmark.
+              echo "No benchmark to run"
+              echo "skip=true" >> "$GITHUB_OUTPUT"
+            else
+              echo "Running: $files"
+              echo "files=$files" >> "$GITHUB_OUTPUT"
+            fi
+          fi
       - name: Run benchmarks on ${{ matrix.ruby }}
-        run: docker compose run --rm -T ${{ matrix.ruby }}
+        if: steps.pick.outputs.skip != 'true'
+        env:
+          FILES: ${{ steps.pick.outputs.files }}
+        run: |
+          # Unquoted on purpose: one argument per file. set -f keeps names like
+          # dig-vs-[]-vs-fetch.rb from being read as glob patterns.
+          set -f
+          # shellcheck disable=SC2086
+          docker compose run --rm -T ${{ matrix.ruby }} $FILES

Gemfile

@@ -5,4 +5,13 @@ gem 'benchmark-ips', '>= 2.0'
 gem 'activesupport', '>= 2.2.1'
 gem 'e2mmap'
 
+# Needed on Ruby 4.0+, where ostruct is no longer a default gem (3.4 warns).
+# Keep the `if`: older Rubies benchmark the ostruct they ship, and the gem
+# does not even parse on Ruby 2.1.
+gem 'ostruct' if RUBY_VERSION >= '3.4'
+
+# JRuby 9.1's bundled jruby-openssl cannot complete a TLS handshake with
+# ips.fastruby.io, so sharing results (SHARE=1) crashed every benchmark.
+gem 'jruby-openssl', '>= 0.10' if RUBY_ENGINE == 'jruby' && Gem::Version.new(JRUBY_VERSION) < Gem::Version.new('9.2')
+
 gem 'rake'

Rakefile

@@ -1,14 +1,18 @@
 desc "run benchmark in current ruby"
 task :run_benchmark do
+  failed = []
+
   Dir["code/general/*.rb"].each do |benchmark|
     puts "$ ruby -v #{benchmark}"
-    system("ruby", "-v", "-W0", benchmark)
+    failed << benchmark unless system("ruby", "-v", "-W0", benchmark)
   end
 
   Dir["code/*/*.rb"].reject { |path| path =~ /^code\/general/ }.each do |benchmark|
     puts "$ ruby -v #{benchmark}"
-    system("ruby", "-v", "-W0", benchmark)
+    failed << benchmark unless system("ruby", "-v", "-W0", benchmark)
   end
+
+  abort "Failed benchmarks:\n#{failed.join("\n")}" unless failed.empty?
 end
 
 task default: :run_benchmark

code/hash/dig-vs-[]-vs-fetch.rb

@@ -3,8 +3,10 @@
 h = { a: { b: { c: { d: { e: "foo" } } } } }
 
 Benchmark.ips do |x|
-  x.report "Hash#dig" do
-    h.dig(:a, :b, :c, :d, :e)
+  if RUBY_VERSION >= "2.3.0"
+    x.report "Hash#dig" do
+      h.dig(:a, :b, :c, :d, :e)
+    end
   end
 
   x.report "Hash#[]" do

code/hash/slice-native-vs-before-native.rb

@@ -41,7 +41,7 @@ def slow
 end
 
 Benchmark.ips do |x|
-  x.report('Hash#native-slice   ') { fastest }
+  x.report('Hash#native-slice   ') { fastest } if RUBY_VERSION >= '2.5.0'
   x.report('Array#each          ') { faster }
   x.report('Array#each_w/_object') { fast }
   x.report('Hash#select-include ') { slow }

code/string/remove-extra-spaces-or-other-chars.rb

@@ -1,10 +1,11 @@
 require 'benchmark/ips'
 
-PASSAGE = <<~LIPSUM
-  Lorem ipsum dolor sit amet,  consectetur adipiscing  elit, sed do eiusmod    tempor incididunt ut  labore et dolore magna aliqua. 
-  Ut enim ad  minim veniam, quis    nostrud exercitation ullamco laboris    nisi ut aliquip ex ea commodo consequat. 
-  Duis aute    irure dolor in reprehenderit    in voluptate velit    esse cillum dolore eu    fugiat nulla pariatur. 
-  Excepteur sint  occaecat cupidatat non    proident, sunt in culpa qui officia    deserunt mollit  anim id est laborum.
+# A plain heredoc, not <<~, which is Ruby 2.3+ (older Rubies cannot parse the file at all).
+PASSAGE = <<LIPSUM
+Lorem ipsum dolor sit amet,  consectetur adipiscing  elit, sed do eiusmod    tempor incididunt ut  labore et dolore magna aliqua. 
+Ut enim ad  minim veniam, quis    nostrud exercitation ullamco laboris    nisi ut aliquip ex ea commodo consequat. 
+Duis aute    irure dolor in reprehenderit    in voluptate velit    esse cillum dolore eu    fugiat nulla pariatur. 
+Excepteur sint  occaecat cupidatat non    proident, sunt in culpa qui officia    deserunt mollit  anim id est laborum.
 LIPSUM
 
 raise unless PASSAGE.gsub(/ +/, " ") == PASSAGE.squeeze(" ")

compose.yaml

@@ -10,6 +10,10 @@ x-benchmark: &benchmark
   working_dir: /app
   environment:
     - SHARE
+    # Only JRuby reads it. Java's default heap cap (1/4 of memory) is too small
+    # for code/array/bsearch-vs-find.rb's 100M element array: JRuby 9.1 fails
+    # even on 16 GB CI runners, JRuby 10 on smaller machines.
+    - JRUBY_OPTS=-J-Xmx6g
   entrypoint: ["/app/docker/run-benchmarks.sh"]
 
 services:

docker/run-benchmarks.sh

@@ -14,7 +14,14 @@ if [ "$#" -eq 0 ]; then
   exec bundle exec rake
 fi
 
+failed=""
 for benchmark in "$@"; do
   echo "\$ ruby -v $benchmark"
-  bundle exec ruby -v -W0 "$benchmark"
+  bundle exec ruby -v -W0 "$benchmark" || failed="$failed
+$benchmark"
 done
+
+if [ -n "$failed" ]; then
+  echo "Failed benchmarks:$failed" >&2
+  exit 1
+fi