From 21a33f3bc76b71b996d90d2b04a9725f002a2a83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20V=C3=A1squez?= Date: Wed, 23 Sep 2026 00:37:09 -0600 Subject: [PATCH 1/4] Step 1: Add ostruct to the Gemfile for Ruby 3.4+ On Ruby 4.0, ruby-head, jruby-head and truffleruby-head, both OpenStruct benchmarks crashed with "cannot load such file -- ostruct (LoadError)": ostruct is no longer a default gem, and bundle exec only loads gems that are in the Gemfile. The gem is conditional because an unconditional one breaks Ruby 2.1 (ostruct 0.1.0 uses &.) and would replace the ostruct older Rubies ship. --- Gemfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Gemfile b/Gemfile index 00d364d9..9f5fa5e0 100644 --- a/Gemfile +++ b/Gemfile @@ -5,4 +5,9 @@ gem 'benchmark-ips', '>= 2.0' gem 'activesupport', '>= 2.2.1' gem 'e2mmap' +# Needed on Ruby 4.0+, where ostruct is no longer a default gem (3.4 warns). +# Keep the `if`: older Rubies benchmark the ostruct they ship, and the gem +# does not even parse on Ruby 2.1. +gem 'ostruct' if RUBY_VERSION >= '3.4' + gem 'rake' From 2e65fd127a44789197a4e750c32777596b73a61c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20V=C3=A1squez?= Date: Wed, 23 Sep 2026 01:07:28 -0600 Subject: [PATCH 2/4] Step 2: Stop benchmarks crashing on Rubies that lack a feature Crashes seen in CI, now fixed: - dig-vs-[]-vs-fetch.rb on 2.1 and 2.2: skip only the Hash#dig report (Ruby 2.3+), the other five still run. - slice-native-vs-before-native.rb on 2.1 to 2.4 and JRuby 9.1: skip only the native Hash#slice report (Ruby 2.5+), so the pre-native versions the file is about still run there. - remove-extra-spaces-or-other-chars.rb on 2.1 and 2.2: <<~ (Ruby 2.3+) made the file fail to parse. A plain heredoc builds the same string (checked byte for byte). - bsearch-vs-find.rb on JRuby: the default Java heap cap (1/4 of memory) is too small for its 100M element array. It failed on JRuby 9.1 in CI (3554MB cap) and on JRuby 10 locally (2232MB). Both now get -J-Xmx6g. - On main (SHARE=1), JRuby 9.1 crashed every benchmark when sharing: its bundled jruby-openssl fails the TLS handshake with ips.fastruby.io ("Received fatal alert: handshake_failure"). The Gemfile gives JRuby before 9.2 a newer jruby-openssl (0.14.6 resolves). JRuby 10 already connects, so it keeps its own. --- Gemfile | 4 ++++ code/hash/dig-vs-[]-vs-fetch.rb | 6 ++++-- code/hash/slice-native-vs-before-native.rb | 2 +- code/string/remove-extra-spaces-or-other-chars.rb | 11 ++++++----- compose.yaml | 4 ++++ 5 files changed, 19 insertions(+), 8 deletions(-) diff --git a/Gemfile b/Gemfile index 9f5fa5e0..853ad202 100644 --- a/Gemfile +++ b/Gemfile @@ -10,4 +10,8 @@ gem 'e2mmap' # does not even parse on Ruby 2.1. gem 'ostruct' if RUBY_VERSION >= '3.4' +# JRuby 9.1's bundled jruby-openssl cannot complete a TLS handshake with +# ips.fastruby.io, so sharing results (SHARE=1) crashed every benchmark. +gem 'jruby-openssl', '>= 0.10' if RUBY_ENGINE == 'jruby' && Gem::Version.new(JRUBY_VERSION) < Gem::Version.new('9.2') + gem 'rake' diff --git a/code/hash/dig-vs-[]-vs-fetch.rb b/code/hash/dig-vs-[]-vs-fetch.rb index d2964416..35bfe5fb 100644 --- a/code/hash/dig-vs-[]-vs-fetch.rb +++ b/code/hash/dig-vs-[]-vs-fetch.rb @@ -3,8 +3,10 @@ h = { a: { b: { c: { d: { e: "foo" } } } } } Benchmark.ips do |x| - x.report "Hash#dig" do - h.dig(:a, :b, :c, :d, :e) + if RUBY_VERSION >= "2.3.0" + x.report "Hash#dig" do + h.dig(:a, :b, :c, :d, :e) + end end x.report "Hash#[]" do diff --git a/code/hash/slice-native-vs-before-native.rb b/code/hash/slice-native-vs-before-native.rb index 85cc70f3..cee3505e 100644 --- a/code/hash/slice-native-vs-before-native.rb +++ b/code/hash/slice-native-vs-before-native.rb @@ -41,7 +41,7 @@ def slow end Benchmark.ips do |x| - x.report('Hash#native-slice ') { fastest } + x.report('Hash#native-slice ') { fastest } if RUBY_VERSION >= '2.5.0' x.report('Array#each ') { faster } x.report('Array#each_w/_object') { fast } x.report('Hash#select-include ') { slow } diff --git a/code/string/remove-extra-spaces-or-other-chars.rb b/code/string/remove-extra-spaces-or-other-chars.rb index fb1349f7..32c9c5c4 100644 --- a/code/string/remove-extra-spaces-or-other-chars.rb +++ b/code/string/remove-extra-spaces-or-other-chars.rb @@ -1,10 +1,11 @@ require 'benchmark/ips' -PASSAGE = <<~LIPSUM - Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. - Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. - Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. - Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum. +# A plain heredoc, not <<~, which is Ruby 2.3+ (older Rubies cannot parse the file at all). +PASSAGE = < Date: Wed, 23 Sep 2026 01:39:05 -0600 Subject: [PATCH 3/4] Step 3: Fail the run when a benchmark crashes The Rakefile ignored each system(...) result, so CI stayed green with 7 benchmarks crashing across Rubies. It now runs every file, then exits non-zero listing the ones that failed, so one crash does not hide others. docker/run-benchmarks.sh with file arguments stopped at the first failing file (set -e). It now runs them all and reports the same way. --- Rakefile | 8 ++++++-- docker/run-benchmarks.sh | 9 ++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/Rakefile b/Rakefile index d6abdec5..a040a1e3 100644 --- a/Rakefile +++ b/Rakefile @@ -1,14 +1,18 @@ desc "run benchmark in current ruby" task :run_benchmark do + failed = [] + Dir["code/general/*.rb"].each do |benchmark| puts "$ ruby -v #{benchmark}" - system("ruby", "-v", "-W0", benchmark) + failed << benchmark unless system("ruby", "-v", "-W0", benchmark) end Dir["code/*/*.rb"].reject { |path| path =~ /^code\/general/ }.each do |benchmark| puts "$ ruby -v #{benchmark}" - system("ruby", "-v", "-W0", benchmark) + failed << benchmark unless system("ruby", "-v", "-W0", benchmark) end + + abort "Failed benchmarks:\n#{failed.join("\n")}" unless failed.empty? end task default: :run_benchmark diff --git a/docker/run-benchmarks.sh b/docker/run-benchmarks.sh index 7969f20a..61cb1e8c 100755 --- a/docker/run-benchmarks.sh +++ b/docker/run-benchmarks.sh @@ -14,7 +14,14 @@ if [ "$#" -eq 0 ]; then exec bundle exec rake fi +failed="" for benchmark in "$@"; do echo "\$ ruby -v $benchmark" - bundle exec ruby -v -W0 "$benchmark" + bundle exec ruby -v -W0 "$benchmark" || failed="$failed +$benchmark" done + +if [ -n "$failed" ]; then + echo "Failed benchmarks:$failed" >&2 + exit 1 +fi From 8468a2f266a42c5259d5db345faec9b10a9c8e35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20V=C3=A1squez?= Date: Wed, 23 Sep 2026 01:39:05 -0600 Subject: [PATCH 4/4] Step 4: Run only the benchmarks a pull request changes Every PR ran all 68 benchmarks on 18 Rubies (about 22 min per job, 18 of the org's 20 concurrent jobs), even for README-only changes. - paths filter: only changes to benchmarks, the Gemfile, Rakefile, compose.yaml, docker/ or this workflow run it, on PRs and on main. - On a PR, only the changed benchmark files run, unless a shared file changed, then everything runs. A PR that only deletes benchmarks runs nothing. Pushes to main always run everything. - File names reach the container through an env var with globbing off, so names like dig-vs-[]-vs-fetch.rb pass through as is and a PR cannot inject shell through a file name. --- .github/workflows/benchmarks.yml | 44 ++++++++++++++++++++++++++++++-- 1 file changed, 42 insertions(+), 2 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 860fcad6..7c904088 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -1,10 +1,19 @@ name: Benchmarks +# Only changes that can affect a benchmark run it: README-only changes do not. on: push: branches: [ main ] + paths: &benchmark_paths + - 'code/**/*.rb' + - 'Gemfile' + - 'Rakefile' + - 'compose.yaml' + - 'docker/**' + - '.github/workflows/benchmarks.yml' pull_request: branches: [ main ] + paths: *benchmark_paths jobs: rake: @@ -27,7 +36,38 @@ jobs: - name: Set Share Env if: github.ref_name == 'main' run: | - echo "SHARE=1" >> $GITHUB_ENV + echo "SHARE=1" >> "$GITHUB_ENV" - uses: actions/checkout@v4 + with: + # A pull_request checks out a merge commit: HEAD^1 is the base. + fetch-depth: 2 + # On a pull request, run only the benchmark files it changes, unless it + # changes something every benchmark depends on. Pushes to main run all. + - name: Pick benchmarks + id: pick + if: github.event_name == 'pull_request' + run: | + changed=$(git diff --name-only HEAD^1 HEAD) + if echo "$changed" | grep -qE '^(Gemfile|Rakefile|compose\.yaml|docker/|\.github/workflows/benchmarks\.yml)'; then + echo "Shared files changed, running every benchmark" + else + files=$(git diff --name-only --diff-filter=d HEAD^1 HEAD -- 'code/**/*.rb' | tr '\n' ' ') + if [ -z "$files" ]; then + # For example a PR that only deletes a benchmark. + echo "No benchmark to run" + echo "skip=true" >> "$GITHUB_OUTPUT" + else + echo "Running: $files" + echo "files=$files" >> "$GITHUB_OUTPUT" + fi + fi - name: Run benchmarks on ${{ matrix.ruby }} - run: docker compose run --rm -T ${{ matrix.ruby }} + if: steps.pick.outputs.skip != 'true' + env: + FILES: ${{ steps.pick.outputs.files }} + run: | + # Unquoted on purpose: one argument per file. set -f keeps names like + # dig-vs-[]-vs-fetch.rb from being read as glob patterns. + set -f + # shellcheck disable=SC2086 + docker compose run --rm -T ${{ matrix.ruby }} $FILES