Fix CI by running benchmarks in Docker

.github/workflows/benchmarks.yml

@@ -9,73 +9,25 @@ on:
 jobs:
   rake:
     runs-on: ubuntu-latest
+    timeout-minutes: 60
 
     strategy:
+      fail-fast: false
       matrix:
-        ruby-version: [
-          'ruby-head', '3.2', '3.1', '3.0', '2.7', '2.6', '2.5', '2.4',
-          'jruby-head',
-          'truffleruby-head'
+        # Service names from compose.yaml
+        ruby: [
+          'ruby_head', 'ruby_4.0', 'ruby_3.4', 'ruby_3.3', 'ruby_3.2', 'ruby_3.1',
+          'ruby_3.0', 'ruby_2.7', 'ruby_2.6', 'ruby_2.5', 'ruby_2.4', 'ruby_2.3',
+          'ruby_2.2', 'ruby_2.1',
+          'jruby_head', 'jruby_9.1',
+          'truffleruby_head', 'truffleruby_22'
         ]
 
     steps:
       - name: Set Share Env
         if: github.ref_name == 'main'
         run: |
           echo "SHARE=1" >> $GITHUB_ENV
-      - uses: actions/checkout@v3
-      - name: Set up Ruby ${{ matrix.ruby-version }}
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: ${{ matrix.ruby-version }}
-      - name: Install dependencies
-        run: bundle install
-      - name: Run benchmarks
-        run: bundle exec rake
-
-  rake_old:
-    runs-on: ubuntu-22.04
-
-    strategy:
-      matrix:
-        ruby-version: [
-          'jruby-9.1',
-          'truffleruby-22'
-        ]
-
-    steps:
-      - name: Set Share Env
-        if: github.ref_name == 'main'
-        run: |
-          echo "SHARE=1" >> $GITHUB_ENV
-      - uses: actions/checkout@v3
-      - name: Set up Ruby ${{ matrix.ruby-version }}
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: ${{ matrix.ruby-version }}
-      - name: Install dependencies
-        run: bundle install
-      - name: Run benchmarks
-        run: bundle exec rake
-
-  rake_older:
-    runs-on: ubuntu-20.04
-
-    strategy:
-      matrix:
-        ruby-version: ['2.3', '2.2', '2.1']
-
-    steps:
-      - name: Set Share Env
-        if: github.ref_name == 'main'
-        run: |
-          echo "SHARE=1" >> $GITHUB_ENV
-      - uses: actions/checkout@v3
-      - name: Set up Ruby ${{ matrix.ruby-version }}
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: ${{ matrix.ruby-version }}
-      - name: Install dependencies
-        run: bundle install
-      - name: Run benchmarks
-        run: bundle exec rake
+      - uses: actions/checkout@v4
+      - name: Run benchmarks on ${{ matrix.ruby }}
+        run: docker compose run --rm -T ${{ matrix.ruby }}

CONTRIBUTING.md

@@ -34,6 +34,23 @@ Run your result:
 $ ruby -v code/your-new/entry.rb
 ```
 
+To run it on a Ruby you don't have installed, use Docker. There is one service
+per Ruby in the CI matrix (see `compose.yaml`):
+
+```
+docker compose run --rm ruby_2.1 code/your-new/entry.rb
+docker compose run --rm truffleruby_head code/your-new/entry.rb
+```
+
+Without a file argument, the service runs every benchmark, the same way CI does.
+
+The `*_head` and `truffleruby_22` images are built once and then reused, so
+the head builds go stale. To get the latest nightly build:
+
+```
+docker compose build --no-cache ruby_head
+```
+
 Thanks in advance!!! Look forward to learning more from you!
 
 <3 [JuanitoFatas](https://twitter.com/juanitofatas)

compose.yaml

@@ -0,0 +1,102 @@
+# Run the benchmarks on any Ruby from the CI matrix without installing it:
+#
+#   docker compose run --rm ruby_2.1
+#
+# CI runs the same services, see .github/workflows/benchmarks.yml.
+
+x-benchmark: &benchmark
+  volumes:
+    - .:/app
+  working_dir: /app
+  environment:
+    - SHARE
+  entrypoint: ["/app/docker/run-benchmarks.sh"]
+
+services:
+  ruby_4.0:
+    <<: *benchmark
+    image: ruby:4.0
+
+  ruby_3.4:
+    <<: *benchmark
+    image: ruby:3.4
+
+  ruby_3.3:
+    <<: *benchmark
+    image: ruby:3.3
+
+  ruby_3.2:
+    <<: *benchmark
+    image: ruby:3.2
+
+  ruby_3.1:
+    <<: *benchmark
+    image: ruby:3.1
+
+  ruby_3.0:
+    <<: *benchmark
+    image: ruby:3.0
+
+  ruby_2.7:
+    <<: *benchmark
+    image: ruby:2.7
+
+  ruby_2.6:
+    <<: *benchmark
+    image: ruby:2.6
+
+  ruby_2.5:
+    <<: *benchmark
+    image: ruby:2.5
+
+  ruby_2.4:
+    <<: *benchmark
+    image: ruby:2.4
+
+  ruby_2.3:
+    <<: *benchmark
+    image: ruby:2.3
+
+  ruby_2.2:
+    <<: *benchmark
+    image: ruby:2.2
+
+  # Only amd64 images exist for these two; they run under emulation on arm64 hosts.
+  ruby_2.1:
+    <<: *benchmark
+    image: ruby:2.1
+    platform: linux/amd64
+
+  jruby_9.1:
+    <<: *benchmark
+    image: jruby:9.1
+    platform: linux/amd64
+
+  # No official images for these, they are built from docker/Dockerfile.
+  truffleruby_22:
+    <<: *benchmark
+    build:
+      context: docker
+      args:
+        RUBY: truffleruby-22.3.1
+
+  ruby_head:
+    <<: *benchmark
+    build:
+      context: docker
+      args:
+        RUBY: ruby-head
+
+  jruby_head:
+    <<: *benchmark
+    build:
+      context: docker
+      args:
+        RUBY: jruby-head
+
+  truffleruby_head:
+    <<: *benchmark
+    build:
+      context: docker
+      args:
+        RUBY: truffleruby-head

docker/Dockerfile

@@ -0,0 +1,41 @@
+# Image for Rubies without an official Docker image: the nightly head builds
+# (the same ones ruby/setup-ruby uses) and TruffleRuby 22.
+FROM ubuntu:24.04
+
+ARG RUBY=ruby-head
+ARG TARGETARCH
+
+ENV DEBIAN_FRONTEND=noninteractive LANG=C.UTF-8
+
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends \
+      ca-certificates curl build-essential git \
+      libyaml-dev libssl-dev libffi-dev libgmp-dev libreadline-dev zlib1g-dev \
+ && case "$RUBY" in jruby-*) apt-get install -y --no-install-recommends openjdk-21-jre-headless ;; esac \
+ && rm -rf /var/lib/apt/lists/*
+
+# The head builds are compiled with this prefix, so they must live here.
+WORKDIR /home/runner/.rubies
+
+RUN case "$TARGETARCH" in \
+      amd64) builder_arch=x64; oracle_arch=amd64 ;; \
+      arm64) builder_arch=arm64; oracle_arch=aarch64 ;; \
+      *) echo "unsupported arch: $TARGETARCH" >&2; exit 1 ;; \
+    esac \
+ && case "$RUBY" in \
+      ruby-head|jruby-head|truffleruby-head) \
+        url="https://github.com/ruby/${RUBY%%-*}-dev-builder/releases/latest/download/$RUBY-ubuntu-24.04-$builder_arch.tar.gz" ;; \
+      truffleruby-22.*) \
+        version="${RUBY#truffleruby-}"; \
+        url="https://github.com/oracle/truffleruby/releases/download/vm-$version/truffleruby-$version-linux-$oracle_arch.tar.gz" ;; \
+      *) echo "unsupported RUBY: $RUBY" >&2; exit 1 ;; \
+    esac \
+ && curl -fsSL "$url" | tar xz \
+ && if [ ! -d "$RUBY" ]; then mv truffleruby-* "$RUBY"; fi \
+ && if [ -x "$RUBY/lib/truffle/post_install_hook.sh" ]; then "$RUBY/lib/truffle/post_install_hook.sh"; fi
+
+ENV PATH=/home/runner/.rubies/${RUBY}/bin:$PATH
+
+RUN ruby -v
+
+WORKDIR /app

docker/run-benchmarks.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+# Runs the benchmarks inside a container: all of them by default, or only the
+# files passed as arguments. The Gemfile is copied out of the mounted repo so
+# each Ruby resolves its own Gemfile.lock and never touches the one on the host.
+set -e
+
+mkdir -p /tmp/bundle
+cp /app/Gemfile /tmp/bundle/Gemfile
+export BUNDLE_GEMFILE=/tmp/bundle/Gemfile
+
+bundle install
+
+if [ "$#" -eq 0 ]; then
+  exec bundle exec rake
+fi
+
+for benchmark in "$@"; do
+  echo "\$ ruby -v $benchmark"
+  bundle exec ruby -v -W0 "$benchmark"
+done