Vendor clickhouse role

.github/workflows/test_clickhouse_role.yml

@@ -0,0 +1,44 @@
+# Tests the clickhouse role's SQL users, quotas and grants and its apt pin
+# (ansible/roles/clickhouse/tests), and adding and removing a replica with oonidata_clickhouse
+# (ansible/roles/oonidata_clickhouse/tests), when the roles, their vars or the schema change
+name: Test ClickHouse roles
+on:
+  pull_request:
+    paths:
+      - "ansible/roles/clickhouse/**"
+      - "ansible/roles/oonidata_clickhouse/**"
+      - "ansible/group_vars/**"
+      - "ansible/host_vars/data*"
+      - "scripts/cluster-migration/schema.sql"
+      - ".github/workflows/test_clickhouse_role.yml"
+  push:
+    branches: [main]
+    paths:
+      - "ansible/roles/clickhouse/**"
+      - "ansible/roles/oonidata_clickhouse/**"
+      - "ansible/group_vars/**"
+      - "ansible/host_vars/data*"
+      - "scripts/cluster-migration/schema.sql"
+      - ".github/workflows/test_clickhouse_role.yml"
+
+jobs:
+  sql-objects:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v4
+      - run: pipx install ansible-core==2.17.7 && sudo apt-get update -qq && sudo apt-get install -y --no-install-recommends python3-yaml
+      - run: ansible/roles/clickhouse/tests/run.sh
+      - if: failure()
+        uses: actions/upload-artifact@v4
+        with: {name: sql-objects-out, path: ansible/roles/clickhouse/tests/out/}
+  replicas:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v4
+      - run: pipx install ansible-core==2.17.7 && sudo apt-get update -qq && sudo apt-get install -y --no-install-recommends python3-yaml
+      - run: ansible/roles/oonidata_clickhouse/tests/run.sh
+      - if: failure()
+        uses: actions/upload-artifact@v4
+        with: {name: replicas-out, path: ansible/roles/oonidata_clickhouse/tests/out/}

.github/workflows/test_clickhouse_upgrade.yml

@@ -0,0 +1,70 @@
+# Rolls a ClickHouse version bump out with the clickhouse role, one host at a time
+# (ansible/roles/clickhouse/tests/upgrade). Slow (it installs ClickHouse twice on three hosts),
+# so it only runs when it has something to test:
+# - a PR that changes clickhouse_version: from the base's version to the PR's;
+# - a PR that changes the role (other than its README and other tests), the schema step or
+#   this workflow: from production's version to tests/upgrade/next_version;
+# - by hand, with any two versions.
+# A change to the vars alone, e.g. a user's settings, doesn't run it.
+name: Test ClickHouse upgrade
+on:
+  pull_request:
+    paths:
+      - "ansible/group_vars/clickhouse/vars.yml"
+      - "ansible/roles/clickhouse/**"
+      - "!ansible/roles/clickhouse/README.md"
+      - "!ansible/roles/clickhouse/tests/**"
+      - "ansible/roles/clickhouse/tests/upgrade/**"
+      - "ansible/roles/oonidata_clickhouse/tasks/schema.yml"
+      - ".github/workflows/test_clickhouse_upgrade.yml"
+  workflow_dispatch:
+    inputs:
+      from_version: {description: "Version to install first (default: production's)", required: false}
+      to_version: {description: "Version to upgrade to (default: tests/upgrade/next_version)", required: false}
+
+jobs:
+  upgrade:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@v4
+        with: {fetch-depth: 0}
+      - name: Choose the versions
+        id: versions
+        env:
+          EVENT: ${{ github.event_name }}
+          BASE: ${{ github.event.pull_request.base.sha }}
+          FROM_INPUT: ${{ inputs.from_version }}
+          TO_INPUT: ${{ inputs.to_version }}
+        run: |
+          # clickhouse_version's value, without quotes or a trailing comment
+          version() { sed -n 's/^clickhouse_version: *["'"'"']\{0,1\}\([0-9.]*\).*/\1/p'; }
+          run=true from=$FROM_INPUT to=$TO_INPUT
+          if [ "$EVENT" = pull_request ]; then
+            base=$(git show "$BASE:ansible/group_vars/clickhouse/vars.yml" | version)
+            head=$(version < ansible/group_vars/clickhouse/vars.yml)
+            if [ "$base" != "$head" ]; then
+              from=$base to=$head
+            else
+              # the files of `paths:` above, other than the vars
+              changed=$(git diff --name-only "$BASE" HEAD \
+                | grep -E '^(ansible/roles/clickhouse/|ansible/roles/oonidata_clickhouse/tasks/schema\.yml$|\.github/workflows/test_clickhouse_upgrade\.yml$)' \
+                | grep -v -E '^ansible/roles/clickhouse/(README\.md$|tests/)' ; \
+                git diff --name-only "$BASE" HEAD | grep '^ansible/roles/clickhouse/tests/upgrade/' || true)
+              [ -n "$changed" ] || run=false  # only the vars changed, not the version
+            fi
+          fi
+          echo "run=$run" >> "$GITHUB_OUTPUT"
+          echo "from=$from" >> "$GITHUB_OUTPUT"
+          echo "to=$to" >> "$GITHUB_OUTPUT"
+          echo "run=$run from=${from:-production} to=${to:-next_version}"
+      - if: steps.versions.outputs.run == 'true'
+        run: pipx install ansible-core==2.17.7 && sudo apt-get update -qq && sudo apt-get install -y --no-install-recommends python3-yaml
+      - if: steps.versions.outputs.run == 'true'
+        env:
+          FROM_VERSION: ${{ steps.versions.outputs.from }}
+          TO_VERSION: ${{ steps.versions.outputs.to }}
+        run: ansible/roles/clickhouse/tests/upgrade/run.sh
+      - if: failure()
+        uses: actions/upload-artifact@v4
+        with: {name: upgrade-out, path: ansible/roles/clickhouse/tests/upgrade/out/}

ansible/deploy-clickhouse.yml

@@ -1,11 +1,14 @@
 ---
 - name: Deploy oonidata clickhouse hosts
+  # Not the notebook: its ClickHouse isn't part of the cluster.
   hosts:
-    - notebook.ooni.org
     - data1.htz-fsn.prod.ooni.nu
     - data2.htz-fsn.prod.ooni.nu
     - data3.htz-fsn.prod.ooni.nu
   become: true
+  # One host at a time: a config change restarts ClickHouse, and data1-3 are
+  # also the Keeper ensemble, so restarting them together loses quorum.
+  serial: 1
   roles:
     - role: prometheus_node_exporter
       tags: [node_exporter]

ansible/group_vars/clickhouse/vars.yml

@@ -164,7 +164,6 @@ clickhouse_distributed_ddl:
   cleanup_delay_period: 60
   max_tasks_in_queue: 1000
 
-clickhouse_role_manage_settings_profiles: True
 clickhouse_default_profiles:
   default:
     readonly: 2
@@ -227,171 +226,107 @@ clickhouse_default_users:
     profile: write
     quota: default
 
+# SQL-driven users, quotas and grants (see the clickhouse role's README.md).
+# Each user carries its own settings and sets readonly = 0, since a user without
+# it gets the default profile's readonly = 2; what it may do comes from
+# clickhouse_custom_grants.
 clickhouse_role_manage_users: True
+clickhouse_role_manage_quotas: True
+clickhouse_role_manage_grants: True
+# Off until each node has been checked for SQL users and quotas not listed here.
+clickhouse_sql_objects_drop_unlisted: false
+
 clickhouse_custom_users:
-  - user:
-    name: fastpath
+  - name: fastpath
     password_type: sha256_hash
     password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_fastpath_password', profile='oonidevops_user_prod') | hash('sha256') }}"
-    networks:
-      - "IP '0.0.0.0/0'"
-    profile:
-      - write
-    quota: "fastpath"
-    databases: [ooni, oonitest]
+    networks: ["IP '0.0.0.0/0'"]
+    settings:
+      - "readonly = 0"
+      - "max_memory_usage = 61001001000"  # 61 GB
 
-  - user:
-    name: oonimeasurements
+  # Only SELECT through its grants. CONST stops queries from raising its limits;
+  # other settings, such as log_comment, can still be set per query.
+  - name: oonimeasurements
     password_type: sha256_hash
     password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonimeasurements_password', profile='oonidevops_user_prod') | hash('sha256') }}"
-    networks:
-      - "IP '0.0.0.0/0'"
+    networks: ["IP '0.0.0.0/0'"]
     settings:
-      # 1 GB
-      - "max_memory_usage = 1001001000"
-      # 60 seconds
-      - "max_execution_time = 30"
-      # 500 GB
-      - "max_bytes_to_read = 501001001000"
-      # 5 B
-      - "max_rows_to_read = 5001001000"
-      # 5s
-      - "timeout_before_checking_execution_speed = 5"
-      # 50k
-      - "max_result_rows = 51000"
-    profile:
-      - readonly
-    quota: oonimeasurements
-    databases: [ooni, oonitest]
-
-  - user:
-    name: ooniprobe
+      - "readonly = 0"
+      - "max_memory_usage = 1001001000 CONST"                  # 1 GB
+      - "max_execution_time = 30 CONST"                        # seconds
+      - "max_bytes_to_read = 501001001000 CONST"               # 500 GB
+      - "max_rows_to_read = 5001001000 CONST"                  # 5 B
+      - "timeout_before_checking_execution_speed = 5 CONST"    # seconds
+      - "max_result_rows = 51000 CONST"
+
+  - name: ooniprobe
     password_type: sha256_hash
     password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_ooniprobe_password', profile='oonidevops_user_prod') | hash('sha256') }}"
-    networks:
-      - "IP '0.0.0.0/0'"
-    profile:
-      - write
-    quota: ooniprobe
-    databases: [ooni, oonitest]
+    networks: ["IP '0.0.0.0/0'"]
+    settings:
+      - "readonly = 0"
+      - "max_memory_usage = 61001001000"  # 61 GB
 
-  - user:
-    name: oonirun
+  - name: oonirun
     password_type: sha256_hash
     password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonirun_password', profile='oonidevops_user_prod') | hash('sha256') }}"
-    networks:
-      - "IP '0.0.0.0/0'"
-    profile:
-      - write
-    quota: oonirun
-    databases: [ooni, oonitest]
+    networks: ["IP '0.0.0.0/0'"]
+    settings:
+      - "readonly = 0"
+      - "max_memory_usage = 61001001000"  # 61 GB
 
-  - user:
-    name: oonitestlists
+  - name: oonitestlists
     password_type: sha256_hash
     password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonitestlists_password', profile='oonidevops_user_prod') | hash('sha256') }}"
-    networks:
-      - "IP '0.0.0.0/0'"
-    profile:
-      - write
-    quota: oonitestlists
-    databases: [ooni, oonitest]
+    networks: ["IP '0.0.0.0/0'"]
+    settings:
+      - "readonly = 0"
+      - "max_memory_usage = 61001001000"  # 61 GB
 
-  - user:
-    name: notebook
+  - name: notebook
     password_type: sha256_hash
     password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_notebook_password', profile='oonidevops_user_prod') | hash('sha256') }}"
-    networks:
-      - "IP '0.0.0.0/0'"
-    profile:
-      - write
-    quota: notebook
-    databases: [ooni]
+    networks: ["IP '0.0.0.0/0'"]
+    settings:
+      - "readonly = 0"
+      - "max_memory_usage = 61001001000"  # 61 GB
 
-clickhouse_role_manage_quotas: True
 clickhouse_custom_quotas:
-  - quota:
-    name: oonimeasurements
-    duration: 600
-    queries: 12000
-    errors: 1000
-    result_rows: 0
-    read_rows: 0
-    execution_time: 1000
-
-    # no limits set
-  - quota:
-    name: ooniprobe
-    duration: 0
-    queries: 0
-    errors: 0
-    result_rows: 0
-    read_rows: 0
-    execution_time: 0
-
-    # no limits set
-  - quota:
-    name: oonirun
-    duration: 0
-    queries: 0
-    errors: 0
-    result_rows: 0
-    read_rows: 0
-    execution_time: 0
-
-  - quota:
-    name: oonitestlists
-    duration: 600
-    queries: 12000
-    errors: 1000
-    result_rows: 0
-    read_rows: 0
-    execution_time: 1000
-
-    # no limits set
-  - quota:
-    name: notebook
-    duration: 0
-    queries: 0
-    errors: 0
-    result_rows: 0
-    read_rows: 0
-    execution_time: 0
+  # Not assigned to the user (no `to`), so each node keeps its current assignment:
+  # the API's node, data2, has never had a quota, and these limits were never
+  # checked against its traffic.
+  - name: oonimeasurements
+    intervals:
+      - "INTERVAL 600 second MAX queries = 12000, errors = 1000, execution_time = 1000"
+  # No limits: these exist on the nodes already, and are listed so that they aren't
+  # dropped if clickhouse_sql_objects_drop_unlisted is turned on
+  - name: ooniprobe
+  - name: oonirun
+  - name: notebook
+  # The test lists editor only inserts url_priorities rows, so a small quota
+  - name: oonitestlists
+    intervals:
+      - "INTERVAL 3600 second MAX queries = 1000, errors = 100, execution_time = 600"
+    to: [oonitestlists]
 
-clickhouse_role_manage_grants: True
 clickhouse_custom_grants:
-- on:
-  databases: [ooni, oonitest]
-  tables: ["*"]
-  privileges: [SELECT]
-  to: [ooniprobe, oonimeasurements, oonirun, fastpath, notebook]
-
-- on:
-  databases: [ooni, oonitest]
-  tables: [url_priorities]
-  privileges: [INSERT]
-  to: [oonitestlists]
-
-- on:
-  databases: [ooni, oonitest]
-  tables: [faulty_measurements]
-  privileges: [INSERT]
-  to: [ooniprobe]
-
-- on:
-  databases: [ooni, oonitest]
-  tables: [fastpath, obs_web, obs_openvpn, jsonl, new_jsonl]
-  privileges: [INSERT]
-  to: [fastpath]
-
-clickhouse_custom_grant_roles:
-  - roles: [oonimeasurements]
-    to: [oonimeasurements]
-
-clickhouse_role_manage_roles: True
-clickhouse_custom_roles:
-  - role:
-    name: oonimeasurements
+  - databases: [ooni, oonitest]
+    tables: ["*"]
+    privileges: [SELECT]
+    to: [ooniprobe, oonimeasurements, oonirun, fastpath, notebook]
+  - databases: [ooni, oonitest]
+    tables: [url_priorities]
+    privileges: [INSERT]
+    to: [oonitestlists]
+  - databases: [ooni, oonitest]
+    tables: [faulty_measurements]
+    privileges: [INSERT]
+    to: [ooniprobe]
+  - databases: [ooni, oonitest]
+    tables: [fastpath, obs_web, obs_openvpn, jsonl, new_jsonl]
+    privileges: [INSERT]
+    to: [fastpath]
 
 clickhouse_default_quotas:
   - quota:

ansible/host_vars/notebook1.htz-fsn.prod.ooni.nu

@@ -152,6 +152,7 @@ clickhouse_listen_hosts:
 
 # this only applies to SQL managed users
 clickhouse_role_manage_users: false
+clickhouse_role_manage_quotas: false
 clickhouse_role_manage_grants: false
 # these are handled via config files
 clickhouse_default_users: