feat(docker): run experimentation infra in docker compose

Makefile

@@ -17,3 +17,14 @@ lint:
 	$(MAKE) -C api lint
 	$(MAKE) -C docs lint
 	$(MAKE) -C frontend lint
+
+# Run `make docker-up flags-only=1` to run Flagsmith without experimentation.
+compose-file = $(if $(flags-only),docker-compose.flags-only.yml,docker-compose.yml)
+
+.PHONY: docker-up
+docker-up:
+	docker compose -f $(compose-file) up -d
+
+.PHONY: docker-down
+docker-down:
+	docker compose -f $(compose-file) down

docker-compose.flags-only.yml

@@ -0,0 +1,90 @@
+# See https://docs.flagsmith.com/deployment/docker for more information on running Flagsmith in Docker
+# This Docker Compose file will run the Flagsmith Platform without experimentation
+
+x-flagsmith-env: &flagsmith-env
+  # All environments variables are available here:
+  # API: https://docs.flagsmith.com/deployment/locally-api#environment-variables
+  # UI: https://docs.flagsmith.com/deployment/locally-frontend#environment-variables
+
+  DATABASE_URL: postgresql://postgres:password@postgres:5432/flagsmith
+  USE_POSTGRES_FOR_ANALYTICS: 'true' # Store API and Flag Analytics data in Postgres
+  ENVIRONMENT: production # set to 'production' in production.
+  DJANGO_ALLOWED_HOSTS: '*' # Change this in production
+  FLAGSMITH_DOMAIN: localhost:8000 # Change this in production
+  DJANGO_SECRET_KEY: secret # Change this in production
+  # PREVENT_SIGNUP: 'true' # Uncomment to prevent any additional signups
+  # ALLOW_REGISTRATION_WITHOUT_INVITE: 'true' # Uncomment and set to false to only allow signups via invitations
+
+  # Enable Task Processor
+  TASK_RUN_METHOD: TASK_PROCESSOR # other options are: SYNCHRONOUSLY, SEPARATE_THREAD (default)
+  PROMETHEUS_ENABLED: 'true'
+
+  # Uncomment if you want to enable Google OAuth. Note this does not turn Google OAuth on. You still need to use
+  # Flagsmith on Flagsmith to enable it - https://docs.flagsmith.com/deployment/#oauth_google
+  # DJANGO_SECURE_CROSS_ORIGIN_OPENER_POLICY: 'same-origin-allow-popups'
+
+  # For more info on configuring E-Mails - https://docs.flagsmith.com/deployment/locally-api#environment-variables
+  # Example SMTP:
+  # EMAIL_BACKEND: django.core.mail.backends.smtp.EmailBackend
+  # EMAIL_HOST: mail.example.com
+  # SENDER_EMAIL: flagsmith@example.com
+  # EMAIL_HOST_USER: flagsmith@example.com
+  # EMAIL_HOST_PASSWORD: smtp_account_password
+  # EMAIL_PORT: 587 # optional
+  # EMAIL_USE_TLS: 'true' # optional
+
+volumes:
+  pgdata:
+
+services:
+  postgres:
+    image: postgres:15.5-alpine
+    environment:
+      POSTGRES_PASSWORD: password
+      POSTGRES_DB: flagsmith
+    container_name: flagsmith_postgres
+    volumes:
+      - pgdata:/var/lib/postgresql/data
+    healthcheck:
+      test: ['CMD-SHELL', 'pg_isready -d flagsmith -U postgres']
+      interval: 2s
+      timeout: 2s
+      retries: 20
+      start_period: 20s
+
+  migrate-db:
+    image: docker.flagsmith.com/flagsmith/flagsmith:latest
+    command:
+      - migrate
+    environment:
+      <<: *flagsmith-env
+    healthcheck:
+      disable: true
+    depends_on:
+      postgres:
+        condition: service_healthy
+
+  flagsmith:
+    image: docker.flagsmith.com/flagsmith/flagsmith:latest
+    command:
+      - serve
+    environment:
+      <<: *flagsmith-env
+    ports:
+      - 8000:8000
+    depends_on:
+      migrate-db:
+        condition: service_completed_successfully
+
+  # The flagsmith_processor service is only needed if TASK_RUN_METHOD set to TASK_PROCESSOR
+  # in the application environment
+  flagsmith-task-processor:
+    image: docker.flagsmith.com/flagsmith/flagsmith:latest
+    environment:
+      <<: *flagsmith-env
+    ports:
+      - 8001:8000
+    depends_on:
+      migrate-db:
+        condition: service_completed_successfully
+    command: run-task-processor

docker-compose.yml

@@ -12,6 +12,7 @@ x-flagsmith-env: &flagsmith-env
   DJANGO_ALLOWED_HOSTS: '*' # Change this in production
   FLAGSMITH_DOMAIN: localhost:8000 # Change this in production
   DJANGO_SECRET_KEY: secret # Change this in production
+  WAREHOUSE_CREDENTIALS_SECRET: &warehouse-credentials-secret secret # Change this in production
   # PREVENT_SIGNUP: 'true' # Uncomment to prevent any additional signups
   # ALLOW_REGISTRATION_WITHOUT_INVITE: 'true' # Uncomment and set to false to only allow signups via invitations
 
@@ -33,8 +34,12 @@ x-flagsmith-env: &flagsmith-env
   # EMAIL_PORT: 587 # optional
   # EMAIL_USE_TLS: 'true' # optional
 
+  # To run without experimentation, use docker-compose.flags-only.yml instead.
+  EXPERIMENTATION_ENABLED: 'true'
+
 volumes:
   pgdata:
+  clickhouse-data:
 
 services:
   postgres:
@@ -88,3 +93,99 @@ services:
       migrate-db:
         condition: service_completed_successfully
     command: run-task-processor
+
+  # Experimentation services.
+  kafka:
+    image: apache/kafka:3.9.1
+    environment:
+      KAFKA_NODE_ID: 1
+      KAFKA_PROCESS_ROLES: broker,controller
+      KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
+      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
+      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+    healthcheck:
+      test: ['CMD-SHELL', '/opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092']
+      interval: 5s
+      timeout: 10s
+      retries: 20
+
+  kafka-init:
+    image: apache/kafka:3.9.1
+    entrypoint: ['/bin/sh', '-c']
+    command:
+      - |
+        for topic in flagsmith_warehouse_events external_warehouse_events external_warehouse_events_retry; do
+          /opt/kafka/bin/kafka-topics.sh --bootstrap-server kafka:9092 --create --if-not-exists --topic $$topic
+        done
+    depends_on:
+      kafka:
+        condition: service_healthy
+
+  # Connect it from Environment Settings > Warehouse > ClickHouse: host `clickhouse`, port 8123,
+  # TLS off, database `flagsmith_exp`, username `flagsmith`, password `password`.
+  clickhouse:
+    image: clickhouse/clickhouse-server:25.12
+    environment:
+      CLICKHOUSE_USER: flagsmith
+      CLICKHOUSE_PASSWORD: password
+    ulimits:
+      nofile:
+        soft: 262144
+        hard: 262144
+    volumes:
+      - clickhouse-data:/var/lib/clickhouse
+    healthcheck:
+      test: ['CMD', 'wget', '--spider', '-q', 'http://localhost:8123/ping']
+      interval: 5s
+      timeout: 3s
+      retries: 20
+
+  # Creates the events table from experimentation/clickhouse/events.sql.
+  clickhouse-init:
+    image: clickhouse/clickhouse-server:25.12
+    entrypoint: ['/bin/sh', '-c']
+    command:
+      - |
+        set -e
+        wget -qO /tmp/events.sql https://raw.githubusercontent.com/Flagsmith/flagsmith/main/experimentation/clickhouse/events.sql
+        clickhouse-client --host clickhouse --user flagsmith --password password --multiquery < /tmp/events.sql
+    depends_on:
+      clickhouse:
+        condition: service_healthy
+
+  ingestion-server:
+    image: flagsmith/events-ingestion-api:latest
+    environment:
+      KAFKA_BOOTSTRAP_SERVERS: kafka:9092
+      KAFKA_AUTH: none
+      KAFKA_TOPIC: flagsmith_warehouse_events
+      KAFKA_EXTERNAL_TOPIC: external_warehouse_events
+      DATABASE_URL: postgres://postgres:password@postgres:5432/flagsmith
+    ports:
+      - 8080:8080
+    depends_on:
+      migrate-db:
+        condition: service_completed_successfully
+      kafka-init:
+        condition: service_completed_successfully
+
+  warehouse-delivery:
+    image: flagsmith/warehouse-delivery:latest
+    environment:
+      KAFKA_BOOTSTRAP_SERVERS: kafka:9092
+      KAFKA_AUTH: none
+      DATABASE_URL: postgresql://postgres:password@postgres:5432/flagsmith
+      WAREHOUSE_CREDENTIALS_SECRET: *warehouse-credentials-secret
+    depends_on:
+      migrate-db:
+        condition: service_completed_successfully
+      kafka-init:
+        condition: service_completed_successfully
+      clickhouse-init:
+        condition: service_completed_successfully

docs/docs/deployment-self-hosting/hosting-guides/docker.md

@@ -16,6 +16,13 @@ docker-compose -f docker-compose.yml up
 
 Wait for the images to download and run, then visit `http://localhost:8000/`. As a first step, you will need to create a new account at [http://localhost:8000/signup](http://localhost:8000/signup)
 
+To run Flagsmith without [experimentation](#experimentation), use the flags-only file instead:
+
+```bash
+curl -o docker-compose.yml https://raw.githubusercontent.com/Flagsmith/flagsmith/main/docker-compose.flags-only.yml
+docker-compose -f docker-compose.yml up
+```
+
 ## Environment Variables
 
 As well as the Environment Variables specified in the [API](/deployment-self-hosting/core-configuration/environment-variables#api-environment-variables) and [Frontend](/deployment-self-hosting/core-configuration/environment-variables#frontend-environment-variables), you can also specify the following:
@@ -60,6 +67,16 @@ You can then access the admin dashboard at [http://localhost:8000/admin/](http:/
 
 The REST API stores all its data within a PostgreSQL database. Schema changes will be carried out automatically when upgrading using Django Migrations.
 
+### Experimentation
+
+These containers run [experimentation](/experimentation/connect-a-warehouse):
+
+- **Ingestion server (port 8080)**: receives exposure and conversion events from the SDKs.
+- **Kafka**: queues events between the ingestion server and warehouse delivery.
+- **Warehouse delivery**: writes events to each environment's connected warehouse.
+- **ClickHouse**: a warehouse you can connect from **Environment Settings > Warehouse**, using host `clickhouse`, port
+  `8123`, TLS off, database `flagsmith_exp`, username `flagsmith` and password `password`.
+
 ## Access Flagsmith Remotely
 
 You will need to either open ports into your Docker host or set up a reverse proxy to access the two Flagsmith services (dashboard and API). You will also need to configure the dashboard environment variable `API_URL`, which tells the dashboard where the REST API is located.

docs/docs/experimentation/connect-a-warehouse.md

@@ -6,6 +6,8 @@ description:
  Choose where experiment events are stored, the managed Flagsmith warehouse or your own ClickHouse or Databricks.
 ---
 
+import CodeBlock from '@theme/CodeBlock'; import eventsSql from '../../../experimentation/clickhouse/events.sql?raw';
+
 :::info Beta
 
 Experimentation is in beta.
@@ -76,34 +78,9 @@ warehouse delivery.
    `CREATE USER`, `GRANT`, `CREATE DATABASE` and `CREATE TABLE` privileges). Replace `<USER>` and `<CHANGE_ME_PASSWORD>`
    with your own values. This is a one-off setup step; the same SQL is available to copy from the Flagsmith dashboard.
 
-   ```sql
-   -- Create the database and events table
-   CREATE DATABASE IF NOT EXISTS flagsmith_exp;
-
-   CREATE TABLE IF NOT EXISTS flagsmith_exp.events
-   (
-       environment_key      LowCardinality(String),
-       event                LowCardinality(String),
-       feature_name         LowCardinality(String),
-       timestamp            DateTime64(3),
-       collected_at         DateTime64(3),
-       identifier           String,
-       value                String                          CODEC(ZSTD(3)),
-       traits               String                          CODEC(ZSTD(3)),
-       metadata             String                          CODEC(ZSTD(3)),
-       sdk_language         LowCardinality(String),
-       sdk_version          LowCardinality(String),
-
-       INDEX idx_identity identifier TYPE bloom_filter GRANULARITY 4,
-
-       CONSTRAINT environment_key_not_empty CHECK environment_key != '',
-       CONSTRAINT event_not_empty           CHECK event != '',
-       CONSTRAINT timestamp_sane            CHECK timestamp > toDateTime64('2020-01-01 00:00:00', 3)
-   )
-   ENGINE = MergeTree
-   PARTITION BY toYYYYMMDD(timestamp)
-   ORDER BY (environment_key, event, feature_name, timestamp, identifier);
+   <CodeBlock language="sql">{eventsSql}</CodeBlock>
 
+   ```sql
    -- Create a dedicated user for Flagsmith
    CREATE USER IF NOT EXISTS <USER>
        IDENTIFIED WITH sha256_password BY '<CHANGE_ME_PASSWORD>';

docs/docusaurus.config.ts

@@ -285,6 +285,13 @@ const config: Config = {
 
     plugins: [
         './plugins/flagsmith-versions',
+        // Lets pages import a file's contents as text, e.g. `import sql from '../file.sql?raw'`.
+        () => ({
+            name: 'raw-source',
+            configureWebpack: () => ({
+                module: { rules: [{ resourceQuery: /raw/, type: 'asset/source' }] },
+            }),
+        }),
         [
             'docusaurus-plugin-openapi-docs',
             {

experimentation/clickhouse/events.sql

@@ -0,0 +1,26 @@
+-- ClickHouse schema for experimentation events.
+CREATE DATABASE IF NOT EXISTS flagsmith_exp;
+
+CREATE TABLE IF NOT EXISTS flagsmith_exp.events
+(
+    environment_key      LowCardinality(String),
+    event                LowCardinality(String),
+    feature_name         LowCardinality(String),
+    timestamp            DateTime64(3),
+    collected_at         DateTime64(3),
+    identifier           String,
+    value                String                          CODEC(ZSTD(3)),
+    traits               String                          CODEC(ZSTD(3)),
+    metadata             String                          CODEC(ZSTD(3)),
+    sdk_language         LowCardinality(String),
+    sdk_version          LowCardinality(String),
+
+    INDEX idx_identity identifier TYPE bloom_filter GRANULARITY 4,
+
+    CONSTRAINT environment_key_not_empty CHECK environment_key != '',
+    CONSTRAINT event_not_empty           CHECK event != '',
+    CONSTRAINT timestamp_sane            CHECK timestamp > toDateTime64('2020-01-01 00:00:00', 3)
+)
+ENGINE = MergeTree
+PARTITION BY toYYYYMMDD(timestamp)
+ORDER BY (environment_key, event, feature_name, timestamp, identifier);

frontend/web/components/pages/environment-settings/tabs/warehouse-tab/__tests__/clickhouseSetupSql.test.ts

@@ -1,5 +1,3 @@
-import fs from 'fs'
-import path from 'path'
 import { CLICKHOUSE_DEFAULTS } from 'components/pages/environment-settings/tabs/warehouse-tab/clickhouseConfig'
 import {
   getClickHouseOnboardingSql,
@@ -46,22 +44,4 @@ describe('getClickHouseOnboardingSql', () => {
     expect(createTable).toBeLessThan(sql.indexOf('CREATE USER'))
     expect(createTable).toBeLessThan(sql.indexOf('GRANT'))
   })
-
-  it('matches the copy of the script in the documentation', () => {
-    const doc = fs.readFileSync(
-      path.join(
-        __dirname,
-        '../../../../../../../../docs/docs/experimentation/connect-a-warehouse.md',
-      ),
-      'utf8',
-    )
-    const fencedSql = doc.match(/```sql\n([\s\S]*?)```/)?.[1] ?? ''
-    const dedented = fencedSql
-      .split('\n')
-      .map((line) => line.replace(/^ {3}/, ''))
-      .join('\n')
-      .trim()
-
-    expect(dedented).toEqual(getClickHouseOnboardingSql())
-  })
 })