diff --git a/Makefile b/Makefile index c21f8c838e7c..37de04d6a6cc 100644 --- a/Makefile +++ b/Makefile @@ -17,3 +17,14 @@ lint: $(MAKE) -C api lint $(MAKE) -C docs lint $(MAKE) -C frontend lint + +# Run `make docker-up flags-only=1` to run Flagsmith without experimentation. +compose-file = $(if $(flags-only),docker-compose.flags-only.yml,docker-compose.yml) + +.PHONY: docker-up +docker-up: + docker compose -f $(compose-file) up -d + +.PHONY: docker-down +docker-down: + docker compose -f $(compose-file) down diff --git a/docker-compose.flags-only.yml b/docker-compose.flags-only.yml new file mode 100644 index 000000000000..f0dd480b6c97 --- /dev/null +++ b/docker-compose.flags-only.yml @@ -0,0 +1,90 @@ +# See https://docs.flagsmith.com/deployment/docker for more information on running Flagsmith in Docker +# This Docker Compose file will run the Flagsmith Platform without experimentation + +x-flagsmith-env: &flagsmith-env + # All environments variables are available here: + # API: https://docs.flagsmith.com/deployment/locally-api#environment-variables + # UI: https://docs.flagsmith.com/deployment/locally-frontend#environment-variables + + DATABASE_URL: postgresql://postgres:password@postgres:5432/flagsmith + USE_POSTGRES_FOR_ANALYTICS: 'true' # Store API and Flag Analytics data in Postgres + ENVIRONMENT: production # set to 'production' in production. + DJANGO_ALLOWED_HOSTS: '*' # Change this in production + FLAGSMITH_DOMAIN: localhost:8000 # Change this in production + DJANGO_SECRET_KEY: secret # Change this in production + # PREVENT_SIGNUP: 'true' # Uncomment to prevent any additional signups + # ALLOW_REGISTRATION_WITHOUT_INVITE: 'true' # Uncomment and set to false to only allow signups via invitations + + # Enable Task Processor + TASK_RUN_METHOD: TASK_PROCESSOR # other options are: SYNCHRONOUSLY, SEPARATE_THREAD (default) + PROMETHEUS_ENABLED: 'true' + + # Uncomment if you want to enable Google OAuth. Note this does not turn Google OAuth on. You still need to use + # Flagsmith on Flagsmith to enable it - https://docs.flagsmith.com/deployment/#oauth_google + # DJANGO_SECURE_CROSS_ORIGIN_OPENER_POLICY: 'same-origin-allow-popups' + + # For more info on configuring E-Mails - https://docs.flagsmith.com/deployment/locally-api#environment-variables + # Example SMTP: + # EMAIL_BACKEND: django.core.mail.backends.smtp.EmailBackend + # EMAIL_HOST: mail.example.com + # SENDER_EMAIL: flagsmith@example.com + # EMAIL_HOST_USER: flagsmith@example.com + # EMAIL_HOST_PASSWORD: smtp_account_password + # EMAIL_PORT: 587 # optional + # EMAIL_USE_TLS: 'true' # optional + +volumes: + pgdata: + +services: + postgres: + image: postgres:15.5-alpine + environment: + POSTGRES_PASSWORD: password + POSTGRES_DB: flagsmith + container_name: flagsmith_postgres + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ['CMD-SHELL', 'pg_isready -d flagsmith -U postgres'] + interval: 2s + timeout: 2s + retries: 20 + start_period: 20s + + migrate-db: + image: docker.flagsmith.com/flagsmith/flagsmith:latest + command: + - migrate + environment: + <<: *flagsmith-env + healthcheck: + disable: true + depends_on: + postgres: + condition: service_healthy + + flagsmith: + image: docker.flagsmith.com/flagsmith/flagsmith:latest + command: + - serve + environment: + <<: *flagsmith-env + ports: + - 8000:8000 + depends_on: + migrate-db: + condition: service_completed_successfully + + # The flagsmith_processor service is only needed if TASK_RUN_METHOD set to TASK_PROCESSOR + # in the application environment + flagsmith-task-processor: + image: docker.flagsmith.com/flagsmith/flagsmith:latest + environment: + <<: *flagsmith-env + ports: + - 8001:8000 + depends_on: + migrate-db: + condition: service_completed_successfully + command: run-task-processor diff --git a/docker-compose.yml b/docker-compose.yml index 464c81ca4808..f1064ea0b2ac 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,6 +12,7 @@ x-flagsmith-env: &flagsmith-env DJANGO_ALLOWED_HOSTS: '*' # Change this in production FLAGSMITH_DOMAIN: localhost:8000 # Change this in production DJANGO_SECRET_KEY: secret # Change this in production + WAREHOUSE_CREDENTIALS_SECRET: &warehouse-credentials-secret secret # Change this in production # PREVENT_SIGNUP: 'true' # Uncomment to prevent any additional signups # ALLOW_REGISTRATION_WITHOUT_INVITE: 'true' # Uncomment and set to false to only allow signups via invitations @@ -33,8 +34,12 @@ x-flagsmith-env: &flagsmith-env # EMAIL_PORT: 587 # optional # EMAIL_USE_TLS: 'true' # optional + # To run without experimentation, use docker-compose.flags-only.yml instead. + EXPERIMENTATION_ENABLED: 'true' + volumes: pgdata: + clickhouse-data: services: postgres: @@ -88,3 +93,99 @@ services: migrate-db: condition: service_completed_successfully command: run-task-processor + + # Experimentation services. + kafka: + image: apache/kafka:3.9.1 + environment: + KAFKA_NODE_ID: 1 + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_LISTENERS: PLAINTEXT://:9092,CONTROLLER://:9093 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092 + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + healthcheck: + test: ['CMD-SHELL', '/opt/kafka/bin/kafka-broker-api-versions.sh --bootstrap-server localhost:9092'] + interval: 5s + timeout: 10s + retries: 20 + + kafka-init: + image: apache/kafka:3.9.1 + entrypoint: ['/bin/sh', '-c'] + command: + - | + for topic in flagsmith_warehouse_events external_warehouse_events external_warehouse_events_retry; do + /opt/kafka/bin/kafka-topics.sh --bootstrap-server kafka:9092 --create --if-not-exists --topic $$topic + done + depends_on: + kafka: + condition: service_healthy + + # Connect it from Environment Settings > Warehouse > ClickHouse: host `clickhouse`, port 8123, + # TLS off, database `flagsmith_exp`, username `flagsmith`, password `password`. + clickhouse: + image: clickhouse/clickhouse-server:25.12 + environment: + CLICKHOUSE_USER: flagsmith + CLICKHOUSE_PASSWORD: password + ulimits: + nofile: + soft: 262144 + hard: 262144 + volumes: + - clickhouse-data:/var/lib/clickhouse + healthcheck: + test: ['CMD', 'wget', '--spider', '-q', 'http://localhost:8123/ping'] + interval: 5s + timeout: 3s + retries: 20 + + # Creates the events table from experimentation/clickhouse/events.sql. + clickhouse-init: + image: clickhouse/clickhouse-server:25.12 + entrypoint: ['/bin/sh', '-c'] + command: + - | + set -e + wget -qO /tmp/events.sql https://raw.githubusercontent.com/Flagsmith/flagsmith/main/experimentation/clickhouse/events.sql + clickhouse-client --host clickhouse --user flagsmith --password password --multiquery < /tmp/events.sql + depends_on: + clickhouse: + condition: service_healthy + + ingestion-server: + image: flagsmith/events-ingestion-api:latest + environment: + KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + KAFKA_AUTH: none + KAFKA_TOPIC: flagsmith_warehouse_events + KAFKA_EXTERNAL_TOPIC: external_warehouse_events + DATABASE_URL: postgres://postgres:password@postgres:5432/flagsmith + ports: + - 8080:8080 + depends_on: + migrate-db: + condition: service_completed_successfully + kafka-init: + condition: service_completed_successfully + + warehouse-delivery: + image: flagsmith/warehouse-delivery:latest + environment: + KAFKA_BOOTSTRAP_SERVERS: kafka:9092 + KAFKA_AUTH: none + DATABASE_URL: postgresql://postgres:password@postgres:5432/flagsmith + WAREHOUSE_CREDENTIALS_SECRET: *warehouse-credentials-secret + depends_on: + migrate-db: + condition: service_completed_successfully + kafka-init: + condition: service_completed_successfully + clickhouse-init: + condition: service_completed_successfully diff --git a/docs/docs/deployment-self-hosting/hosting-guides/docker.md b/docs/docs/deployment-self-hosting/hosting-guides/docker.md index 4010f66ed928..11cfbe05e6e9 100644 --- a/docs/docs/deployment-self-hosting/hosting-guides/docker.md +++ b/docs/docs/deployment-self-hosting/hosting-guides/docker.md @@ -16,6 +16,13 @@ docker-compose -f docker-compose.yml up Wait for the images to download and run, then visit `http://localhost:8000/`. As a first step, you will need to create a new account at [http://localhost:8000/signup](http://localhost:8000/signup) +To run Flagsmith without [experimentation](#experimentation), use the flags-only file instead: + +```bash +curl -o docker-compose.yml https://raw.githubusercontent.com/Flagsmith/flagsmith/main/docker-compose.flags-only.yml +docker-compose -f docker-compose.yml up +``` + ## Environment Variables As well as the Environment Variables specified in the [API](/deployment-self-hosting/core-configuration/environment-variables#api-environment-variables) and [Frontend](/deployment-self-hosting/core-configuration/environment-variables#frontend-environment-variables), you can also specify the following: @@ -60,6 +67,16 @@ You can then access the admin dashboard at [http://localhost:8000/admin/](http:/ The REST API stores all its data within a PostgreSQL database. Schema changes will be carried out automatically when upgrading using Django Migrations. +### Experimentation + +These containers run [experimentation](/experimentation/connect-a-warehouse): + +- **Ingestion server (port 8080)**: receives exposure and conversion events from the SDKs. +- **Kafka**: queues events between the ingestion server and warehouse delivery. +- **Warehouse delivery**: writes events to each environment's connected warehouse. +- **ClickHouse**: a warehouse you can connect from **Environment Settings > Warehouse**, using host `clickhouse`, port + `8123`, TLS off, database `flagsmith_exp`, username `flagsmith` and password `password`. + ## Access Flagsmith Remotely You will need to either open ports into your Docker host or set up a reverse proxy to access the two Flagsmith services (dashboard and API). You will also need to configure the dashboard environment variable `API_URL`, which tells the dashboard where the REST API is located. diff --git a/docs/docs/experimentation/connect-a-warehouse.md b/docs/docs/experimentation/connect-a-warehouse.md index 812970ad0080..56a92cc19319 100644 --- a/docs/docs/experimentation/connect-a-warehouse.md +++ b/docs/docs/experimentation/connect-a-warehouse.md @@ -6,6 +6,8 @@ description: Choose where experiment events are stored, the managed Flagsmith warehouse or your own ClickHouse or Databricks. --- +import CodeBlock from '@theme/CodeBlock'; import eventsSql from '../../../experimentation/clickhouse/events.sql?raw'; + :::info Beta Experimentation is in beta. @@ -76,34 +78,9 @@ warehouse delivery. `CREATE USER`, `GRANT`, `CREATE DATABASE` and `CREATE TABLE` privileges). Replace `` and `` with your own values. This is a one-off setup step; the same SQL is available to copy from the Flagsmith dashboard. - ```sql - -- Create the database and events table - CREATE DATABASE IF NOT EXISTS flagsmith_exp; - - CREATE TABLE IF NOT EXISTS flagsmith_exp.events - ( - environment_key LowCardinality(String), - event LowCardinality(String), - feature_name LowCardinality(String), - timestamp DateTime64(3), - collected_at DateTime64(3), - identifier String, - value String CODEC(ZSTD(3)), - traits String CODEC(ZSTD(3)), - metadata String CODEC(ZSTD(3)), - sdk_language LowCardinality(String), - sdk_version LowCardinality(String), - - INDEX idx_identity identifier TYPE bloom_filter GRANULARITY 4, - - CONSTRAINT environment_key_not_empty CHECK environment_key != '', - CONSTRAINT event_not_empty CHECK event != '', - CONSTRAINT timestamp_sane CHECK timestamp > toDateTime64('2020-01-01 00:00:00', 3) - ) - ENGINE = MergeTree - PARTITION BY toYYYYMMDD(timestamp) - ORDER BY (environment_key, event, feature_name, timestamp, identifier); + {eventsSql} + ```sql -- Create a dedicated user for Flagsmith CREATE USER IF NOT EXISTS IDENTIFIED WITH sha256_password BY ''; diff --git a/docs/docusaurus.config.ts b/docs/docusaurus.config.ts index af1287e18695..a78746d2626c 100644 --- a/docs/docusaurus.config.ts +++ b/docs/docusaurus.config.ts @@ -285,6 +285,13 @@ const config: Config = { plugins: [ './plugins/flagsmith-versions', + // Lets pages import a file's contents as text, e.g. `import sql from '../file.sql?raw'`. + () => ({ + name: 'raw-source', + configureWebpack: () => ({ + module: { rules: [{ resourceQuery: /raw/, type: 'asset/source' }] }, + }), + }), [ 'docusaurus-plugin-openapi-docs', { diff --git a/experimentation/clickhouse/events.sql b/experimentation/clickhouse/events.sql new file mode 100644 index 000000000000..0c71e1c8a02c --- /dev/null +++ b/experimentation/clickhouse/events.sql @@ -0,0 +1,26 @@ +-- ClickHouse schema for experimentation events. +CREATE DATABASE IF NOT EXISTS flagsmith_exp; + +CREATE TABLE IF NOT EXISTS flagsmith_exp.events +( + environment_key LowCardinality(String), + event LowCardinality(String), + feature_name LowCardinality(String), + timestamp DateTime64(3), + collected_at DateTime64(3), + identifier String, + value String CODEC(ZSTD(3)), + traits String CODEC(ZSTD(3)), + metadata String CODEC(ZSTD(3)), + sdk_language LowCardinality(String), + sdk_version LowCardinality(String), + + INDEX idx_identity identifier TYPE bloom_filter GRANULARITY 4, + + CONSTRAINT environment_key_not_empty CHECK environment_key != '', + CONSTRAINT event_not_empty CHECK event != '', + CONSTRAINT timestamp_sane CHECK timestamp > toDateTime64('2020-01-01 00:00:00', 3) +) +ENGINE = MergeTree +PARTITION BY toYYYYMMDD(timestamp) +ORDER BY (environment_key, event, feature_name, timestamp, identifier); diff --git a/frontend/web/components/pages/environment-settings/tabs/warehouse-tab/__tests__/clickhouseSetupSql.test.ts b/frontend/web/components/pages/environment-settings/tabs/warehouse-tab/__tests__/clickhouseSetupSql.test.ts index dee3e4b38eea..d2c7b32f439a 100644 --- a/frontend/web/components/pages/environment-settings/tabs/warehouse-tab/__tests__/clickhouseSetupSql.test.ts +++ b/frontend/web/components/pages/environment-settings/tabs/warehouse-tab/__tests__/clickhouseSetupSql.test.ts @@ -1,5 +1,3 @@ -import fs from 'fs' -import path from 'path' import { CLICKHOUSE_DEFAULTS } from 'components/pages/environment-settings/tabs/warehouse-tab/clickhouseConfig' import { getClickHouseOnboardingSql, @@ -46,22 +44,4 @@ describe('getClickHouseOnboardingSql', () => { expect(createTable).toBeLessThan(sql.indexOf('CREATE USER')) expect(createTable).toBeLessThan(sql.indexOf('GRANT')) }) - - it('matches the copy of the script in the documentation', () => { - const doc = fs.readFileSync( - path.join( - __dirname, - '../../../../../../../../docs/docs/experimentation/connect-a-warehouse.md', - ), - 'utf8', - ) - const fencedSql = doc.match(/```sql\n([\s\S]*?)```/)?.[1] ?? '' - const dedented = fencedSql - .split('\n') - .map((line) => line.replace(/^ {3}/, '')) - .join('\n') - .trim() - - expect(dedented).toEqual(getClickHouseOnboardingSql()) - }) })