diff --git a/.github/workflows/backport_branches.yml b/.github/workflows/backport_branches.yml index cc3b3701555ac..5038bdc28089b 100644 --- a/.github/workflows/backport_branches.yml +++ b/.github/workflows/backport_branches.yml @@ -777,11 +777,11 @@ jobs: prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_old_analyzer_1_6: + integration_tests_amd_asan_db_disk_old_analyzer_1_8: runs-on: [self-hosted, amd-medium] needs: [build_amd_asan, config_workflow, dockers_build_amd, dockers_build_arm] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDEvNik=') }} - name: "Integration tests (amd_asan, db disk, old analyzer, 1/6)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDEvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 1/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -811,15 +811,15 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 1/6)' --workflow "BackportPR" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 1/8)' --workflow "BackportPR" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_old_analyzer_2_6: + integration_tests_amd_asan_db_disk_old_analyzer_2_8: runs-on: [self-hosted, amd-medium] needs: [build_amd_asan, config_workflow, dockers_build_amd, dockers_build_arm] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDIvNik=') }} - name: "Integration tests (amd_asan, db disk, old analyzer, 2/6)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDIvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 2/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -849,15 +849,15 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 2/6)' --workflow "BackportPR" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 2/8)' --workflow "BackportPR" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_old_analyzer_3_6: + integration_tests_amd_asan_db_disk_old_analyzer_3_8: runs-on: [self-hosted, amd-medium] needs: [build_amd_asan, config_workflow, dockers_build_amd, dockers_build_arm] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDMvNik=') }} - name: "Integration tests (amd_asan, db disk, old analyzer, 3/6)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDMvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 3/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -887,15 +887,15 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 3/6)' --workflow "BackportPR" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 3/8)' --workflow "BackportPR" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_old_analyzer_4_6: + integration_tests_amd_asan_db_disk_old_analyzer_4_8: runs-on: [self-hosted, amd-medium] needs: [build_amd_asan, config_workflow, dockers_build_amd, dockers_build_arm] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDQvNik=') }} - name: "Integration tests (amd_asan, db disk, old analyzer, 4/6)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDQvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 4/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -925,15 +925,15 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 4/6)' --workflow "BackportPR" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 4/8)' --workflow "BackportPR" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_old_analyzer_5_6: + integration_tests_amd_asan_db_disk_old_analyzer_5_8: runs-on: [self-hosted, amd-medium] needs: [build_amd_asan, config_workflow, dockers_build_amd, dockers_build_arm] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDUvNik=') }} - name: "Integration tests (amd_asan, db disk, old analyzer, 5/6)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDUvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 5/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -963,15 +963,15 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 5/6)' --workflow "BackportPR" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 5/8)' --workflow "BackportPR" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_old_analyzer_6_6: + integration_tests_amd_asan_db_disk_old_analyzer_6_8: runs-on: [self-hosted, amd-medium] needs: [build_amd_asan, config_workflow, dockers_build_amd, dockers_build_arm] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDYvNik=') }} - name: "Integration tests (amd_asan, db disk, old analyzer, 6/6)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDYvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 6/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1001,7 +1001,83 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 6/6)' --workflow "BackportPR" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 6/8)' --workflow "BackportPR" --ci 2>&1 | python3 -u -c 'import sys,datetime + prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") + for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log + + integration_tests_amd_asan_db_disk_old_analyzer_7_8: + runs-on: [self-hosted, amd-medium] + needs: [build_amd_asan, config_workflow, dockers_build_amd, dockers_build_arm] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDcvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 7/8)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + set -o pipefail + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 7/8)' --workflow "BackportPR" --ci 2>&1 | python3 -u -c 'import sys,datetime + prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") + for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log + + integration_tests_amd_asan_db_disk_old_analyzer_8_8: + runs-on: [self-hosted, amd-medium] + needs: [build_amd_asan, config_workflow, dockers_build_amd, dockers_build_arm] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDgvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 8/8)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + set -o pipefail + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 8/8)' --workflow "BackportPR" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log @@ -1235,7 +1311,7 @@ jobs: finish_workflow: runs-on: [self-hosted, style-checker-aarch64] - needs: [build_amd_asan, build_amd_darwin, build_amd_debug, build_amd_release, build_amd_tsan, build_arm_darwin, build_arm_release, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_db_disk_old_analyzer_1_6, integration_tests_amd_asan_db_disk_old_analyzer_2_6, integration_tests_amd_asan_db_disk_old_analyzer_3_6, integration_tests_amd_asan_db_disk_old_analyzer_4_6, integration_tests_amd_asan_db_disk_old_analyzer_5_6, integration_tests_amd_asan_db_disk_old_analyzer_6_6, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, stateless_tests_amd_asan_db_disk_distributed_plan_sequential, stateless_tests_amd_asan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_distributed_plan_parallel_2_2, stress_test_amd_tsan] + needs: [build_amd_asan, build_amd_darwin, build_amd_debug, build_amd_release, build_amd_tsan, build_arm_darwin, build_arm_release, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_db_disk_old_analyzer_8_8, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, stateless_tests_amd_asan_db_disk_distributed_plan_sequential, stateless_tests_amd_asan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_distributed_plan_parallel_2_2, stress_test_amd_tsan] if: ${{ always() }} name: "Finish Workflow" outputs: diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 276934a463192..dfeca755e74be 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -2986,11 +2986,11 @@ jobs: prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_old_analyzer_1_6: + integration_tests_amd_asan_db_disk_old_analyzer_1_8: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] needs: [build_amd_asan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDEvNik=') }} - name: "Integration tests (amd_asan, db disk, old analyzer, 1/6)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDEvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 1/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -3005,7 +3005,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Integration tests (amd_asan, db disk, old analyzer, 1/6)" + test_name: "Integration tests (amd_asan, db disk, old analyzer, 1/8)" - name: Prepare env script run: | @@ -3027,15 +3027,15 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 1/6)' --workflow "MasterCI" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 1/8)' --workflow "MasterCI" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_old_analyzer_2_6: + integration_tests_amd_asan_db_disk_old_analyzer_2_8: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] needs: [build_amd_asan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDIvNik=') }} - name: "Integration tests (amd_asan, db disk, old analyzer, 2/6)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDIvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 2/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -3050,7 +3050,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Integration tests (amd_asan, db disk, old analyzer, 2/6)" + test_name: "Integration tests (amd_asan, db disk, old analyzer, 2/8)" - name: Prepare env script run: | @@ -3072,15 +3072,15 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 2/6)' --workflow "MasterCI" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 2/8)' --workflow "MasterCI" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_old_analyzer_3_6: + integration_tests_amd_asan_db_disk_old_analyzer_3_8: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] needs: [build_amd_asan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDMvNik=') }} - name: "Integration tests (amd_asan, db disk, old analyzer, 3/6)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDMvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 3/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -3095,7 +3095,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Integration tests (amd_asan, db disk, old analyzer, 3/6)" + test_name: "Integration tests (amd_asan, db disk, old analyzer, 3/8)" - name: Prepare env script run: | @@ -3117,15 +3117,15 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 3/6)' --workflow "MasterCI" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 3/8)' --workflow "MasterCI" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_old_analyzer_4_6: + integration_tests_amd_asan_db_disk_old_analyzer_4_8: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] needs: [build_amd_asan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDQvNik=') }} - name: "Integration tests (amd_asan, db disk, old analyzer, 4/6)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDQvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 4/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -3140,7 +3140,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Integration tests (amd_asan, db disk, old analyzer, 4/6)" + test_name: "Integration tests (amd_asan, db disk, old analyzer, 4/8)" - name: Prepare env script run: | @@ -3162,15 +3162,15 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 4/6)' --workflow "MasterCI" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 4/8)' --workflow "MasterCI" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_old_analyzer_5_6: + integration_tests_amd_asan_db_disk_old_analyzer_5_8: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] needs: [build_amd_asan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDUvNik=') }} - name: "Integration tests (amd_asan, db disk, old analyzer, 5/6)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDUvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 5/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -3185,7 +3185,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Integration tests (amd_asan, db disk, old analyzer, 5/6)" + test_name: "Integration tests (amd_asan, db disk, old analyzer, 5/8)" - name: Prepare env script run: | @@ -3207,15 +3207,15 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 5/6)' --workflow "MasterCI" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 5/8)' --workflow "MasterCI" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_old_analyzer_6_6: + integration_tests_amd_asan_db_disk_old_analyzer_6_8: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] needs: [build_amd_asan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDYvNik=') }} - name: "Integration tests (amd_asan, db disk, old analyzer, 6/6)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDYvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 6/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -3230,7 +3230,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Integration tests (amd_asan, db disk, old analyzer, 6/6)" + test_name: "Integration tests (amd_asan, db disk, old analyzer, 6/8)" - name: Prepare env script run: | @@ -3252,7 +3252,97 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 6/6)' --workflow "MasterCI" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 6/8)' --workflow "MasterCI" --ci 2>&1 | python3 -u -c 'import sys,datetime + prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") + for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log + + integration_tests_amd_asan_db_disk_old_analyzer_7_8: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] + needs: [build_amd_asan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDcvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 7/8)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Integration tests (amd_asan, db disk, old analyzer, 7/8)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + set -o pipefail + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 7/8)' --workflow "MasterCI" --ci 2>&1 | python3 -u -c 'import sys,datetime + prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") + for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log + + integration_tests_amd_asan_db_disk_old_analyzer_8_8: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] + needs: [build_amd_asan, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDgvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 8/8)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Integration tests (amd_asan, db disk, old analyzer, 8/8)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + set -o pipefail + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 8/8)' --workflow "MasterCI" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log @@ -5633,7 +5723,7 @@ jobs: finish_workflow: runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64] - needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_amd_ubsan, ast_fuzzer_arm_asan, build_amd_asan, build_amd_binary, build_amd_coverage, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_amd_ubsan, build_arm_asan, build_arm_binary, build_arm_release, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_amd_ubsan, buzzhouse_arm_asan, clickbench_amd_release, clickbench_arm_release, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_db_disk_old_analyzer_1_6, integration_tests_amd_asan_db_disk_old_analyzer_2_6, integration_tests_amd_asan_db_disk_old_analyzer_3_6, integration_tests_amd_asan_db_disk_old_analyzer_4_6, integration_tests_amd_asan_db_disk_old_analyzer_5_6, integration_tests_amd_asan_db_disk_old_analyzer_6_6, integration_tests_amd_binary_1_5, integration_tests_amd_binary_2_5, integration_tests_amd_binary_3_5, integration_tests_amd_binary_4_5, integration_tests_amd_binary_5_5, integration_tests_amd_msan_1_6, integration_tests_amd_msan_2_6, integration_tests_amd_msan_3_6, integration_tests_amd_msan_4_6, integration_tests_amd_msan_5_6, integration_tests_amd_msan_6_6, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, sqllogic_test, sqltest, stateless_tests_amd_asan_db_disk_distributed_plan_sequential, stateless_tests_amd_asan_distributed_plan_parallel_1_4, stateless_tests_amd_asan_distributed_plan_parallel_2_4, stateless_tests_amd_asan_distributed_plan_parallel_3_4, stateless_tests_amd_asan_distributed_plan_parallel_4_4, stateless_tests_amd_coverage_1_8, stateless_tests_amd_coverage_2_8, stateless_tests_amd_coverage_3_8, stateless_tests_amd_coverage_4_8, stateless_tests_amd_coverage_5_8, stateless_tests_amd_coverage_6_8, stateless_tests_amd_coverage_7_8, stateless_tests_amd_coverage_8_8, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_msan_wasmedge_parallel_1_4, stateless_tests_amd_msan_wasmedge_parallel_2_4, stateless_tests_amd_msan_wasmedge_parallel_3_4, stateless_tests_amd_msan_wasmedge_parallel_4_4, stateless_tests_amd_msan_wasmedge_sequential_1_2, stateless_tests_amd_msan_wasmedge_sequential_2_2, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_amd_tsan_s3_storage_parallel_1_2, stateless_tests_amd_tsan_s3_storage_parallel_2_2, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_amd_ubsan_parallel, stateless_tests_amd_ubsan_sequential, stateless_tests_arm_asan_azure_parallel_1_4, stateless_tests_arm_asan_azure_parallel_2_4, stateless_tests_arm_asan_azure_parallel_3_4, stateless_tests_arm_asan_azure_parallel_4_4, stateless_tests_arm_asan_azure_sequential_1_2, stateless_tests_arm_asan_azure_sequential_2_2, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_release, stress_test_amd_tsan, stress_test_amd_ubsan, stress_test_arm_asan, stress_test_arm_asan_s3, stress_test_azure_amd_msan, stress_test_azure_amd_tsan, unit_tests_asan, unit_tests_msan, unit_tests_tsan, unit_tests_ubsan] + needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_amd_ubsan, ast_fuzzer_arm_asan, build_amd_asan, build_amd_binary, build_amd_coverage, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_amd_ubsan, build_arm_asan, build_arm_binary, build_arm_release, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_amd_ubsan, buzzhouse_arm_asan, clickbench_amd_release, clickbench_arm_release, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_db_disk_old_analyzer_8_8, integration_tests_amd_binary_1_5, integration_tests_amd_binary_2_5, integration_tests_amd_binary_3_5, integration_tests_amd_binary_4_5, integration_tests_amd_binary_5_5, integration_tests_amd_msan_1_6, integration_tests_amd_msan_2_6, integration_tests_amd_msan_3_6, integration_tests_amd_msan_4_6, integration_tests_amd_msan_5_6, integration_tests_amd_msan_6_6, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, sqllogic_test, sqltest, stateless_tests_amd_asan_db_disk_distributed_plan_sequential, stateless_tests_amd_asan_distributed_plan_parallel_1_4, stateless_tests_amd_asan_distributed_plan_parallel_2_4, stateless_tests_amd_asan_distributed_plan_parallel_3_4, stateless_tests_amd_asan_distributed_plan_parallel_4_4, stateless_tests_amd_coverage_1_8, stateless_tests_amd_coverage_2_8, stateless_tests_amd_coverage_3_8, stateless_tests_amd_coverage_4_8, stateless_tests_amd_coverage_5_8, stateless_tests_amd_coverage_6_8, stateless_tests_amd_coverage_7_8, stateless_tests_amd_coverage_8_8, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_msan_wasmedge_parallel_1_4, stateless_tests_amd_msan_wasmedge_parallel_2_4, stateless_tests_amd_msan_wasmedge_parallel_3_4, stateless_tests_amd_msan_wasmedge_parallel_4_4, stateless_tests_amd_msan_wasmedge_sequential_1_2, stateless_tests_amd_msan_wasmedge_sequential_2_2, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_amd_tsan_s3_storage_parallel_1_2, stateless_tests_amd_tsan_s3_storage_parallel_2_2, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_amd_ubsan_parallel, stateless_tests_amd_ubsan_sequential, stateless_tests_arm_asan_azure_parallel_1_4, stateless_tests_arm_asan_azure_parallel_2_4, stateless_tests_arm_asan_azure_parallel_3_4, stateless_tests_arm_asan_azure_parallel_4_4, stateless_tests_arm_asan_azure_sequential_1_2, stateless_tests_arm_asan_azure_sequential_2_2, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_release, stress_test_amd_tsan, stress_test_amd_ubsan, stress_test_arm_asan, stress_test_arm_asan_s3, stress_test_azure_amd_msan, stress_test_azure_amd_tsan, unit_tests_asan, unit_tests_msan, unit_tests_tsan, unit_tests_ubsan] if: ${{ always() && needs.config_workflow.outputs.pipeline_status != '' }} name: "Finish Workflow" outputs: @@ -5788,12 +5878,14 @@ jobs: - stateless_tests_arm_asan_azure_parallel_4_4 - stateless_tests_arm_asan_azure_sequential_1_2 - stateless_tests_arm_asan_azure_sequential_2_2 - - integration_tests_amd_asan_db_disk_old_analyzer_1_6 - - integration_tests_amd_asan_db_disk_old_analyzer_2_6 - - integration_tests_amd_asan_db_disk_old_analyzer_3_6 - - integration_tests_amd_asan_db_disk_old_analyzer_4_6 - - integration_tests_amd_asan_db_disk_old_analyzer_5_6 - - integration_tests_amd_asan_db_disk_old_analyzer_6_6 + - integration_tests_amd_asan_db_disk_old_analyzer_1_8 + - integration_tests_amd_asan_db_disk_old_analyzer_2_8 + - integration_tests_amd_asan_db_disk_old_analyzer_3_8 + - integration_tests_amd_asan_db_disk_old_analyzer_4_8 + - integration_tests_amd_asan_db_disk_old_analyzer_5_8 + - integration_tests_amd_asan_db_disk_old_analyzer_6_8 + - integration_tests_amd_asan_db_disk_old_analyzer_7_8 + - integration_tests_amd_asan_db_disk_old_analyzer_8_8 - integration_tests_amd_binary_1_5 - integration_tests_amd_binary_2_5 - integration_tests_amd_binary_3_5 diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index b71270cb3c475..eb283d43733d4 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -2842,11 +2842,11 @@ jobs: prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_old_analyzer_1_6: + integration_tests_amd_asan_db_disk_old_analyzer_1_8: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] needs: [build_amd_asan, build_amd_debug, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, stateless_tests_amd_asan_distributed_plan_parallel_1_4, stateless_tests_amd_asan_distributed_plan_parallel_2_4, stateless_tests_amd_asan_distributed_plan_parallel_3_4, stateless_tests_amd_asan_distributed_plan_parallel_4_4, stateless_tests_amd_debug_parallel, stateless_tests_arm_binary_parallel] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDEvNik=') }} - name: "Integration tests (amd_asan, db disk, old analyzer, 1/6)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDEvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 1/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2861,7 +2861,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Integration tests (amd_asan, db disk, old analyzer, 1/6)" + test_name: "Integration tests (amd_asan, db disk, old analyzer, 1/8)" - name: Prepare env script run: | @@ -2883,15 +2883,15 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 1/6)' --workflow "PR" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 1/8)' --workflow "PR" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_old_analyzer_2_6: + integration_tests_amd_asan_db_disk_old_analyzer_2_8: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] needs: [build_amd_asan, build_amd_debug, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, stateless_tests_amd_asan_distributed_plan_parallel_1_4, stateless_tests_amd_asan_distributed_plan_parallel_2_4, stateless_tests_amd_asan_distributed_plan_parallel_3_4, stateless_tests_amd_asan_distributed_plan_parallel_4_4, stateless_tests_amd_debug_parallel, stateless_tests_arm_binary_parallel] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDIvNik=') }} - name: "Integration tests (amd_asan, db disk, old analyzer, 2/6)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDIvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 2/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2906,7 +2906,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Integration tests (amd_asan, db disk, old analyzer, 2/6)" + test_name: "Integration tests (amd_asan, db disk, old analyzer, 2/8)" - name: Prepare env script run: | @@ -2928,15 +2928,15 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 2/6)' --workflow "PR" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 2/8)' --workflow "PR" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_old_analyzer_3_6: + integration_tests_amd_asan_db_disk_old_analyzer_3_8: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] needs: [build_amd_asan, build_amd_debug, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, stateless_tests_amd_asan_distributed_plan_parallel_1_4, stateless_tests_amd_asan_distributed_plan_parallel_2_4, stateless_tests_amd_asan_distributed_plan_parallel_3_4, stateless_tests_amd_asan_distributed_plan_parallel_4_4, stateless_tests_amd_debug_parallel, stateless_tests_arm_binary_parallel] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDMvNik=') }} - name: "Integration tests (amd_asan, db disk, old analyzer, 3/6)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDMvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 3/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2951,7 +2951,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Integration tests (amd_asan, db disk, old analyzer, 3/6)" + test_name: "Integration tests (amd_asan, db disk, old analyzer, 3/8)" - name: Prepare env script run: | @@ -2973,15 +2973,15 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 3/6)' --workflow "PR" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 3/8)' --workflow "PR" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_old_analyzer_4_6: + integration_tests_amd_asan_db_disk_old_analyzer_4_8: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] needs: [build_amd_asan, build_amd_debug, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, stateless_tests_amd_asan_distributed_plan_parallel_1_4, stateless_tests_amd_asan_distributed_plan_parallel_2_4, stateless_tests_amd_asan_distributed_plan_parallel_3_4, stateless_tests_amd_asan_distributed_plan_parallel_4_4, stateless_tests_amd_debug_parallel, stateless_tests_arm_binary_parallel] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDQvNik=') }} - name: "Integration tests (amd_asan, db disk, old analyzer, 4/6)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDQvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 4/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2996,7 +2996,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Integration tests (amd_asan, db disk, old analyzer, 4/6)" + test_name: "Integration tests (amd_asan, db disk, old analyzer, 4/8)" - name: Prepare env script run: | @@ -3018,15 +3018,15 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 4/6)' --workflow "PR" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 4/8)' --workflow "PR" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_old_analyzer_5_6: + integration_tests_amd_asan_db_disk_old_analyzer_5_8: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] needs: [build_amd_asan, build_amd_debug, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, stateless_tests_amd_asan_distributed_plan_parallel_1_4, stateless_tests_amd_asan_distributed_plan_parallel_2_4, stateless_tests_amd_asan_distributed_plan_parallel_3_4, stateless_tests_amd_asan_distributed_plan_parallel_4_4, stateless_tests_amd_debug_parallel, stateless_tests_arm_binary_parallel] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDUvNik=') }} - name: "Integration tests (amd_asan, db disk, old analyzer, 5/6)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDUvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 5/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -3041,7 +3041,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Integration tests (amd_asan, db disk, old analyzer, 5/6)" + test_name: "Integration tests (amd_asan, db disk, old analyzer, 5/8)" - name: Prepare env script run: | @@ -3063,15 +3063,15 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 5/6)' --workflow "PR" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 5/8)' --workflow "PR" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_old_analyzer_6_6: + integration_tests_amd_asan_db_disk_old_analyzer_6_8: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] needs: [build_amd_asan, build_amd_debug, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, stateless_tests_amd_asan_distributed_plan_parallel_1_4, stateless_tests_amd_asan_distributed_plan_parallel_2_4, stateless_tests_amd_asan_distributed_plan_parallel_3_4, stateless_tests_amd_asan_distributed_plan_parallel_4_4, stateless_tests_amd_debug_parallel, stateless_tests_arm_binary_parallel] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDYvNik=') }} - name: "Integration tests (amd_asan, db disk, old analyzer, 6/6)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDYvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 6/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -3086,7 +3086,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Integration tests (amd_asan, db disk, old analyzer, 6/6)" + test_name: "Integration tests (amd_asan, db disk, old analyzer, 6/8)" - name: Prepare env script run: | @@ -3108,7 +3108,97 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 6/6)' --workflow "PR" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 6/8)' --workflow "PR" --ci 2>&1 | python3 -u -c 'import sys,datetime + prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") + for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log + + integration_tests_amd_asan_db_disk_old_analyzer_7_8: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] + needs: [build_amd_asan, build_amd_debug, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, stateless_tests_amd_asan_distributed_plan_parallel_1_4, stateless_tests_amd_asan_distributed_plan_parallel_2_4, stateless_tests_amd_asan_distributed_plan_parallel_3_4, stateless_tests_amd_asan_distributed_plan_parallel_4_4, stateless_tests_amd_debug_parallel, stateless_tests_arm_binary_parallel] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDcvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 7/8)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Integration tests (amd_asan, db disk, old analyzer, 7/8)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + set -o pipefail + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 7/8)' --workflow "PR" --ci 2>&1 | python3 -u -c 'import sys,datetime + prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") + for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log + + integration_tests_amd_asan_db_disk_old_analyzer_8_8: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] + needs: [build_amd_asan, build_amd_debug, build_arm_binary, config_workflow, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, stateless_tests_amd_asan_distributed_plan_parallel_1_4, stateless_tests_amd_asan_distributed_plan_parallel_2_4, stateless_tests_amd_asan_distributed_plan_parallel_3_4, stateless_tests_amd_asan_distributed_plan_parallel_4_4, stateless_tests_amd_debug_parallel, stateless_tests_arm_binary_parallel] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDgvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 8/8)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Integration tests (amd_asan, db disk, old analyzer, 8/8)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + set -o pipefail + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 8/8)' --workflow "PR" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log @@ -5444,7 +5534,7 @@ jobs: finish_workflow: runs-on: [self-hosted, altinity-on-demand, altinity-style-checker-aarch64] - needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_debug_targeted, ast_fuzzer_amd_debug_targeted_old_compatibility, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_amd_ubsan, ast_fuzzer_arm_asan, build_amd_asan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_amd_ubsan, build_arm_asan, build_arm_binary, build_arm_release, build_arm_tsan, build_toolchain_pgo_bolt_aarch64, build_toolchain_pgo_bolt_amd64, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_amd_ubsan, buzzhouse_arm_asan, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_db_disk_old_analyzer_1_6, integration_tests_amd_asan_db_disk_old_analyzer_2_6, integration_tests_amd_asan_db_disk_old_analyzer_3_6, integration_tests_amd_asan_db_disk_old_analyzer_4_6, integration_tests_amd_asan_db_disk_old_analyzer_5_6, integration_tests_amd_asan_db_disk_old_analyzer_6_6, integration_tests_amd_asan_targeted, integration_tests_amd_binary_1_5, integration_tests_amd_binary_2_5, integration_tests_amd_binary_3_5, integration_tests_amd_binary_4_5, integration_tests_amd_binary_5_5, integration_tests_amd_msan_1_6, integration_tests_amd_msan_2_6, integration_tests_amd_msan_3_6, integration_tests_amd_msan_4_6, integration_tests_amd_msan_5_6, integration_tests_amd_msan_6_6, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, quick_functional_tests, sqllogic_test, stateless_tests_amd_asan_db_disk_distributed_plan_sequential, stateless_tests_amd_asan_distributed_plan_parallel_1_4, stateless_tests_amd_asan_distributed_plan_parallel_2_4, stateless_tests_amd_asan_distributed_plan_parallel_3_4, stateless_tests_amd_asan_distributed_plan_parallel_4_4, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_msan_wasmedge_parallel_1_4, stateless_tests_amd_msan_wasmedge_parallel_2_4, stateless_tests_amd_msan_wasmedge_parallel_3_4, stateless_tests_amd_msan_wasmedge_parallel_4_4, stateless_tests_amd_msan_wasmedge_sequential_1_2, stateless_tests_amd_msan_wasmedge_sequential_2_2, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_amd_tsan_s3_storage_parallel_1_2, stateless_tests_amd_tsan_s3_storage_parallel_2_2, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_amd_ubsan_parallel, stateless_tests_amd_ubsan_sequential, stateless_tests_arm_asan_azure_parallel_1_4, stateless_tests_arm_asan_azure_parallel_2_4, stateless_tests_arm_asan_azure_parallel_3_4, stateless_tests_arm_asan_azure_parallel_4_4, stateless_tests_arm_asan_azure_sequential_1_2, stateless_tests_arm_asan_azure_sequential_2_2, stateless_tests_arm_asan_targeted, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_release, stress_test_amd_tsan, stress_test_amd_ubsan, stress_test_arm_asan, stress_test_arm_asan_s3, unit_tests_asan, unit_tests_msan, unit_tests_tsan, unit_tests_ubsan] + needs: [ast_fuzzer_amd_debug, ast_fuzzer_amd_debug_targeted, ast_fuzzer_amd_debug_targeted_old_compatibility, ast_fuzzer_amd_msan, ast_fuzzer_amd_tsan, ast_fuzzer_amd_ubsan, ast_fuzzer_arm_asan, build_amd_asan, build_amd_binary, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_amd_ubsan, build_arm_asan, build_arm_binary, build_arm_release, build_arm_tsan, build_toolchain_pgo_bolt_aarch64, build_toolchain_pgo_bolt_amd64, buzzhouse_amd_debug, buzzhouse_amd_msan, buzzhouse_amd_tsan, buzzhouse_amd_ubsan, buzzhouse_arm_asan, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, dockers_build_multiplatform_manifest, fast_test, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_db_disk_old_analyzer_8_8, integration_tests_amd_asan_targeted, integration_tests_amd_binary_1_5, integration_tests_amd_binary_2_5, integration_tests_amd_binary_3_5, integration_tests_amd_binary_4_5, integration_tests_amd_binary_5_5, integration_tests_amd_msan_1_6, integration_tests_amd_msan_2_6, integration_tests_amd_msan_3_6, integration_tests_amd_msan_4_6, integration_tests_amd_msan_5_6, integration_tests_amd_msan_6_6, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, integration_tests_arm_binary_distributed_plan_1_4, integration_tests_arm_binary_distributed_plan_2_4, integration_tests_arm_binary_distributed_plan_3_4, integration_tests_arm_binary_distributed_plan_4_4, quick_functional_tests, sqllogic_test, stateless_tests_amd_asan_db_disk_distributed_plan_sequential, stateless_tests_amd_asan_distributed_plan_parallel_1_4, stateless_tests_amd_asan_distributed_plan_parallel_2_4, stateless_tests_amd_asan_distributed_plan_parallel_3_4, stateless_tests_amd_asan_distributed_plan_parallel_4_4, stateless_tests_amd_debug_distributed_plan_s3_storage_parallel, stateless_tests_amd_debug_distributed_plan_s3_storage_sequential, stateless_tests_amd_debug_parallel, stateless_tests_amd_debug_sequential, stateless_tests_amd_msan_wasmedge_parallel_1_4, stateless_tests_amd_msan_wasmedge_parallel_2_4, stateless_tests_amd_msan_wasmedge_parallel_3_4, stateless_tests_amd_msan_wasmedge_parallel_4_4, stateless_tests_amd_msan_wasmedge_sequential_1_2, stateless_tests_amd_msan_wasmedge_sequential_2_2, stateless_tests_amd_tsan_parallel_1_2, stateless_tests_amd_tsan_parallel_2_2, stateless_tests_amd_tsan_s3_storage_parallel_1_2, stateless_tests_amd_tsan_s3_storage_parallel_2_2, stateless_tests_amd_tsan_s3_storage_sequential_1_2, stateless_tests_amd_tsan_s3_storage_sequential_2_2, stateless_tests_amd_tsan_sequential_1_2, stateless_tests_amd_tsan_sequential_2_2, stateless_tests_amd_ubsan_parallel, stateless_tests_amd_ubsan_sequential, stateless_tests_arm_asan_azure_parallel_1_4, stateless_tests_arm_asan_azure_parallel_2_4, stateless_tests_arm_asan_azure_parallel_3_4, stateless_tests_arm_asan_azure_parallel_4_4, stateless_tests_arm_asan_azure_sequential_1_2, stateless_tests_arm_asan_azure_sequential_2_2, stateless_tests_arm_asan_targeted, stateless_tests_arm_binary_parallel, stateless_tests_arm_binary_sequential, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_release, stress_test_amd_tsan, stress_test_amd_ubsan, stress_test_arm_asan, stress_test_arm_asan_s3, unit_tests_asan, unit_tests_msan, unit_tests_tsan, unit_tests_ubsan] if: ${{ always() && needs.config_workflow.outputs.pipeline_status != '' }} name: "Finish Workflow" outputs: @@ -5595,12 +5685,14 @@ jobs: - stateless_tests_arm_asan_azure_parallel_4_4 - stateless_tests_arm_asan_azure_sequential_1_2 - stateless_tests_arm_asan_azure_sequential_2_2 - - integration_tests_amd_asan_db_disk_old_analyzer_1_6 - - integration_tests_amd_asan_db_disk_old_analyzer_2_6 - - integration_tests_amd_asan_db_disk_old_analyzer_3_6 - - integration_tests_amd_asan_db_disk_old_analyzer_4_6 - - integration_tests_amd_asan_db_disk_old_analyzer_5_6 - - integration_tests_amd_asan_db_disk_old_analyzer_6_6 + - integration_tests_amd_asan_db_disk_old_analyzer_1_8 + - integration_tests_amd_asan_db_disk_old_analyzer_2_8 + - integration_tests_amd_asan_db_disk_old_analyzer_3_8 + - integration_tests_amd_asan_db_disk_old_analyzer_4_8 + - integration_tests_amd_asan_db_disk_old_analyzer_5_8 + - integration_tests_amd_asan_db_disk_old_analyzer_6_8 + - integration_tests_amd_asan_db_disk_old_analyzer_7_8 + - integration_tests_amd_asan_db_disk_old_analyzer_8_8 - integration_tests_amd_binary_1_5 - integration_tests_amd_binary_2_5 - integration_tests_amd_binary_3_5 diff --git a/.github/workflows/pull_request_community.yml b/.github/workflows/pull_request_community.yml index 09fbf6e89d003..05a09823fee73 100644 --- a/.github/workflows/pull_request_community.yml +++ b/.github/workflows/pull_request_community.yml @@ -2251,11 +2251,11 @@ jobs: prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_old_analyzer_1_6: + integration_tests_amd_asan_db_disk_old_analyzer_1_8: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] needs: [build_amd_asan, build_amd_debug, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_distributed_plan_parallel_1_4, stateless_tests_amd_asan_distributed_plan_parallel_2_4, stateless_tests_amd_asan_distributed_plan_parallel_3_4, stateless_tests_amd_asan_distributed_plan_parallel_4_4, stateless_tests_amd_debug_parallel, stateless_tests_arm_binary_parallel] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDEvNik=') }} - name: "Integration tests (amd_asan, db disk, old analyzer, 1/6)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDEvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 1/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2270,7 +2270,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Integration tests (amd_asan, db disk, old analyzer, 1/6)" + test_name: "Integration tests (amd_asan, db disk, old analyzer, 1/8)" - name: Prepare env script run: | @@ -2298,15 +2298,15 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 1/6)' --workflow "Community PR" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 1/8)' --workflow "Community PR" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_old_analyzer_2_6: + integration_tests_amd_asan_db_disk_old_analyzer_2_8: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] needs: [build_amd_asan, build_amd_debug, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_distributed_plan_parallel_1_4, stateless_tests_amd_asan_distributed_plan_parallel_2_4, stateless_tests_amd_asan_distributed_plan_parallel_3_4, stateless_tests_amd_asan_distributed_plan_parallel_4_4, stateless_tests_amd_debug_parallel, stateless_tests_arm_binary_parallel] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDIvNik=') }} - name: "Integration tests (amd_asan, db disk, old analyzer, 2/6)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDIvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 2/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2321,7 +2321,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Integration tests (amd_asan, db disk, old analyzer, 2/6)" + test_name: "Integration tests (amd_asan, db disk, old analyzer, 2/8)" - name: Prepare env script run: | @@ -2349,15 +2349,15 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 2/6)' --workflow "Community PR" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 2/8)' --workflow "Community PR" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_old_analyzer_3_6: + integration_tests_amd_asan_db_disk_old_analyzer_3_8: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] needs: [build_amd_asan, build_amd_debug, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_distributed_plan_parallel_1_4, stateless_tests_amd_asan_distributed_plan_parallel_2_4, stateless_tests_amd_asan_distributed_plan_parallel_3_4, stateless_tests_amd_asan_distributed_plan_parallel_4_4, stateless_tests_amd_debug_parallel, stateless_tests_arm_binary_parallel] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDMvNik=') }} - name: "Integration tests (amd_asan, db disk, old analyzer, 3/6)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDMvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 3/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2372,7 +2372,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Integration tests (amd_asan, db disk, old analyzer, 3/6)" + test_name: "Integration tests (amd_asan, db disk, old analyzer, 3/8)" - name: Prepare env script run: | @@ -2400,15 +2400,15 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 3/6)' --workflow "Community PR" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 3/8)' --workflow "Community PR" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_old_analyzer_4_6: + integration_tests_amd_asan_db_disk_old_analyzer_4_8: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] needs: [build_amd_asan, build_amd_debug, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_distributed_plan_parallel_1_4, stateless_tests_amd_asan_distributed_plan_parallel_2_4, stateless_tests_amd_asan_distributed_plan_parallel_3_4, stateless_tests_amd_asan_distributed_plan_parallel_4_4, stateless_tests_amd_debug_parallel, stateless_tests_arm_binary_parallel] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDQvNik=') }} - name: "Integration tests (amd_asan, db disk, old analyzer, 4/6)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDQvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 4/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2423,7 +2423,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Integration tests (amd_asan, db disk, old analyzer, 4/6)" + test_name: "Integration tests (amd_asan, db disk, old analyzer, 4/8)" - name: Prepare env script run: | @@ -2451,15 +2451,15 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 4/6)' --workflow "Community PR" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 4/8)' --workflow "Community PR" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_old_analyzer_5_6: + integration_tests_amd_asan_db_disk_old_analyzer_5_8: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] needs: [build_amd_asan, build_amd_debug, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_distributed_plan_parallel_1_4, stateless_tests_amd_asan_distributed_plan_parallel_2_4, stateless_tests_amd_asan_distributed_plan_parallel_3_4, stateless_tests_amd_asan_distributed_plan_parallel_4_4, stateless_tests_amd_debug_parallel, stateless_tests_arm_binary_parallel] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDUvNik=') }} - name: "Integration tests (amd_asan, db disk, old analyzer, 5/6)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDUvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 5/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2474,7 +2474,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Integration tests (amd_asan, db disk, old analyzer, 5/6)" + test_name: "Integration tests (amd_asan, db disk, old analyzer, 5/8)" - name: Prepare env script run: | @@ -2502,15 +2502,15 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 5/6)' --workflow "Community PR" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 5/8)' --workflow "Community PR" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_old_analyzer_6_6: + integration_tests_amd_asan_db_disk_old_analyzer_6_8: runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] needs: [build_amd_asan, build_amd_debug, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_distributed_plan_parallel_1_4, stateless_tests_amd_asan_distributed_plan_parallel_2_4, stateless_tests_amd_asan_distributed_plan_parallel_3_4, stateless_tests_amd_asan_distributed_plan_parallel_4_4, stateless_tests_amd_debug_parallel, stateless_tests_arm_binary_parallel] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDYvNik=') }} - name: "Integration tests (amd_asan, db disk, old analyzer, 6/6)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDYvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 6/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -2525,7 +2525,7 @@ jobs: - name: Docker setup uses: ./.github/actions/docker_setup with: - test_name: "Integration tests (amd_asan, db disk, old analyzer, 6/6)" + test_name: "Integration tests (amd_asan, db disk, old analyzer, 6/8)" - name: Prepare env script run: | @@ -2553,7 +2553,109 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 6/6)' --workflow "Community PR" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 6/8)' --workflow "Community PR" --ci 2>&1 | python3 -u -c 'import sys,datetime + prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") + for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log + + integration_tests_amd_asan_db_disk_old_analyzer_7_8: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] + needs: [build_amd_asan, build_amd_debug, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_distributed_plan_parallel_1_4, stateless_tests_amd_asan_distributed_plan_parallel_2_4, stateless_tests_amd_asan_distributed_plan_parallel_3_4, stateless_tests_amd_asan_distributed_plan_parallel_4_4, stateless_tests_amd_debug_parallel, stateless_tests_arm_binary_parallel] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDcvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 7/8)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Integration tests (amd_asan, db disk, old analyzer, 7/8)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_AMD_ASAN + uses: actions/download-artifact@v8 + with: + name: CH_AMD_ASAN + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + set -o pipefail + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 7/8)' --workflow "Community PR" --ci 2>&1 | python3 -u -c 'import sys,datetime + prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") + for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log + + integration_tests_amd_asan_db_disk_old_analyzer_8_8: + runs-on: [self-hosted, altinity-on-demand, altinity-func-tester, 16c] + needs: [build_amd_asan, build_amd_debug, build_arm_binary, config_workflow, fast_test, stateless_tests_amd_asan_distributed_plan_parallel_1_4, stateless_tests_amd_asan_distributed_plan_parallel_2_4, stateless_tests_amd_asan_distributed_plan_parallel_3_4, stateless_tests_amd_asan_distributed_plan_parallel_4_4, stateless_tests_amd_debug_parallel, stateless_tests_arm_binary_parallel] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDgvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 8/8)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Setup + uses: ./.github/actions/runner_setup + - name: Docker setup + uses: ./.github/actions/docker_setup + with: + test_name: "Integration tests (amd_asan, db disk, old analyzer, 8/8)" + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Download artifact CH_AMD_ASAN + uses: actions/download-artifact@v8 + with: + name: CH_AMD_ASAN + path: ./ci/tmp + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + set -o pipefail + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 8/8)' --workflow "Community PR" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 272c246b68633..6ecf8497e2c5a 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -775,11 +775,11 @@ jobs: prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_1_4: + integration_tests_amd_asan_db_disk_1_8: runs-on: [self-hosted, amd-medium] needs: [build_amd_asan, config_workflow, dockers_build_amd, dockers_build_arm] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCAxLzQp') }} - name: "Integration tests (amd_asan, db disk, 1/4)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCAxLzgp') }} + name: "Integration tests (amd_asan, db disk, 1/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -809,15 +809,15 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, 1/4)' --workflow "ReleaseBranchCI" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, 1/8)' --workflow "ReleaseBranchCI" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_2_4: + integration_tests_amd_asan_db_disk_2_8: runs-on: [self-hosted, amd-medium] needs: [build_amd_asan, config_workflow, dockers_build_amd, dockers_build_arm] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCAyLzQp') }} - name: "Integration tests (amd_asan, db disk, 2/4)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCAyLzgp') }} + name: "Integration tests (amd_asan, db disk, 2/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -847,15 +847,15 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, 2/4)' --workflow "ReleaseBranchCI" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, 2/8)' --workflow "ReleaseBranchCI" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_3_4: + integration_tests_amd_asan_db_disk_3_8: runs-on: [self-hosted, amd-medium] needs: [build_amd_asan, config_workflow, dockers_build_amd, dockers_build_arm] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCAzLzQp') }} - name: "Integration tests (amd_asan, db disk, 3/4)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCAzLzgp') }} + name: "Integration tests (amd_asan, db disk, 3/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -885,15 +885,15 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, 3/4)' --workflow "ReleaseBranchCI" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, 3/8)' --workflow "ReleaseBranchCI" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_4_4: + integration_tests_amd_asan_db_disk_4_8: runs-on: [self-hosted, amd-medium] needs: [build_amd_asan, config_workflow, dockers_build_amd, dockers_build_arm] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCA0LzQp') }} - name: "Integration tests (amd_asan, db disk, 4/4)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCA0Lzgp') }} + name: "Integration tests (amd_asan, db disk, 4/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -923,15 +923,15 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, 4/4)' --workflow "ReleaseBranchCI" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, 4/8)' --workflow "ReleaseBranchCI" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_old_analyzer_1_6: + integration_tests_amd_asan_db_disk_5_8: runs-on: [self-hosted, amd-medium] needs: [build_amd_asan, config_workflow, dockers_build_amd, dockers_build_arm] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDEvNik=') }} - name: "Integration tests (amd_asan, db disk, old analyzer, 1/6)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCA1Lzgp') }} + name: "Integration tests (amd_asan, db disk, 5/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -961,15 +961,15 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 1/6)' --workflow "ReleaseBranchCI" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, 5/8)' --workflow "ReleaseBranchCI" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_old_analyzer_2_6: + integration_tests_amd_asan_db_disk_6_8: runs-on: [self-hosted, amd-medium] needs: [build_amd_asan, config_workflow, dockers_build_amd, dockers_build_arm] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDIvNik=') }} - name: "Integration tests (amd_asan, db disk, old analyzer, 2/6)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCA2Lzgp') }} + name: "Integration tests (amd_asan, db disk, 6/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -999,15 +999,15 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 2/6)' --workflow "ReleaseBranchCI" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, 6/8)' --workflow "ReleaseBranchCI" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_old_analyzer_3_6: + integration_tests_amd_asan_db_disk_7_8: runs-on: [self-hosted, amd-medium] needs: [build_amd_asan, config_workflow, dockers_build_amd, dockers_build_arm] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDMvNik=') }} - name: "Integration tests (amd_asan, db disk, old analyzer, 3/6)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCA3Lzgp') }} + name: "Integration tests (amd_asan, db disk, 7/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1037,15 +1037,15 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 3/6)' --workflow "ReleaseBranchCI" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, 7/8)' --workflow "ReleaseBranchCI" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_old_analyzer_4_6: + integration_tests_amd_asan_db_disk_8_8: runs-on: [self-hosted, amd-medium] needs: [build_amd_asan, config_workflow, dockers_build_amd, dockers_build_arm] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDQvNik=') }} - name: "Integration tests (amd_asan, db disk, old analyzer, 4/6)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCA4Lzgp') }} + name: "Integration tests (amd_asan, db disk, 8/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1075,15 +1075,15 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 4/6)' --workflow "ReleaseBranchCI" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, 8/8)' --workflow "ReleaseBranchCI" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_old_analyzer_5_6: + integration_tests_amd_asan_db_disk_old_analyzer_1_8: runs-on: [self-hosted, amd-medium] needs: [build_amd_asan, config_workflow, dockers_build_amd, dockers_build_arm] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDUvNik=') }} - name: "Integration tests (amd_asan, db disk, old analyzer, 5/6)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDEvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 1/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1113,15 +1113,15 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 5/6)' --workflow "ReleaseBranchCI" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 1/8)' --workflow "ReleaseBranchCI" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log - integration_tests_amd_asan_db_disk_old_analyzer_6_6: + integration_tests_amd_asan_db_disk_old_analyzer_2_8: runs-on: [self-hosted, amd-medium] needs: [build_amd_asan, config_workflow, dockers_build_amd, dockers_build_arm] - if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDYvNik=') }} - name: "Integration tests (amd_asan, db disk, old analyzer, 6/6)" + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDIvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 2/8)" outputs: data: ${{ steps.run.outputs.DATA }} pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} @@ -1151,7 +1151,235 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh set -o pipefail - PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 6/6)' --workflow "ReleaseBranchCI" --ci 2>&1 | python3 -u -c 'import sys,datetime + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 2/8)' --workflow "ReleaseBranchCI" --ci 2>&1 | python3 -u -c 'import sys,datetime + prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") + for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log + + integration_tests_amd_asan_db_disk_old_analyzer_3_8: + runs-on: [self-hosted, amd-medium] + needs: [build_amd_asan, config_workflow, dockers_build_amd, dockers_build_arm] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDMvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 3/8)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + set -o pipefail + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 3/8)' --workflow "ReleaseBranchCI" --ci 2>&1 | python3 -u -c 'import sys,datetime + prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") + for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log + + integration_tests_amd_asan_db_disk_old_analyzer_4_8: + runs-on: [self-hosted, amd-medium] + needs: [build_amd_asan, config_workflow, dockers_build_amd, dockers_build_arm] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDQvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 4/8)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + set -o pipefail + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 4/8)' --workflow "ReleaseBranchCI" --ci 2>&1 | python3 -u -c 'import sys,datetime + prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") + for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log + + integration_tests_amd_asan_db_disk_old_analyzer_5_8: + runs-on: [self-hosted, amd-medium] + needs: [build_amd_asan, config_workflow, dockers_build_amd, dockers_build_arm] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDUvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 5/8)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + set -o pipefail + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 5/8)' --workflow "ReleaseBranchCI" --ci 2>&1 | python3 -u -c 'import sys,datetime + prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") + for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log + + integration_tests_amd_asan_db_disk_old_analyzer_6_8: + runs-on: [self-hosted, amd-medium] + needs: [build_amd_asan, config_workflow, dockers_build_amd, dockers_build_arm] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDYvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 6/8)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + set -o pipefail + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 6/8)' --workflow "ReleaseBranchCI" --ci 2>&1 | python3 -u -c 'import sys,datetime + prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") + for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log + + integration_tests_amd_asan_db_disk_old_analyzer_7_8: + runs-on: [self-hosted, amd-medium] + needs: [build_amd_asan, config_workflow, dockers_build_amd, dockers_build_arm] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDcvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 7/8)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + set -o pipefail + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 7/8)' --workflow "ReleaseBranchCI" --ci 2>&1 | python3 -u -c 'import sys,datetime + prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") + for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log + + integration_tests_amd_asan_db_disk_old_analyzer_8_8: + runs-on: [self-hosted, amd-medium] + needs: [build_amd_asan, config_workflow, dockers_build_amd, dockers_build_arm] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'SW50ZWdyYXRpb24gdGVzdHMgKGFtZF9hc2FuLCBkYiBkaXNrLCBvbGQgYW5hbHl6ZXIsIDgvOCk=') }} + name: "Integration tests (amd_asan, db disk, old analyzer, 8/8)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + set -o pipefail + PYTHONUNBUFFERED=1 python3 -m praktika run 'Integration tests (amd_asan, db disk, old analyzer, 8/8)' --workflow "ReleaseBranchCI" --ci 2>&1 | python3 -u -c 'import sys,datetime prefix=lambda: datetime.datetime.now().strftime("[%Y-%m-%d %H:%M:%S]") for line in sys.stdin: sys.stdout.write(prefix() + " " + line); sys.stdout.flush()' | tee ./ci/tmp/job.log @@ -1651,7 +1879,7 @@ jobs: finish_workflow: runs-on: [self-hosted, style-checker-aarch64] - needs: [build_amd_asan, build_amd_darwin, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_amd_ubsan, build_arm_asan, build_arm_darwin, build_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_db_disk_1_4, integration_tests_amd_asan_db_disk_2_4, integration_tests_amd_asan_db_disk_3_4, integration_tests_amd_asan_db_disk_4_4, integration_tests_amd_asan_db_disk_old_analyzer_1_6, integration_tests_amd_asan_db_disk_old_analyzer_2_6, integration_tests_amd_asan_db_disk_old_analyzer_3_6, integration_tests_amd_asan_db_disk_old_analyzer_4_6, integration_tests_amd_asan_db_disk_old_analyzer_5_6, integration_tests_amd_asan_db_disk_old_analyzer_6_6, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, stateless_tests_amd_asan_db_disk_distributed_plan_sequential, stateless_tests_amd_asan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_distributed_plan_parallel_2_2, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_release, stress_test_amd_tsan, stress_test_amd_ubsan, stress_test_arm_asan, stress_test_arm_asan_s3] + needs: [build_amd_asan, build_amd_darwin, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_amd_ubsan, build_arm_asan, build_arm_darwin, build_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_db_disk_1_8, integration_tests_amd_asan_db_disk_2_8, integration_tests_amd_asan_db_disk_3_8, integration_tests_amd_asan_db_disk_4_8, integration_tests_amd_asan_db_disk_5_8, integration_tests_amd_asan_db_disk_6_8, integration_tests_amd_asan_db_disk_7_8, integration_tests_amd_asan_db_disk_8_8, integration_tests_amd_asan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_db_disk_old_analyzer_8_8, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, stateless_tests_amd_asan_db_disk_distributed_plan_sequential, stateless_tests_amd_asan_distributed_plan_parallel_1_2, stateless_tests_amd_asan_distributed_plan_parallel_2_2, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_release, stress_test_amd_tsan, stress_test_amd_ubsan, stress_test_arm_asan, stress_test_arm_asan_s3] if: ${{ always() }} name: "Finish Workflow" outputs: diff --git a/.gitmodules b/.gitmodules index 52c8eac2b300c..4959a04c45dc2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -214,7 +214,7 @@ url = https://github.com/ClickHouse/nats.c [submodule "contrib/vectorscan"] path = contrib/vectorscan - url = https://github.com/ClickHouse/vectorscan.git + url = https://github.com/ClickHouse/vectorscan [submodule "contrib/llvm-project"] path = contrib/llvm-project url = https://github.com/ClickHouse/llvm-project @@ -335,10 +335,10 @@ url = https://github.com/ClickHouse/double-conversion.git [submodule "contrib/mongo-cxx-driver"] path = contrib/mongo-cxx-driver - url = https://github.com/mongodb/mongo-cxx-driver.git + url = https://github.com/ClickHouse/mongo-cxx-driver.git [submodule "contrib/mongo-c-driver"] path = contrib/mongo-c-driver - url = https://github.com/ClickHouse/mongo-c-driver.git + url = https://github.com/ClickHouse/mongo-c-driver [submodule "contrib/numactl"] path = contrib/numactl url = https://github.com/ClickHouse/numactl.git diff --git a/base/poco/Foundation/include/Poco/ListMap.h b/base/poco/Foundation/include/Poco/ListMap.h index f276c58dd83ed..2aa9f330fa84c 100644 --- a/base/poco/Foundation/include/Poco/ListMap.h +++ b/base/poco/Foundation/include/Poco/ListMap.h @@ -139,10 +139,21 @@ class ListMap /// sequentially at all times. /// Returns iterator pointing to the newly inserted value { - Iterator it = find(val.first); - while (it != _list.end() && isEqual(it->first, val.first)) - ++it; - return _list.insert(it, val); + /// All entries with the same key sit next to each other, so the new + /// value belongs right after the last entry with this key. Look for + /// that entry starting from the back of the list. This makes adding + /// many values with the same key (e.g. a repeated HTTP query parameter) + /// cheap, because the match is usually the last element. If the key + /// is not in the map we still walk the whole list and append at the + /// end, same as before. + typename Container::reverse_iterator it = _list.rbegin(); + typename Container::reverse_iterator itEnd = _list.rend(); + for (; it != itEnd; ++it) + { + if (isEqual(it->first, val.first)) + return _list.insert(it.base(), val); + } + return _list.insert(_list.end(), val); } void erase(Iterator it) { _list.erase(it); } diff --git a/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureSocketImpl.h b/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureSocketImpl.h index 0d10487ffbbff..d90170053063d 100644 --- a/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureSocketImpl.h +++ b/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureSocketImpl.h @@ -141,6 +141,13 @@ namespace Net poco_socket_t sockfd(); /// Returns the underlying socket descriptor. + SSL * ssl() const; + /// Returns the underlying OpenSSL SSL object, or null if the SSL handshake + /// has not been performed yet. + /// + /// The object is normally guarded by the socket's mutex; a caller that uses it + /// directly must ensure the socket is not accessed concurrently. + X509 * peerCertificate() const; /// Returns the peer's certificate. @@ -209,7 +216,7 @@ namespace Net /// Returns true iff the given host name is the local host /// (either "localhost" or "127.0.0.1"). - bool mustRetry(int rc, Poco::Timespan & remaining_time); + bool mustRetry(int rc, int sslError, int socketError, Poco::Timespan & remaining_time); /// Returns true if the last operation should be retried, /// otherwise false. /// @@ -223,7 +230,7 @@ namespace Net /// not become readable or writable within the sockets /// receive or send timeout. - int handleError(int rc); + int handleError(int rc, int sslError, int socketError, unsigned long errorCode); /// Handles an SSL error by throwing an appropriate exception. void reset(); @@ -244,6 +251,10 @@ namespace Net Poco::AutoPtr _pSocket; Context::Ptr _pContext; bool _needHandshake; + bool _fatalError; + bool _pendingWrite = false; + /// Whether the last `SSL_write` returned `SSL_ERROR_WANT_WRITE`. OpenSSL keeps that record pending + /// until `SSL_write` is retried, even after other operations change what `SSL_get_error` reports. std::string _peerHostName; Session::Ptr _pSession; @@ -263,6 +274,12 @@ namespace Net } + inline SSL * SecureSocketImpl::ssl() const + { + return _pSSL; + } + + inline Context::Ptr SecureSocketImpl::context() const { return _pContext; diff --git a/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureStreamSocketImpl.h b/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureStreamSocketImpl.h index e23a17ae3146f..026d73f27cf37 100644 --- a/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureStreamSocketImpl.h +++ b/base/poco/NetSSL_OpenSSL/include/Poco/Net/SecureStreamSocketImpl.h @@ -179,6 +179,10 @@ namespace Net Context::Ptr context() const; /// Returns the SSL context used by this socket. + SSL * ssl() const; + /// Returns the underlying OpenSSL SSL object, or null if the SSL handshake + /// has not been performed yet. + void setLazyHandshake(bool flag = true); /// Enable lazy SSL handshake. If enabled, the SSL handshake /// will be performed the first time date is sent or @@ -283,6 +287,12 @@ namespace Net } + inline SSL * SecureStreamSocketImpl::ssl() const + { + return _impl.ssl(); + } + + inline Session::Ptr SecureStreamSocketImpl::currentSession() { return _impl.currentSession(); diff --git a/base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp b/base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp index 3a8a9025dcd5c..6ae909f5349e0 100644 --- a/base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp +++ b/base/poco/NetSSL_OpenSSL/src/SecureSocketImpl.cpp @@ -25,6 +25,7 @@ #include "Poco/NumberFormatter.h" #include "Poco/NumberParser.h" #include "Poco/Format.h" +#include #include #include @@ -60,11 +61,52 @@ struct RemainingTimeCounter Poco::Timestamp start; }; +struct SSLOperationResult +{ + int rc = 0; + int sslError = SSL_ERROR_NONE; + int socketError = 0; + unsigned long errorCode = 0; +}; + + +template +SSLOperationResult performSSLOperation(SSL * ssl, Operation && operation, bool zeroIsError = true) +{ + /// The error queue contract for TLS I/O and `SSL_get_error`: + /// https://docs.openssl.org/3.5/man3/SSL_get_error/ + /// The queue is cleared with `ERR_clear_error`: + /// https://docs.openssl.org/3.5/man3/ERR_clear_error/ + ERR_clear_error(); + + SSLOperationResult result; + /// `errno` is only meaningful if it was set by this operation. In particular, a custom `BIO` + /// can report `SSL_ERROR_SYSCALL` without changing it, so do not inherit another syscall result. + errno = 0; + result.rc = operation(); + + if (result.rc < 0 || (zeroIsError && result.rc == 0)) + { + /// Save `errno` before calling another function. `SSL_get_error` must be the first + /// OpenSSL call after the operation and must observe the queue created by that operation. + result.socketError = errno; + result.sslError = SSL_get_error(ssl, result.rc); + result.errorCode = ERR_get_error(); + } + + /// `errorCode` above preserves the diagnostic used by `handleError`. Do not leave any + /// additional entries in the thread-local queue for another connection on this thread. + ERR_clear_error(); + return result; +} + + SecureSocketImpl::SecureSocketImpl(Poco::AutoPtr pSocketImpl, Context::Ptr pContext): _pSSL(nullptr), _pSocket(pSocketImpl), _pContext(pContext), - _needHandshake(false) + _needHandshake(false), + _fatalError(false) { poco_check_ptr (_pSocket); poco_check_ptr (_pContext); @@ -102,6 +144,8 @@ void SecureSocketImpl::acceptSSL() { std::lock_guard lock(_mutex); poco_assert (!_pSSL); + _fatalError = false; + _pendingWrite = false; BIO* pBIO = BIO_new(BIO_s_socket()); if (!pBIO) throw SSLException("Cannot create BIO object"); @@ -168,6 +212,8 @@ void SecureSocketImpl::connectSSL(bool performHandshake) std::lock_guard lock(_mutex); poco_assert (!_pSSL); poco_assert (_pSocket->initialized()); + _fatalError = false; + _pendingWrite = false; BIO* pBIO = BIO_new(BIO_s_socket()); if (!pBIO) throw SSLException("Cannot create SSL BIO object"); @@ -193,31 +239,39 @@ void SecureSocketImpl::connectSSL(bool performHandshake) SSL_set_session(_pSSL, _pSession->sslSession()); } + SSL_set_connect_state(_pSSL); + _needHandshake = true; + try { if (performHandshake && _pSocket->getBlocking()) { - int ret; + SSLOperationResult result; Poco::Timespan remaining_time = getMaxTimeoutOrLimit(); do { RemainingTimeCounter counter(remaining_time); - ret = SSL_connect(_pSSL); + result = performSSLOperation(_pSSL, [this] + { + return SSL_connect(_pSSL); + }); + } + while (mustRetry(result.rc, result.sslError, result.socketError, remaining_time)); + if (result.rc <= 0) + { + if (handleError(result.rc, result.sslError, result.socketError, result.errorCode) < 0) + throw Poco::TimeoutException("SSL handshake timed out"); + throw SSLConnectionUnexpectedlyClosedException(); } - while (mustRetry(ret, remaining_time)); - handleError(ret); + _needHandshake = false; verifyPeerCertificate(); } - else - { - SSL_set_connect_state(_pSSL); - _needHandshake = true; - } } catch (...) { SSL_free(_pSSL); _pSSL = 0; + _fatalError = false; throw; } } @@ -246,6 +300,16 @@ void SecureSocketImpl::shutdown() std::lock_guard lock(_mutex); if (_pSSL) { + if (_fatalError) + { + /// OpenSSL forbids `SSL_shutdown` after a fatal TLS or syscall error. + /// https://docs.openssl.org/3.5/man3/SSL_shutdown/ + /// Close the underlying transport without attempting an orderly TLS shutdown. + if (_pSocket->getBlocking()) + _pSocket->shutdown(); + return; + } + // Don't shut down the socket more than once. int shutdownState = SSL_get_shutdown(_pSSL); bool shutdownSent = (shutdownState & SSL_SENT_SHUTDOWN) == SSL_SENT_SHUTDOWN; @@ -258,8 +322,24 @@ void SecureSocketImpl::shutdown() // most web browsers, so we just set the shutdown // flag by calling SSL_shutdown() once and be // done with it. - int rc = SSL_shutdown(_pSSL); - if (rc < 0) handleError(rc); + /// A zero result is not an error for `SSL_shutdown` and must not be passed to + /// `SSL_get_error`; it means that `close_notify` was sent but not received yet. + SSLOperationResult result; + Poco::Timespan remaining_time = getMaxTimeoutOrLimit(); + do + { + RemainingTimeCounter counter(remaining_time); + result = performSSLOperation(_pSSL, [this] + { + return SSL_shutdown(_pSSL); + }, false); + } + /// OpenSSL does not dispatch the `close_notify` alert while a record write from an + /// earlier `SSL_write` is still pending, so retrying cannot make progress then. + while (!_pendingWrite && result.rc < 0 + && mustRetry(result.rc, result.sslError, result.socketError, remaining_time)); + if (result.rc < 0) + handleError(result.rc, result.sslError, result.socketError, result.errorCode); if (_pSocket->getBlocking()) { _pSocket->shutdown(); @@ -303,24 +383,22 @@ int SecureSocketImpl::sendBytes(const void* buffer, int length, int flags) return rc; } + SSLOperationResult result; Poco::Timespan remaining_time = getMaxTimeoutOrLimit(); do { RemainingTimeCounter counter(remaining_time); - rc = SSL_write(_pSSL, buffer, length); + result = performSSLOperation(_pSSL, [this, buffer, length] + { + return SSL_write(_pSSL, buffer, length); + }); + _pendingWrite = result.sslError == SSL_ERROR_WANT_WRITE; } - while (mustRetry(rc, remaining_time)); + while (mustRetry(result.rc, result.sslError, result.socketError, remaining_time)); + rc = result.rc; if (rc <= 0) { - // At this stage we still can have last not yet received SSL message containing SSL error - // so make a read to force SSL to process possible SSL error - if (SSL_get_error(_pSSL, rc) == SSL_ERROR_SYSCALL && SocketImpl::lastError() == POCO_ECONNRESET) - { - char c = 0; - SSL_read(_pSSL, &c, 1); - } - - rc = handleError(rc); + rc = handleError(rc, result.sslError, result.socketError, result.errorCode); if (rc == 0) throw SSLConnectionUnexpectedlyClosedException(); if (rc < 0 && _pSocket->getBlocking()) throw Poco::TimeoutException("SSL_write timed out"); @@ -354,6 +432,7 @@ int SecureSocketImpl::receiveBytes(void* buffer, int length, int flags) return rc; } + SSLOperationResult result; Poco::Timespan remaining_time = getMaxTimeoutOrLimit(); do { @@ -361,12 +440,16 @@ int SecureSocketImpl::receiveBytes(void* buffer, int length, int flags) /// so thread can be blocked on recv/send and epoll_wait several times /// until SSL_read will return rc > 0. Let's use our own time counter. RemainingTimeCounter counter(remaining_time); - rc = SSL_read(_pSSL, buffer, length); + result = performSSLOperation(_pSSL, [this, buffer, length] + { + return SSL_read(_pSSL, buffer, length); + }); } - while (mustRetry(rc, remaining_time)); + while (mustRetry(result.rc, result.sslError, result.socketError, remaining_time)); + rc = result.rc; if (rc <= 0) { - rc = handleError(rc); + rc = handleError(rc, result.sslError, result.socketError, result.errorCode); if (rc < 0 && _pSocket->getBlocking()) throw Poco::TimeoutException("SSL_read timed out"); return rc; @@ -394,16 +477,21 @@ int SecureSocketImpl::completeHandshake() poco_check_ptr (_pSSL); int rc; + SSLOperationResult result; Poco::Timespan remaining_time = getMaxTimeoutOrLimit(); do { RemainingTimeCounter counter(remaining_time); - rc = SSL_do_handshake(_pSSL); + result = performSSLOperation(_pSSL, [this] + { + return SSL_do_handshake(_pSSL); + }); } - while (mustRetry(rc, remaining_time)); + while (mustRetry(result.rc, result.sslError, result.socketError, remaining_time)); + rc = result.rc; if (rc <= 0) { - rc = handleError(rc); + rc = handleError(rc, result.sslError, result.socketError, result.errorCode); if (rc < 0 && _pSocket->getBlocking()) throw Poco::TimeoutException("SSL handshake timed out"); return rc; @@ -511,15 +599,13 @@ Poco::Timespan SecureSocketImpl::getMaxTimeoutOrLimit() return remaining_time; } -bool SecureSocketImpl::mustRetry(int rc, Poco::Timespan& remaining_time) +bool SecureSocketImpl::mustRetry(int rc, int sslError, int socketError, Poco::Timespan& remaining_time) { if (remaining_time == 0) return false; std::lock_guard lock(_mutex); if (rc <= 0) { - int sslError = SSL_get_error(_pSSL, rc); - int socketError = _pSocket->lastError(); switch (sslError) { case SSL_ERROR_WANT_READ: @@ -546,21 +632,20 @@ bool SecureSocketImpl::mustRetry(int rc, Poco::Timespan& remaining_time) case SSL_ERROR_SYSCALL: return socketError == POCO_EAGAIN || socketError == POCO_EINTR; default: - return socketError == POCO_EINTR; + /// `errno` is only meaningful for `SSL_ERROR_SYSCALL`; other errors must not + /// be retried because of a leftover `EINTR`, especially `SSL_ERROR_SSL`. + return false; } } return false; } -int SecureSocketImpl::handleError(int rc) +int SecureSocketImpl::handleError(int rc, int sslError, int error, unsigned long errorCode) { std::lock_guard lock(_mutex); if (rc > 0) return rc; - int sslError = SSL_get_error(_pSSL, rc); - int error = SocketImpl::lastError(); - switch (sslError) { case SSL_ERROR_ZERO_RETURN: @@ -573,18 +658,21 @@ int SecureSocketImpl::handleError(int rc) case SSL_ERROR_WANT_ACCEPT: case SSL_ERROR_WANT_X509_LOOKUP: // these should not occur + _fatalError = true; poco_bugcheck(); return rc; case SSL_ERROR_SYSCALL: + _fatalError = true; if (error != 0) { SocketImpl::error(error); } - // fallthrough + [[fallthrough]]; + case SSL_ERROR_SSL: default: { - long lastError = ERR_get_error(); - if (lastError == 0) + _fatalError = true; + if (errorCode == 0) { if (rc == 0) { @@ -606,7 +694,7 @@ int SecureSocketImpl::handleError(int rc) else { char buffer[256]; - ERR_error_string_n(lastError, buffer, sizeof(buffer)); + ERR_error_string_n(errorCode, buffer, sizeof(buffer)); std::string msg(buffer); throw SSLException(msg); } @@ -633,6 +721,7 @@ void SecureSocketImpl::reset() SSL_free(_pSSL); _pSSL = nullptr; } + _fatalError = false; } diff --git a/ci/defs/defs.py b/ci/defs/defs.py index 95fb1a18e1cf4..ec924258783f5 100644 --- a/ci/defs/defs.py +++ b/ci/defs/defs.py @@ -468,6 +468,26 @@ class ArtifactNames: LLVM_FT_NUM_BATCHES = 3 LLVM_IT_NUM_BATCHES = 5 +# Batch count of the two ASan integration-test flavors, which run the whole integration suite +# on `AMD_MEDIUM` with three xdist workers against a two-hour pytest `--session-timeout`. +# +# Eight, not six: the suite measures about 112000 test-seconds, so six batches put the heaviest +# shard at 119.7 of the 120 available minutes - no margin at all, however well they are +# balanced - and `amd_asan_ubsan, db disk, old analyzer` timed out 133 times in the three weeks +# after it went to six. Eight brings the heaviest shard to 82 minutes, 68% of the budget. +# +# Batches are the right lever rather than a longer timeout: the aggregate compute is unchanged, +# since the work is the same and only the parallelism differs, so the only cost is one more +# per-batch setup - and the job gets its answer 25% sooner. +# +# Both flavors must keep the same count: they run the same suite with the same worker count +# against the same budget, so a count that does not fit one does not fit the other. +# +# To re-measure, score the packing of `get_optimal_test_batch` against per-module +# `sum(test_duration_ms)` from CIDB rather than against its own `TEST_DURATIONS` table - that +# table currently sums to 97000 test-seconds against a measured 112000, so it reports every +# shard as evenly packed while the measured spread across six is 1.24. +ASAN_IT_NUM_BATCHES = 8 LLVM_FT_ARTIFACTS_LIST = [ # default.profdata files for 3 batches from Stateless(Functional) tests ArtifactNames.LLVM_COVERAGE_FILE + f"_ft_{batch}" diff --git a/ci/defs/job_configs.py b/ci/defs/job_configs.py index 823dde9349c23..998d2f76da6df 100644 --- a/ci/defs/job_configs.py +++ b/ci/defs/job_configs.py @@ -2,6 +2,7 @@ from praktika.utils import Utils from ci.defs.defs import ( + ASAN_IT_NUM_BATCHES, LLVM_ARTIFACTS_LIST, LLVM_FT_NUM_BATCHES, LLVM_IT_NUM_BATCHES, @@ -847,7 +848,8 @@ class JobConfigs: requires=[ArtifactNames.DEB_AMD_RELEASE], ), ) - # why it's master only? + # Despite the name, only release_branches.py uses these. + # `ASAN_IT_NUM_BATCHES` explains the batch count; keep it in step with the flavor below. integration_test_asan_master_jobs = common_integration_test_job_config.parametrize( *[ Job.ParamSet( @@ -855,7 +857,7 @@ class JobConfigs: runs_on=RunnerLabels.AMD_MEDIUM, requires=[ArtifactNames.CH_AMD_ASAN], ) - for total_batches in (4,) + for total_batches in (ASAN_IT_NUM_BATCHES,) for batch in range(1, total_batches + 1) ] ) @@ -866,7 +868,7 @@ class JobConfigs: runs_on=RunnerLabels.AMD_MEDIUM, requires=[ArtifactNames.CH_AMD_ASAN], ) - for total_batches in (6,) + for total_batches in (ASAN_IT_NUM_BATCHES,) for batch in range(1, total_batches + 1) ], *[ diff --git a/ci/jobs/check_style.py b/ci/jobs/check_style.py index 26bb13f6f7621..af24e4a4ae8d6 100644 --- a/ci/jobs/check_style.py +++ b/ci/jobs/check_style.py @@ -2,6 +2,7 @@ import math import multiprocessing import os +import pathlib import re from concurrent.futures import ProcessPoolExecutor from pathlib import Path @@ -488,6 +489,125 @@ def check_catch_all(files) -> str: return "\n".join(violations) +# Storage classes whose tables can be deferred behind `StorageTableProxy`, which means a pointer +# taken from `DatabaseCatalog` may be the proxy rather than the engine. +DEFERRABLE_STORAGE_CLASSES = ( + "MergeTreeData", + "StorageMergeTree", + "StorageReplicatedMergeTree", + "StorageSharedMergeTree", + "StorageSetOrJoinBase", + "StorageSet", + "StorageJoin", + "StorageSharedSet", + "StorageSharedJoin", + "StorageEmbeddedRocksDB", + "IKeyValueEntity", + "IStorageURLBase", + "IBackgroundOperation", + "StorageWithCommonVirtualColumns", + "StorageLog", + "StorageStripeLog", + "StorageURL", + "StorageObjectStorage", + "StorageKeeperMap", + "StorageMySQL", + "StoragePostgreSQL", + "StorageMongoDB", + "StorageRedis", + "StorageSQLite", + "StorageXDBC", + "StorageHive", + "StorageArrowFlight", + "StorageYTsaurus", + "StorageBigQuery", + "StorageKafka", + "StorageKafka2", + "StorageFileLog", + "StorageRabbitMQ", + "StorageNATS", + "StorageObjectStorageQueue", + "IStreamingStorage", +) + +# Casts on an operand that cannot be a catalog pointer, so no proxy can be in the way. +_NOT_A_CATALOG_POINTER = re.compile( + r"^(?:\*?this\b" + r"|shared_from_this\(\)" + r"|&?\w*(?:snapshot|storage_snapshot)->storage\b" + r"|&?\w*reading->getMergeTreeData\(\)" + r")" +) + + +def _without_comments(text): + """Blanks out comments, keeping the line layout so offsets and line numbers still match.""" + return re.sub( + r"//[^\n]*|/\*.*?\*/", lambda m: re.sub(r"[^\n]", " ", m.group(0)), text, flags=re.S + ) + + +def _cast_operand(text, open_paren): + """The argument of a cast whose '(' is at `open_paren`, or None when unbalanced.""" + depth = 0 + for i in range(open_paren, min(open_paren + 2000, len(text))): + depth += (text[i] == "(") - (text[i] == ")") + if not depth: + return " ".join(text[open_paren + 1 : i].split()) + return None + + +def check_storage_casts(files) -> str: + """Require `castStorage` for casts to an engine that supports deferred loading. + + Such a table lives behind `StorageTableProxy` until its first access and the catalog keeps + handing out that proxy afterwards, so a direct cast fails for the whole life of the table. + """ + types = "|".join(DEFERRABLE_STORAGE_CLASSES) + cast_head = re.compile( + r"\b(?Pdynamic_cast|typeid_cast|dynamic_pointer_cast|static_pointer_cast)\s*<\s*" + r"(?:const\s+)?(?:" + types + r")\s*[*&]?\s*>\s*\(" + ) + # `IStorage::as()` is a `typeid_cast` on the receiver, so the operand is what precedes it. + as_cast = re.compile( + r"(?P[\w.\[\]()]+)(?:->|\.)as\s*<\s*(?:const\s+)?(?:" + types + r")\s*>\s*\(\s*\)" + ) + resolvers = ("castStorage", "resolveStorageProxy", "resolveStorageProxyLoading") + + violations = [] + for path in files: + # The helpers and the proxy itself have to reach the nested storage directly. + if not path.endswith((".cpp", ".h")) or path.endswith( + ("StorageProxy.h", "StorageTableProxy.h", "StorageTableFunction.h") + ): + continue + try: + lines = pathlib.Path(path).read_text(errors="replace").splitlines() + except OSError: + continue + + text = _without_comments("\n".join(lines)) + casts = [(m, m.group("cast"), _cast_operand(text, m.end() - 1)) for m in cast_head.finditer(text)] + casts += [(m, "as", m.group("operand")) for m in as_cast.finditer(text)] + + for match, cast, operand in casts: + if not operand or _NOT_A_CATALOG_POINTER.match(operand) or any(r in operand for r in resolvers): + continue + line = text.count("\n", 0, match.start()) + 1 + # The marker goes on the cast or, when the line is long, the one above it. + if any("NOLINT(storage-cast)" in lines[i] for i in (line - 1, line - 2) if i >= 0): + continue + violations.append( + f"{path}:{line}: {cast} to a deferrable storage engine on `{operand[:60]}`. Such a table " + "is reached through StorageTableProxy, so this cast fails for the whole life of the table " + "and whatever it guards is silently skipped. Use castStorage(ptr, " + "DeferredTable::Load) when the query names this table, or DeferredTable::Skip " + "when this walks every table and must not load one. If the pointer cannot come from " + "DatabaseCatalog, say why in a `/// NOLINT(storage-cast)` comment." + ) + return "\n".join(violations) + + def check_file_names(files): files_set = set() for file in files: @@ -609,6 +729,15 @@ def parse_args(): files=cpp_files, ) ) + testname = "storage_casts" + if testpattern.lower() in testname.lower(): + results.append( + run_check_concurrent( + check_name=testname, + check_function=check_storage_casts, + files=cpp_files, + ) + ) testname = "cpp" if testpattern.lower() in testname.lower(): results.append( diff --git a/ci/jobs/docker_server.py b/ci/jobs/docker_server.py index 1eac701797183..ed1a622feaff4 100644 --- a/ci/jobs/docker_server.py +++ b/ci/jobs/docker_server.py @@ -3,6 +3,7 @@ import json import logging import os +import shlex import tempfile import traceback from pathlib import Path @@ -289,6 +290,62 @@ def build_and_push_image( return result +def is_distroless_image(docker_image: str) -> bool: + _, tag = docker_image.rsplit(":", 1) + return "distroless" in tag.split("-") + + +def get_official_images_variant(docker_image: str) -> str: + # The official-images test runner derives its lookup variant from the final + # tag suffix. For example, head-distroless-amd64 is looked up as repo:amd64. + _, tag = docker_image.rsplit(":", 1) + return tag.rsplit("-", 1)[-1] + + +def write_distroless_docker_library_config(docker_image: str, config_dir: Path) -> Path: + """Map arch-suffixed distroless tags to the distroless-safe config tests.""" + # Generate a short config fragment for local arch-suffixed distroless CI tags. + # The runner derives tags like head-distroless-amd64 as repo:amd64; map that + # derived key to the distroless-safe tests because this helper is only used + # for images already identified as distroless. + repo, _ = docker_image.rsplit(":", 1) + variant = get_official_images_variant(docker_image) + image_variant = shlex.quote(f"{repo}:{variant}") + tests_var = ( + "keeperDistrolessSafeTests" + if "clickhouse-keeper" in repo + else "clickhouseDistrolessSafeTests" + ) + + generated_config = None + try: + with tempfile.NamedTemporaryFile( + "w", + prefix="docker-library-distroless-", + suffix=".sh", + dir=config_dir, + delete=False, + encoding="utf-8", + ) as f: + generated_config = Path(f.name) + f.write( + "#!/usr/bin/env bash\n" + "\n" + "explicitTests+=(\n" + f"\t[{image_variant}]=1\n" + ")\n" + "\n" + "imageTests+=(\n" + f'\t[{image_variant}]="${{{tests_var}}}"\n' + ")\n" + ) + return generated_config + except Exception: + if generated_config: + generated_config.unlink(missing_ok=True) + raise + + def test_docker_library(test_results) -> None: """we test our images vs the official docker library repository to track integrity""" arch = "amd64" if Utils.is_amd() else "arm64" @@ -311,10 +368,24 @@ def test_docker_library(test_results) -> None: raise RuntimeError(f"Failed to clone {repo}") run_sh = (repo_path / "test/run.sh").absolute() for image in check_images: - cmd = f"{run_sh} {image} -c {repo_path / 'test/config.sh'} -c {config_override}" - test_results.append( - Result.from_commands_run(name=f"{test_name} ({image})", command=cmd) - ) + generated_config = None + try: + configs = [repo_path / "test/config.sh", config_override] + if is_distroless_image(image): + generated_config = write_distroless_docker_library_config( + image, config_override.parent + ) + configs.append(generated_config) + config_args = " ".join( + f"-c {shlex.quote(Path(config).as_posix())}" for config in configs + ) + cmd = f"{shlex.quote(run_sh.as_posix())} {shlex.quote(image)} {config_args}" + test_results.append( + Result.from_commands_run(name=f"{test_name} ({image})", command=cmd) + ) + finally: + if generated_config: + generated_config.unlink(missing_ok=True) except Exception as e: logging.error("Failed while testing the docker library image: %s", e) diff --git a/ci/jobs/scripts/check_style/experimental_settings_ignore.txt b/ci/jobs/scripts/check_style/experimental_settings_ignore.txt index fc0de187e0335..bced383c7bc1d 100644 --- a/ci/jobs/scripts/check_style/experimental_settings_ignore.txt +++ b/ci/jobs/scripts/check_style/experimental_settings_ignore.txt @@ -9,6 +9,7 @@ allow_deprecated_database_ordinary allow_deprecated_snowflake_conversion_functions allow_distributed_ddl allow_drop_detached +allow_executable_tables allow_execute_multiif_columnar allow_experimental_alter_materialized_view_structure allow_experimental_analyzer @@ -73,3 +74,5 @@ allow_statistics_optimize allow_unrestricted_reads_from_keeper enable_vector_similarity_index allow_nullable_tuple_in_extracted_subcolumns +allow_experimental_cleanup_old_data_files_compaction +allow_experimental_iceberg_compaction diff --git a/ci/jobs/scripts/docker_server/config.sh b/ci/jobs/scripts/docker_server/config.sh index 84882b1745df8..0ad7b5e393884 100644 --- a/ci/jobs/scripts/docker_server/config.sh +++ b/ci/jobs/scripts/docker_server/config.sh @@ -5,12 +5,27 @@ currentDir="$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" # iterate over all directories in current path clickhouseTests=$( find "$currentDir"/tests/ -maxdepth 1 -name 'clickhouse-*' -not -name 'clickhouse-distroless-*' -type d -exec basename {} \; ) -clickhouseDistrolessTests=$( find "$currentDir"/tests/ -maxdepth 1 -name 'clickhouse-distroless-*' -type d -exec basename {} \; ) keeperTests=$( find "$currentDir"/tests/ -maxdepth 1 -name 'keeper-*' -type d -exec basename {} \; ) +# Distroless images have no shell or coreutils inside the container, so run +# only tests known to work with that constraint. +clickhouseDistrolessTests=$( find "$currentDir"/tests/ -maxdepth 1 -name 'clickhouse-distroless-*' -type d -exec basename {} \; ) +clickhouseDistrolessSafeTests="clickhouse-basics ${clickhouseDistrolessTests}" +keeperDistrolessTests=$( find "$currentDir"/tests/ -maxdepth 1 -name 'keeper-distroless-*' -type d -exec basename {} \; ) +keeperDistrolessSafeTests="clickhouse-distroless-no-shell ${keeperDistrolessTests}" + +# Distroless images cannot run the official-images global tests that assume a +# shell/coreutils are available. Mark these image variants as explicit so the +# runner uses only the imageTests listed below instead of adding global tests. +# Arch-suffixed local CI tags are handled in docker_server.py. +explicitTests+=( + ['clickhouse/clickhouse-server:distroless']=1 + ['clickhouse/clickhouse-keeper:distroless']=1 +) + imageTests+=( ['clickhouse/clickhouse-server']="${clickhouseTests}" - ['clickhouse/clickhouse-server:distroless']="${clickhouseTests} ${clickhouseDistrolessTests}" + ['clickhouse/clickhouse-server:distroless']="${clickhouseDistrolessSafeTests}" ['clickhouse/clickhouse-keeper']="${keeperTests}" - ['clickhouse/clickhouse-keeper:distroless']="${keeperTests}" + ['clickhouse/clickhouse-keeper:distroless']="${keeperDistrolessSafeTests}" ) diff --git a/ci/tests/data/integration_test_durations_sample.tsv b/ci/tests/data/integration_test_durations_sample.tsv new file mode 100644 index 0000000000000..55fbf44d0ff0d --- /dev/null +++ b/ci/tests/data/integration_test_durations_sample.tsv @@ -0,0 +1,1114 @@ +# Median per-module wall-clock of the integration test suite, in milliseconds, sampled from +# CIDB on 2026-08-28 over the preceding 14 days of `Integration tests (amd_asan_ubsan%)` runs +# on master. This is the same query kept next to `RAW_TEST_DURATIONS` in +# `ci/jobs/scripts/integration_tests_configs.py`, but without its `HAVING dur > 20000` floor, +# so every module that ran is present here. +# +# It is a test fixture only - the packer never reads it. It stands in for the real runtime of +# the suite so that `test_integration_batch_balance.py` can score the packer's estimates +# against something other than themselves. Refresh it together with `RAW_TEST_DURATIONS`. +test_storage_s3_queue/test_6.py 1616014 +test_storage_delta/test.py 1494562 +test_storage_kafka/test_batch_fast.py 1490645 +test_database_replicated_settings/test.py 1160798 +test_replicated_database/test.py 1103720 +test_storage_nats/test_nats_core.py 999120 +test_distributed_ddl/test.py 994994 +test_max_bytes_ratio_before_external_order_group_by_for_server/test.py 988963 +test_multiple_disks/test.py 968160 +test_backup_restore_s3/test.py 957420 +test_storage_rabbitmq/test.py 947108 +test_dictionaries_all_layouts_separate_sources/test_mongo.py 935530 +test_keeper_session/test.py 934939 +test_backup_restore_new/test.py 906356 +test_storage_s3/test.py 869928 +test_refreshable_mat_view/test.py 823846 +test_dictionaries_redis/test.py 815978 +test_distributed_load_balancing/test.py 790506 +test_storage_s3_queue/test_system_stop.py 732918 +test_postgresql_replica_database_engine/test_3.py 723415 +test_storage_s3_queue/test_5.py 713399 +test_storage_azure_blob_storage/test.py 644714 +test_storage_nats/test_nats_jet_stream.py 624550 +test_restore_db_replica/test.py 606610 +test_dictionaries_all_layouts_separate_sources/test_clickhouse_remote.py 592646 +test_dictionaries_all_layouts_separate_sources/test_clickhouse_local.py 585430 +test_ttl_move/test.py 580569 +test_storage_kafka/test_system_stop.py 579738 +test_dictionaries_all_layouts_separate_sources/test_https.py 578064 +test_dictionaries_all_layouts_separate_sources/test_mysql.py 578042 +test_dictionaries_all_layouts_separate_sources/test_http.py 575292 +test_backup_restore_on_cluster/test_concurrency.py 568740 +test_unknown_config_option/test.py 560730 +test_storage_s3_queue/test_0.py 560468 +test_parallel_replicas_insert_select/test.py 550342 +test_async_load_databases/test.py 535642 +test_mask_sensitive_info/test.py 512318 +test_storage_s3_queue/test_2.py 506180 +test_dns_cache/test.py 491303 +test_refreshable_mv/test.py 489998 +test_storage_iceberg_with_spark/test_minmax_pruning.py 480981 +test_backup_restore_on_cluster/test.py 477239 +test_storage_nats/test_system_stop.py 471394 +test_storage_kafka/test_partition_affinity.py 454762 +test_named_collections/test.py 454221 +test_s3_cluster_restart/test.py 450620 +test_storage_s3_queue/test_3.py 445172 +test_postgresql_replica_database_engine/test_1.py 428644 +test_database_iceberg/test.py 423117 +test_storage_rabbitmq/test_system_stop.py 422460 +test_refreshable_mat_view_replicated/test.py 412588 +test_storage_iceberg_with_spark/test_cluster_table_function.py 407994 +test_distributed_directory_monitor_split_batch_on_failure/test.py 407787 +test_storage_hdfs/test.py 406028 +test_concurrent_ttl_merges/test.py 403624 +test_checking_s3_blobs_paranoid/test.py 390668 +test_lost_part_during_startup/test.py 381729 +test_storage_s3_queue/test_1.py 374710 +test_database_delta/test.py 373347 +test_merge_tree_s3/test.py 372586 +test_parallel_replicas_over_distributed/test.py 367391 +test_insert_distributed_async_send/test.py 367255 +test_mysql_database_engine/test.py 366220 +test_storage_iceberg_schema_evolution/test_evolved_schema_simple.py 363624 +test_executable_table_function/test.py 361915 +test_scheduler_io/test.py 353404 +test_ttl_replicated/test.py 347312 +test_storage_kafka/test_batch_slow_1.py 344995 +test_kafka_bad_messages/test.py 343528 +test_cluster_discovery/test.py 341637 +test_mysql57_database_engine/test.py 339696 +test_filesystem_cache/test.py 338846 +test_storage_kafka/test_batch_slow_4.py 336236 +test_named_collections_encrypted2/test.py 332032 +test_text_index_upgrade/test.py 329574 +test_mysql_protocol/test.py 320786 +test_filesystem_split_cache/test.py 319537 +test_database_glue/test.py 318482 +test_drop_is_lock_free/test.py 317572 +test_dictionaries_dependency/test.py 314930 +test_dictionaries_ddl/test.py 305455 +test_ytsaurus/test_tables.py 300906 +test_crash_log/test.py 299175 +test_azure_403_handling/test.py 296521 +test_postgresql_replica_database_engine/test_2.py 295468 +test_row_policy/test.py 292768 +test_rename_column/test.py 290758 +test_create_as_select_on_cluster_distributed/test.py 288555 +test_storage_iceberg_with_spark/test_partition_pruning.py 287590 +test_prometheus_protocols/test_evaluation.py 287439 +test_parallel_replicas_custom_key_failover/test.py 286354 +test_wasm_parser/test.py 278966 +test_storage_iceberg_with_spark/test_expire_snapshots.py 273594 +test_storage_kafka/test_batch_slow_5.py 273274 +test_dictionaries_all_layouts_separate_sources/test_file.py 272082 +test_ai_functions/test.py 271658 +test_storage_iceberg_with_spark/test_system_iceberg_metadata.py 268926 +test_parallel_replicas_custom_key_load_balancing/test.py 268092 +test_group_by_top_k_distributed/test.py 267160 +test_storage_url/test.py 260162 +test_storage_kafka/test_compression_codec.py 258730 +test_parallel_replicas_invisible_parts/test.py 257780 +test_postpone_failed_tasks/test.py 255489 +test_backward_compatibility/test_aggregate_function_state.py 251694 +test_statistics_cache/test.py 251199 +test_s3_plain_rewritable/test.py 249952 +test_storage_bigquery/test.py 249639 +test_create_handler/test.py 249593 +test_storage_iceberg_schema_evolution/test_tuple_evolved_simple.py 245674 +test_table_db_num_limit/test.py 241817 +test_storage_kafka/test_batch_slow_6.py 240683 +test_refreshable_mv_no_multi_read/test.py 239207 +test_hedged_requests/test.py 238980 +test_cleanup_dir_after_bad_zk_conn/test.py 236966 +test_distributed_inter_server_secret/test.py 235657 +test_keeper_ttl_nodes/test.py 235416 +test_s3_cluster/test.py 231724 +test_backup_restore_on_cluster/test_cancel_backup.py 230732 +test_postgresql_replica_database_engine/test_0.py 228178 +test_polymorphic_parts/test.py 227736 +test_parallel_replicas_custom_key/test.py 219820 +test_storage_kafka/test_batch_slow_0.py 219579 +test_storage_s3_queue/test_4.py 216644 +test_throttling/test.py 214834 +test_http_handlers_config/test.py 211559 +test_insert_into_distributed/test.py 208753 +test_storage_iceberg_with_spark/test_position_deletes.py 205822 +test_storage_postgresql/test.py 205499 +test_implicit_index_upgrade/test.py 205363 +test_lost_part/test.py 203962 +test_storage_s3_queue/test_flush.py 202305 +test_storage_iceberg_schema_evolution/test_array_evolved_nested.py 201028 +test_delayed_replica_failover/test.py 197427 +test_insert_distributed_load_balancing/test.py 197026 +test_broken_projections/test.py 196858 +test_parallel_replicas_distributed_skip_shards/test.py 196836 +test_dictionaries_all_layouts_separate_sources/test_executable_hashed.py 196605 +test_storage_mysql/test.py 195660 +test_postgresql_database_engine/test.py 192632 +test_string_aggregation_compatibility/test.py 188636 +test_dictionaries_all_layouts_separate_sources/test_mongo_uri.py 188494 +test_scheduler_memory/test.py 185382 +test_storage_mongodb/test.py 185362 +test_backup_restore_new/test_cancel_backup.py 185206 +test_storage_s3/test_sts.py 184932 +test_keeper_container_nodes/test.py 183989 +test_storage_iceberg_no_spark/test_writes_statistics_by_minmax_pruning.py 182229 +test_backward_compatibility/test_convert_ordinary.py 181319 +test_grant_and_revoke/test_with_table_engine_grant.py 180454 +test_replicated_users/test.py 180260 +test_parallel_replicas_alias_columns/test.py 179548 +test_file_cluster/test.py 179196 +test_cross_replication/test.py 178377 +test_backward_compatibility/test_pr_protocol_with_stream_id.py 177881 +test_distributed_ddl_on_cross_replication/test.py 177659 +test_replicated_mutations/test.py 177194 +test_jbod_balancer/test.py 176566 +test_storage_azure_blob_storage/test_cluster.py 176154 +test_disk_over_web_server/test.py 175494 +test_storage_kerberized_kafka/test.py 174267 +test_ytsaurus/test_dictionaries.py 174234 +test_hive_query/test.py 173761 +test_merge_tree_azure_blob_storage/test.py 173026 +test_executable_udf_async_metrics/test.py 172148 +test_s3_cluster_insert_select/test.py 171188 +test_storage_iceberg_with_spark/test_writes.py 169606 +test_quorum_inserts/test.py 169178 +test_storage_kafka/test_batch_slow_2.py 167490 +test_storage_kafka/test_kafka_zone_awareness.py 167034 +test_system_logs/test_system_logs.py 167028 +test_parallel_replicas_protocol/test.py 166400 +test_replicated_fetches_bandwidth/test.py 165792 +test_distributed_frozen_replica/test.py 165152 +test_stop_insert_when_disk_close_to_full/test.py 163503 +test_max_rows_to_read_leaf_with_view/test.py 162144 +test_secure_socket/test.py 162060 +test_keeper_two_nodes_cluster/test.py 161698 +test_dictionaries_all_layouts_separate_sources/test_executable_cache.py 161148 +test_groupBitmapAnd_on_distributed/test_groupBitmapAndState_on_distributed_table.py 160290 +test_groupBitmapAnd_on_distributed/test.py 160180 +test_plain_rewritable_backward_compatibility/test.py 160126 +test_truncate_database/test_distributed.py 159952 +test_insert_into_distributed_through_materialized_view/test.py 159017 +test_keeper_snapshot_chunked_transfer/test.py 157386 +test_interserver_dns_retires/test.py 156439 +test_manipulate_statistics/test.py 155372 +test_drop_database_replica/test.py 154301 +test_keeper_internal_secure/test.py 154124 +test_distributed_default_database/test.py 153528 +test_dictionaries_update_and_reload/test.py 153312 +test_storage_iceberg_with_spark/test_remove_orphan_files.py 152428 +test_distributed_storage_configuration/test.py 152252 +test_encrypted_disk/test.py 150924 +test_prometheus_protocols/test_different_table_engines.py 150815 +test_s3_plain_rewritable_rotate_tables/test.py 150344 +test_s3_aws_sdk_has_slightly_unreliable_behaviour/test.py 149976 +test_distributed_config/test.py 146980 +test_unknown_column_dist_table_with_alias/test.py 146305 +test_storage_iceberg_with_spark/test_writes_mutate_delete.py 144582 +test_mutations_with_tampered_parts/test.py 144260 +test_database_backup/test.py 144008 +test_keeper_zookeeper_converter/test.py 143702 +test_restore_replica/test.py 143266 +test_keeper_map/test.py 140802 +test_transactions/test.py 138900 +test_log_query_probability/test.py 138150 +test_system_clusters_actual_information/test.py 137811 +test_storage_iceberg_disks/test.py 135830 +test_cluster_discovery/test_auxiliary_keeper.py 135804 +test_default_session_user/test.py 135509 +test_refreshable_mv_skip_old_temp_table_ddls/test.py 135363 +test_scheduler_query/test.py 132120 +test_kafka_bad_messages/test_mv_target_missing.py 131865 +test_system_logs_recreate/test.py 130042 +test_attach_without_fetching/test.py 128826 +test_distributed_index_analysis/test.py 128692 +test_ddl_worker_replicas/test.py 127608 +test_storage_kafka/test_keeper_session_loss_direct_read.py 127415 +test_quota/test.py 126958 +test_merges_memory_limit/test.py 123236 +test_corrupted_part_files/test.py 122005 +test_distributed_ddl_parallel/test.py 122002 +test_postgresql_ssl/test.py 121939 +test_recompression_ttl/test.py 121381 +test_version_update_after_mutation/test.py 120658 +test_disk_access_storage/test.py 119657 +test_recovery_replica/test.py 119216 +test_merge_tree_hdfs/test.py 118630 +test_storage_iceberg_schema_evolution/test_tuple_evolved_nested.py 118022 +test_storage_kafka/test_produce_http_interface.py 117776 +test_MemoryTracking/test.py 117166 +test_storage_iceberg_with_spark/test_writes_create_partitioned_table.py 116358 +test_disk_configuration/test.py 115544 +test_mutations_with_merge_tree/test.py 112968 +test_named_collections_encrypted2/test_integr.py 112803 +test_replicated_user_defined_functions/test.py 110990 +test_reloading_storage_configuration/test.py 110854 +test_zookeeper_config/test_secure.py 108830 +test_storage_iceberg_with_spark/test_manifest_compaction.py 108550 +test_storage_hudi/test.py 108458 +test_partition/test.py 108346 +test_database_remote/test.py 108142 +test_system_merges/test.py 107666 +test_backup_restore_on_cluster_with_checksum_data_file_name/test.py 107487 +test_migration_deduplication_hash/test.py 106927 +test_database_disk_setting/test.py 105208 +test_paimon_incremental_read/test.py 105106 +test_alter_settings_or_comment_on_cluster/test.py 104259 +test_backup_restore_azure_blob_storage/test.py 102966 +test_dictionaries_postgresql/test.py 102937 +test_executable_dictionary/test.py 102592 +test_create_union_system_log_tables/test.py 102109 +test_azure_blob_storage_plain_rewritable/test.py 102036 +test_alter_moving_garbage/test.py 101982 +test_allowed_url_from_config/test.py 101237 +test_hedged_requests_parallel/test.py 101221 +test_s3_table_functions/test.py 100929 +test_merge_tree_load_parts/test.py 100440 +test_backup_restore_keeper_map/test.py 99469 +test_storage_delta_disks/test.py 99384 +test_dictionary_lazy_load/test.py 98840 +test_scheduler_cpu/test.py 98383 +test_dictionaries_mysql/test.py 97392 +test_distributed_plan_replicated_merge_tree/test.py 96486 +test_drop_replica_with_auxiliary_zookeepers/test.py 96176 +test_executable_user_defined_function/test.py 96054 +test_modify_engine_on_restart/test_ordinary.py 95658 +test_arrowflight_interface/test_sql_server.py 95365 +test_settings_profile/test.py 95245 +test_globs_in_filepath/test.py 94044 +test_keeper_reconfig_replace_leader_in_one_command/test.py 92104 +test_keeper_back_to_back/test.py 91904 +test_storage_iceberg_with_spark/test_query_condition_cache.py 91852 +test_cluster_discovery/test_dynamic_clusters.py 91516 +test_executable_udf_profile_events/test.py 90467 +test_system_start_stop_listen/test.py 90279 +test_check_table/test.py 89633 +test_rocksdb_options/test.py 89556 +test_distributed_ddl/test_replicated_alter.py 88540 +test_random_inserts/test.py 88512 +test_keeper_incorrect_config/test.py 88198 +test_restore_replica_metadata_version/test.py 88000 +test_keeper_password/test.py 87310 +test_storage_iceberg_with_spark/test_metadata_file_format_with_uuid.py 87155 +test_keeper_remove_rejoin_leader/test.py 87126 +test_format_schema_source/test.py 86700 +test_storage_iceberg_with_spark/test_metadata_file_selection.py 86544 +test_storage_iceberg_with_spark/test_writes_mutate_update.py 86466 +test_keeper_three_nodes_two_alive/test.py 85669 +test_scheduler_cpu_preemptive/test.py 85255 +test_server_reload/test.py 85146 +test_backup_restore_on_cluster/test_disallow_concurrency.py 85117 +test_storage_iceberg_with_spark/test_schema_evolution_with_time_travel.py 85010 +test_backup_source_grants/test.py 84900 +test_storage_s3_queue/test_parallel_inserts.py 84713 +test_modify_engine_on_restart/test.py 84392 +test_access_control_with_custom_setup/test.py 84348 +test_restore_external_engines/test.py 84090 +test_remote_blobs_naming/test_backward_compatibility.py 83620 +test_system_metrics/test.py 83426 +test_s3_table_function_with_http_proxy/test.py 83044 +test_zookeeper_send_window_broken_promise/test.py 82942 +test_ddl_create_then_alter_offline_replica/test.py 82900 +test_s3_table_function_with_https_proxy/test.py 82642 +test_storage_iceberg_with_spark/test_schema_inference.py 82254 +test_jbod_ha/test.py 82192 +test_keeper_opentelemetry_tracing/test.py 82072 +test_refreshable_mv_watch_fault/test.py 81637 +test_always_fetch_merged/test.py 80848 +test_system_flush_logs/test.py 80564 +test_store_cleanup/test.py 80458 +test_rmv_access_denied_on_rename_race/test.py 80328 +test_replicated_merge_tree_compatibility/test.py 79758 +test_clickhouse_server_wait_server_pool/test.py 79623 +test_replicated_database_interserver_host/test.py 79450 +test_replication_credentials/test.py 79376 +test_consistant_parts_after_move_partition/test.py 79220 +test_https_replication/test.py 78942 +test_query_runner/test.py 78862 +test_s3_credentials_hardening/test.py 77971 +test_dictionaries_dependency_xml/test.py 77866 +test_client_auto_secure_port/test.py 76975 +test_catboost_evaluate/test.py 76901 +test_mark_cache_profile_events/test.py 75708 +test_file_schema_inference_cache/test.py 75324 +test_storage_kafka_sasl/test.py 74674 +test_database_catalog_shutdown_system_logs/test.py 74009 +test_storage_kafka/test_avro_schema_registry.py 73944 +test_keeper_force_recovery/test.py 73924 +test_storage_iceberg_with_spark/test_writes_create_table.py 73787 +test_replicated_merge_tree_encryption_codec/test.py 73508 +test_keeper_disks/test.py 73220 +test_index_filename_upgrade/test.py 73091 +test_user_memory_tracker_log_drift/test.py 73031 +test_log_family_hdfs/test.py 73012 +test_storage_iceberg_schema_evolution/test_array_evolved_with_struct.py 72746 +test_keeper_ttl_nodes/test_disabled.py 72270 +test_multi_access_storage_role_management/test.py 72265 +test_inserts_with_keeper_retries/test.py 71442 +test_replicated_table_attach/test.py 70826 +test_keeper_container_nodes/test_disabled.py 70276 +test_merge_tree_s3_failover/test.py 70183 +test_storage_s3_queue/test_sts_smoke.py 69911 +test_server_overload/test.py 69874 +test_keeper_persistent_watches/test.py 69764 +test_replicated_database_cluster_groups/test.py 69689 +test_http_failover/test.py 69590 +test_storage_iceberg_schema_evolution/test_evolved_schema_complex.py 69329 +test_sparsity_exact_num_defaults_compat/test.py 69179 +test_replicated_database_recover_digest_mismatch/test.py 68903 +test_default_compression_codec/test.py 68522 +test_keeper_nodes_remove/test.py 68346 +test_server_metadata_files/test.py 67297 +test_sharding_key_from_default_column/test.py 67046 +test_replicated_merge_tree_wait_on_shutdown/test.py 66766 +test_backward_compatibility/test_aggregate_function_state_contingency_functions.py 66468 +test_storage_iceberg_with_trino/test.py 66382 +test_redirect_url_storage/test.py 66134 +test_named_collections_if_exists_on_cluster/test.py 65527 +test_storage_iceberg_with_spark/test_iceberg_snapshot_reads.py 64894 +test_storage_kafka/test_poll_timeout_after_assignment.py 64428 +test_parallel_replicas_snapshot_from_initiator/test.py 64056 +test_postgresql_remote_host_filter/test.py 63944 +test_packed_io/test.py 63775 +test_keeper_4lw_reconfiguration/test.py 63676 +test_async_insert_memory/test.py 63552 +test_attach_tampered_detached_parts/test.py 63164 +test_graphite_merge_tree/test.py 63027 +test_alter_on_mixed_type_cluster/test.py 62554 +test_storage_iceberg_with_spark/test_row_lineage_pruning.py 62156 +test_backup_restore_on_cluster/test_huge_concurrent_restore.py 61518 +test_storage_iceberg_with_spark/test_explicit_metadata_file.py 61320 +test_graphite_merge_tree_typed/test.py 61192 +test_backward_compatibility/test_block_marshalling.py 60941 +test_lightweight_updates/test.py 60738 +test_http_connection_socket_buffer_settings/test.py 60708 +test_host_regexp_multiple_ptr_records/test.py 60094 +test_keeper_max_append_byte_size/test.py 60000 +test_auth_method_grants_deferred_expiry/test.py 59966 +test_limited_replicated_fetches/test.py 59659 +test_dictionaries_replace/test.py 59572 +test_background_operations_config/test.py 59563 +test_group_array_element_size/test.py 59514 +test_distributed_format/test.py 59443 +test_azure_blob_storage_native_copy/test.py 59166 +test_settings_constraints/test.py 59145 +test_keeper_auth/test.py 58956 +test_phantom_parts_in_mutations/test.py 58866 +test_allow_feature_tier/test.py 58591 +test_table_function_mongodb/test.py 58528 +test_backup_restore_new/test_shutdown_wait_backup.py 58366 +test_executable_pool_udf_profile_events/test.py 58366 +test_replicated_merge_tree_s3/test.py 58346 +test_storage_iceberg_with_spark/test_writes_schema_evolution.py 57977 +test_storage_iceberg_with_spark/test_delete_files.py 57794 +test_storage_redis/test.py 57414 +test_zookeeper_config_load_balancing/test.py 57323 +test_no_merges_volume_ttl/test.py 57190 +test_distributed_ddl_password/test.py 57153 +test_keeper_snapshot_small_distance/test.py 57091 +test_send_request_to_leader_replica/test.py 56948 +test_replicated_merge_tree_encrypted_disk/test.py 56944 +test_backup_restore_on_cluster/test_two_shards_two_replicas.py 56794 +test_backward_compatibility/test_aggregate_function_state_tuple_return_type.py 56720 +test_keeper_reconfig_remove_many/test.py 56690 +test_storage_iceberg_concurrent/test_concurrent_reads.py 56684 +test_grant_and_revoke/test_without_table_engine_grant.py 56444 +test_system_detached_tables/test.py 56393 +test_backups_from_disk/test.py 55830 +test_https_replication/test_change_ip.py 55442 +test_lightweight_updates_compatibility/test.py 55389 +test_user_valid_until/test.py 54948 +test_attach_partition_using_copy/test.py 54912 +test_async_insert_adaptive_busy_timeout/test.py 54646 +test_ddl_on_cluster_stop_waiting_for_offline_hosts/test.py 54573 +test_named_collections_encrypted/test.py 54127 +test_mysql_kill_query/test.py 53999 +test_keeper_nodes_add/test.py 53930 +test_system_ddl_worker_queue/test.py 53846 +test_storage_iceberg_with_spark/test_async_metadata_refresh.py 53560 +test_old_parts_finally_removed/test.py 53498 +test_keeper_mntr_pressure/test.py 53198 +test_keeper_block_acl/test.py 53146 +test_prometheus_protocols/test_compliance.py 53080 +test_storage_kafka/test_intent_sizes.py 52996 +test_read_only_table/test.py 52664 +test_storage_iceberg_schema_evolution/test_array_map_evolved_with_struct.py 52614 +test_warning_broken_tables/test.py 52609 +test_database_iceberg_lakekeeper_catalog/test.py 52546 +test_quorum_inserts_parallel/test.py 52023 +test_storage_iceberg_with_spark/test_format_version_upgrade.py 51998 +test_memory_limit_observer/test.py 51755 +test_replicated_merge_tree_with_auxiliary_zookeepers/test.py 51672 +test_storage_iceberg_with_spark/test_writes_with_partitioned_table.py 51654 +test_disabled_access_control_improvements/test_row_policy.py 51529 +test_nullable_tuple_subcolumns/test.py 51361 +test_storage_numbers/test.py 51310 +test_backward_compatibility/test_bucketed_map_order.py 51296 +test_storage_iceberg_with_spark/test_read_in_order.py 50994 +test_replace_partition/test.py 50696 +test_mutations_in_partitions_of_merge_tree/test.py 50390 +test_storage_iceberg_with_spark/test_bucket_partition_pruning.py 49964 +test_reload_auxiliary_zookeepers/test.py 49856 +test_max_suspicious_broken_parts_replicated/test.py 49714 +test_storage_alias_replicated/test.py 49667 +test_version_update/test.py 49486 +test_temporary_data_in_cache/test.py 49481 +test_consistent_parts_after_clone_replica/test.py 49426 +test_keeper_map_retries/test.py 49395 +test_user_directories/test.py 49278 +test_replicated_access/test.py 49232 +test_tmp_policy/test.py 49184 +test_modify_engine_on_restart/test_unsafe_name.py 49175 +test_concurrent_queries_restriction_by_query_kind/test.py 49159 +test_backward_compatibility/test_parallel_replicas_protocol.py 48424 +test_storage_kafka/test_schema_registry_skip_bytes.py 48400 +test_database_iceberg_nessie_catalog/test.py 48312 +test_fetch_partition_should_reset_mutation/test.py 48298 +test_encrypted_disk_replication/test.py 48200 +test_keeper_readahead/test.py 47850 +test_config_substitutions/test.py 47796 +test_matview_union_replicated/test.py 47784 +test_backup_restore_on_cluster/test_slow_rmt.py 47718 +test_fetch_partition_from_auxiliary_zookeeper/test.py 47648 +test_storage_kafka/test_zookeeper_locks.py 47641 +test_statistics_minmax_upgrade/test.py 47547 +test_backward_compatibility/test_aggregation_with_out_of_order_buckets.py 47252 +test_kafka_bad_messages/test_1.py 47226 +test_distributed_insert_backward_compatibility/test.py 46896 +test_https_s3_table_function_with_http_proxy_no_tunneling/test.py 46894 +test_grpc_protocol/test.py 46795 +test_parallel_replicas_increase_error_count/test.py 46667 +test_reload_clusters_config/test.py 46405 +test_on_cluster_timeouts/test.py 46226 +test_fetch_partition_with_outdated_parts/test.py 46127 +test_keeper_as_server/test.py 45995 +test_backup_restore_on_cluster_s3_credentials/test.py 45981 +test_backup_restore/test.py 45890 +test_non_default_compression/test.py 45678 +test_zookeeper_config/test.py 45632 +test_race_condition_for_replicated_merge_tree/test.py 45553 +test_keeper_znode_time/test.py 45491 +test_settings_constraints_distributed/test.py 45446 +test_backup_restore_on_cluster/test_different_versions.py 45036 +test_drop_replica/test.py 44960 +test_join_set_family_s3/test.py 44927 +test_replicated_s3_zero_copy_drop_partition/test.py 44725 +test_disks_app_func/test.py 44540 +test_storage_log_damaged_array/test.py 44380 +test_postgresql_kill_query/test.py 43984 +test_variant_escaping_merge_tree_compatibility/test.py 43937 +test_part_uuid/test.py 43877 +test_keeper_snapshots/test.py 43559 +test_concurrent_threads_soft_limit/test.py 43224 +test_undrop_query/test.py 42998 +test_wasm_udf_fail_close/test.py 42760 +test_zookeeper_connection_log/test.py 42550 +test_ldap_external_user_directory/test.py 42480 +test_keeper_multinode_simple/test.py 42072 +test_keeper_snapshot_chunked_transfer/test_concurrent.py 41962 +test_storage_s3/test_invalid_env_credentials.py 41958 +test_storage_kafka/test_batch_slow_7.py 41492 +test_keeper_four_word_command/test.py 41389 +test_replicated_fetches_min_part_level/test.py 41372 +test_attach_with_different_projections_or_indices/test.py 41032 +test_dictionary_ddl_on_cluster/test.py 40786 +test_parallel_table_shutdown/test.py 40624 +test_ssl_cert_authentication/test.py 40509 +test_dictionaries_redis/test_long.py 40338 +test_force_drop_table/test.py 40239 +test_dictionaries_config_reload/test.py 40230 +test_insert_over_http_query_log/test.py 40125 +test_arrowflight_interface/test.py 40073 +test_keeper_feature_flags_config/test.py 39890 +test_part_loading_tree_rollback/test.py 39872 +test_keeper_session_refuse_stale_server/test.py 39826 +test_disabled_access_control_improvements/test_users_without_row_policies_can_read_rows.py 39807 +test_zookeeper_config/test_password.py 39577 +test_keeper_nodes_move/test.py 39394 +test_arrowflight_storage/test.py 39269 +test_executable_user_defined_functions_config_reload/test.py 39158 +test_backup_restore_storage_policy/test.py 39037 +test_modify_engine_on_restart/test_storage_policies.py 38645 +test_alter_database_on_cluster/test.py 38595 +test_ddl_worker_stale_task_name/test.py 38478 +test_backward_compatibility/test_ip_types_binary_compatibility.py 38400 +test_storage_iceberg_with_spark_cache/test_metadata_cache.py 38308 +test_parts_delete_zookeeper/test.py 38237 +test_db_ordinary_deprecated_warning/test.py 38158 +test_transposed_metric_log/test.py 38002 +test_keeper_force_recovery_single_node/test.py 37942 +test_config_decryption/test_wrong_settings.py 37940 +test_backward_compatibility/test_cte_distributed.py 37824 +test_storage_iceberg_with_spark/test_metadata_file_selection_from_version_hint.py 37740 +test_keeper_broken_logs/test.py 37707 +test_parallel_replicas_failover/test.py 37660 +test_storage_iceberg_interoperability_azure/test_interoperability.py 37585 +test_storage_delta_shuffles/test.py 37375 +test_alternative_keeper_config/test.py 37330 +test_storage_iceberg_schema_evolution/test_map_evolved_nested.py 37212 +test_force_deduplication/test.py 37057 +test_zero_copy_drop_table_with_leftover/test.py 36996 +test_database_hms/test.py 36669 +test_database_hms/test_ttransport_exception_reproduction.py 36588 +test_storage_delta/test_cdf.py 36544 +test_odbc_interaction/test.py 36466 +test_prometheus_protocols/test_upgrade_from_prealpha.py 36125 +test_atomic_drop_table/test.py 36074 +test_keeper_max_request_size/test.py 36062 +test_keeper_log_gap_before_committed/test.py 35918 +test_move_partition_to_volume_async/test.py 35880 +test_reload_zookeeper/test.py 35848 +test_storage_iceberg_with_spark/test_file_stats_logging.py 35818 +test_s3_storage_conf_proxy/test.py 35785 +test_reload_client_certificate/test.py 35744 +test_storage_iceberg_with_spark/test_optimize.py 35688 +test_storage_iceberg_no_spark/test_writes_rename_column.py 35668 +test_dictionaries_select_all/test.py 35585 +test_storage_iceberg_no_spark/test_identity_partition_column_projection.py 35482 +test_ddl_alter_query/test.py 35469 +test_replicated_fetches_timeouts/test.py 35432 +test_keeper_reconfig_remove/test.py 35378 +test_keeper_reconfig_replace_leader/test.py 35320 +test_max_suspicious_broken_parts/test.py 35131 +test_dremio_engine/test.py 35056 +test_profile_max_sessions_for_user/test.py 34924 +test_postgresql_protocol/test.py 34814 +test_backward_compatibility/test_adaptive_codec.py 34784 +test_attach_partition_with_large_destination/test.py 34746 +test_prometheus_endpoint/test.py 34743 +test_storage_iceberg_with_spark/test_explanation.py 34668 +test_keeper_three_nodes_start/test.py 34650 +test_startup_scripts/test.py 34600 +test_mutations_with_projection/test.py 34570 +test_keeper_dynamic_settings/test.py 34544 +test_remove_stale_moving_parts/test.py 34462 +test_access_control_on_cluster/test.py 34446 +test_distributed_async_insert_for_node_changes/test.py 34212 +test_session_log/test.py 34152 +test_select_access_rights/test_from_system_tables.py 33968 +test_keeper_s3_snapshot/test.py 33854 +test_keeper_snapshot_on_exit/test.py 33608 +test_modify_engine_on_restart/test_mv.py 33512 +test_storage_iceberg_interoperability_local/test_interoperability.py 33390 +test_distributed_respect_user_timeouts/test.py 33143 +test_disk_checker/test.py 33131 +test_storage_s3_queue/test_dimensional_metrics.py 33014 +test_restart_server/test.py 32945 +test_replication_without_zookeeper/test.py 32832 +test_zookeeper_fallback_session/test.py 32658 +test_force_restore_data_flag_for_keeper_dataloss/test.py 32472 +test_keeper_snapshot_rotation_race/test.py 32428 +test_analyzer_compatibility/test.py 31685 +test_ddl_worker_non_leader/test.py 31678 +test_hot_reload_storage_policy/test.py 31656 +test_rocksdb_read_only/test.py 31573 +test_user_query_log_config_validation/test.py 31521 +test_server_startup_and_shutdown_logs/test.py 31471 +test_asynchronous_metrics_pk_bytes_fields/test.py 31432 +test_mongodb_kill_query/test.py 31429 +test_suggestions/test.py 31411 +test_acme_tls/test_multi_node.py 31320 +test_reloading_settings_from_users_xml/test.py 31194 +test_modify_engine_on_restart/test_zk_path_exists.py 31178 +test_distributed_structure_fetch/test.py 31124 +test_ttl_multilevel_group_by/test.py 30680 +test_materialize_projections_on_merge/test.py 30638 +test_webassembly_udf/test.py 30616 +test_search_orphaned_parts/test.py 30550 +test_compression_nested_columns/test.py 30261 +test_dictionaries_wait_for_load/test.py 30228 +test_access_cache_recompute_coalescing/test.py 30218 +test_distributed_plan_worker_exchange_port/test.py 30216 +test_keeper_persistent_log_multinode/test.py 30072 +test_compressed_marks_restart/test.py 30047 +test_acme_tls/test_single_node.py 30011 +test_backward_compatibility/test_functions.py 29930 +test_concurrent_part_removal_threshold_for_remote_disk/test.py 29792 +test_keeper_reconfig_add/test.py 29784 +test_cache_bypass_on_disk_failure/test.py 29782 +test_disabled_access_control_improvements/test_select_from_system_tables.py 29744 +test_storage_s3_queue/test_file_iterator_ttl.py 29682 +test_play_reconcile_startup/test.py 29666 +test_merge_tree_s3_with_cache/test.py 29594 +test_disable_insertion_and_mutation/test.py 29290 +test_parallel_replicas_no_replicas/test.py 29263 +test_storage_iceberg_with_spark/test_column_names_with_dots.py 29210 +test_sync_replica_on_cluster/test.py 29184 +test_modify_engine_on_restart/test_args.py 29180 +test_parallel_replicas_all_marks_read/test.py 29068 +test_backup_restore_s3/test_throttling.py 29044 +test_user_defined_object_persistence/test.py 29016 +test_auth_method_grants_on_cluster/test.py 28950 +test_modify_engine_on_restart/test_unusual_path.py 28947 +test_rabbitmq_malicious_broker/test.py 28895 +test_auth_method_valid_until_stateful_protocols/test.py 28866 +test_shutdown_wait_unfinished_queries/test.py 28834 +test_sqlite_kill_query/test.py 28751 +test_keeper_azure_s3_plain/test.py 28690 +test_ddl_worker_with_loopback_hosts/test.py 28680 +test_log_family_s3/test.py 28604 +test_keeper_unpreprocessed_logs_livelock/test.py 28552 +test_system_queries/test.py 28546 +test_parallel_replicas_skip_inactive_replicas_all_groups/test.py 28406 +test_dictionary_asynchronous_metrics/test.py 28402 +test_permissions_drop_replica/test.py 28386 +test_parallel_replicas_skip_inactive_replicas/test.py 28356 +test_validate_only_initial_alter_query/test_replicated_database.py 28293 +test_match_process_uid_against_data_owner/test.py 28273 +test_paimon_rest_catalog/test.py 28082 +test_keeper_follower_metrics/test.py 28046 +test_dictionary_allow_read_expired_keys/test_dict_get_or_default.py 28019 +test_filesystem_cache_eviction_metrics/test.py 27994 +test_format_avro_confluent/test.py 27931 +test_insert_into_distributed_sync_async/test.py 27795 +test_placement_info/test.py 27787 +test_drop_if_empty/test.py 27745 +test_dictionary_allow_read_expired_keys/test_dict_get.py 27666 +test_keeper_raft_cert_reload/test.py 27556 +test_covered_by_broken_exists/test.py 27505 +test_filesystem_layout/test.py 27469 +test_experimental_codec_config_default/test.py 27466 +test_compatibility_merge_tree_settings/test.py 27408 +test_dictionary_allow_read_expired_keys/test_default_reading.py 27394 +test_ssh/test.py 27332 +test_storage_iceberg_with_spark/test_minmax_pruning_with_null.py 27322 +test_storage_iceberg_with_spark/test_writes_field_ids_spark_read.py 27288 +test_keeper_persistent_log/test.py 27287 +test_system_logs_hostname/test_replicated.py 27189 +test_reset_ddl_worker/test.py 27180 +test_storage_iceberg_with_spark/test_writes_from_zero.py 27066 +test_fix_metadata_version/test.py 27012 +test_extreme_deduplication/test.py 26808 +test_part_log_table/test.py 26758 +test_keeper_remove_acl/test.py 26639 +test_bind_host/test.py 26632 +test_s3_access_headers/test.py 26506 +test_replica_is_active/test.py 26472 +test_replicated_access/test_invalid_entity.py 26369 +test_cleanup_after_start/test.py 26312 +test_executable_udf_names_in_system_query_log/test.py 26280 +test_backup_log/test.py 26242 +test_storage_iceberg_no_spark/test_iceberg_history_large_summary.py 26214 +test_ddl_worker_retry_when_dropping_db_failed/test.py 26074 +test_check_table_name_length_2/test.py 26022 +test_detached_parts_metrics/test.py 25876 +test_mutation_fetch_fallback/test.py 25820 +test_settings_constraints_distributed_ddl/test.py 25808 +test_settings_from_server/test.py 25728 +test_alter_comment_on_cluster/test.py 25606 +test_disabled_mysql_server/test.py 25425 +test_server_start_and_ip_conversions/test.py 25361 +test_storage_iceberg_schema_evolution/test_correct_column_mapper_is_chosen.py 24830 +test_parallel_replicas_insert_select_coordinator_reuse/test.py 24692 +test_userspace_page_cache/test.py 24648 +test_access_for_functions/test.py 24643 +test_default_database_on_cluster/test.py 24584 +test_no_password_existing_user/test.py 24558 +test_old_versions/test.py 24546 +test_topk_alpha_map_compatibility/test.py 24498 +test_table_function_redis/test.py 24471 +test_distributed_ddl_on_database_cluster/test.py 24454 +test_azure_blob_storage_listobjects_prefix/test.py 24422 +test_alter_settings_on_cluster/test.py 24394 +test_storage_iceberg_with_spark/test_row_lineage.py 24310 +test_cow_policy/test.py 24304 +test_default_compression_in_mergetree_settings/test.py 24284 +test_zero_copy_expand_macros/test.py 24179 +test_keeper_leader_metrics/test.py 24155 +test_acme_tls/test_no_certificate.py 24153 +test_keeper_snapshots_multinode/test.py 24107 +test_optimize_on_insert/test.py 24104 +test_external_cluster/test.py 24030 +test_broken_part_during_merge/test.py 24018 +test_mutations_hardlinks/test.py 23948 +test_threadpool_readers/test.py 23876 +test_log_lz4_streaming/test.py 23826 +test_broken_tmp_txn_version_startup/test.py 23808 +test_asynchronous_metric_log_table/test.py 23626 +test_zero_copy_lock_leak/test.py 23542 +test_early_memory_limit_exception/test.py 23538 +test_intersecting_parts/test.py 23415 +test_ddl_config_hostname/test.py 23399 +test_keeper_mntr_data_size/test.py 23282 +test_totp_auth/test_totp.py 23246 +test_storage_iceberg_no_spark/test_cluster_partition_pruning_reads.py 23182 +test_keeper_restore_from_snapshot/test_disk_s3.py 23156 +test_move_partition_to_disk_on_cluster/test.py 23120 +test_profile_events_s3/test.py 23068 +test_max_authentication_methods_per_user/test.py 23056 +test_storage_s3/test_parquet_prewhere.py 23031 +test_paimon_metadata_files_cache/test.py 22898 +test_cluster_discovery/test_password.py 22894 +test_external_http_authenticator/test.py 22878 +test_storage_iceberg_no_spark/test_local_path_traversal.py 22874 +test_peak_memory_usage/test.py 22831 +test_parameterized_view/test.py 22806 +test_replicated_database_alter_modify_order_by/test.py 22759 +test_truncate_database/test_replicated.py 22757 +test_recovery_time_metric/test.py 22730 +test_deduplicated_attached_part_rename/test.py 22709 +test_azure_disk_unreachable/test.py 22639 +test_storage_iceberg_with_spark/test_writes_complex_type.py 22587 +test_storage_iceberg_schema_evolution/test_full_drop.py 22503 +test_replicated_merge_tree_thread_schedule_timeouts/test.py 22395 +test_replicated_database_with_auxiliary_zookeepers/test.py 22381 +test_sql_user_defined_functions_on_cluster/test.py 22354 +test_format_schema_on_server/test.py 22253 +test_s3_low_cardinality_right_border/test.py 22242 +test_s3_storage_conf_new_proxy/test.py 22194 +test_aliases_in_default_expr_not_break_table_structure/test.py 22157 +test_ldap_follow_referrals/test.py 22117 +test_parallel_replicas_local_replica_forced_inactive/test.py 22091 +test_limit_by_transform_kill_query/test.py 22062 +test_prometheus_protocols/test_write_read.py 21808 +test_storage_dict/test.py 21750 +test_storage_iceberg_with_spark_cache/test_filesystem_cache.py 21745 +test_prefer_global_in_and_join/test.py 21730 +test_point_in_polygon_cache_size/test.py 21631 +test_executable_user_defined_function/test_system_table.py 21618 +test_executable_user_defined_function_lifetime_reload/test.py 21567 +test_shard_level_const_function/test.py 21497 +test_replicated_database_system_clusters_log_level/test.py 21487 +test_merge_tree_empty_parts/test.py 21462 +test_prometheus_before_tables/test.py 21457 +test_sql_roles_for_xml_users/test.py 21435 +test_storage_iceberg_no_spark/test_read_in_order_with_pyiceberg.py 21424 +test_http_limits/test_hard_limit.py 21360 +test_storage_iceberg_with_spark/test_multiple_iceberg_file.py 21313 +test_storage_delta/test_imds.py 21280 +test_parallel_replicas_cluster_shadows_replicated_db/test.py 21279 +test_create_query_constraints/test.py 21278 +test_restart_with_unavailable_azure/test.py 21225 +test_zookeeper_session_on_config_reload/test.py 21185 +test_auth_method_grants_disabled_method/test.py 21148 +test_attach_table_from_s3_plain_readonly/test.py 21143 +test_storage_iceberg_with_spark/test_geometry_types.py 21122 +test_index_uncompressed_cache_zero_size/test.py 21074 +test_replicated_merge_tree_replicated_db_ttl/test.py 21066 +test_runtime_configurable_cache_size/test.py 21036 +test_keeper_client_config/test.py 21030 +test_oom_canary/test.py 20906 +test_system_zookeeper_watches/test.py 20845 +test_storage_iceberg_no_spark/test_writes_multiple_files.py 20779 +test_s3_imds/test_simple.py 20755 +test_storage_policies/test.py 20684 +test_keeper_watches/test.py 20571 +test_backward_compatibility/test_nullable_sparse_compatibility.py 20497 +test_system_reconnect_zookeeper/test.py 20472 +test_replicated_detach_table/test.py 20391 +test_keeper_client/test.py 20338 +test_storage_delta/test_azure_cluster.py 20277 +test_keeper_read_during_close/test.py 20111 +test_attach_table_normalizer/test.py 20019 +test_async_connect_to_multiple_ips/test.py 19994 +test_introspection_port/test.py 19974 +test_union_header/test.py 19872 +test_webterminal/test.py 19854 +test_mutations_analyzer_override/test.py 19817 +test_storage_iceberg_no_spark/test_writes_multiple_threads.py 19791 +test_keeper_catchup_response_queue/test.py 19770 +test_backward_compatibility/test.py 19738 +test_insert_deduplication_version_guard/test.py 19733 +test_jbod_load_balancing/test.py 19693 +test_os_thread_nice_value/test.py 19592 +test_backward_compatibility/test_rocksdb_upgrade.py 19590 +test_geojson_format/test.py 19578 +test_replica_can_become_leader/test.py 19558 +test_allowed_client_hosts/test.py 19542 +test_create_dictionary_in_startup_script/test.py 19534 +test_backward_compatibility/test_aggregate_fixed_key.py 19496 +test_insert_distributed_async_extra_dirs/test.py 19382 +test_replicated_engine_arguments/test.py 19361 +test_role/test_replicated_ddl_current_roles.py 19352 +test_replicated_merge_tree_config/test.py 19006 +test_thread_pool_free_size_shutdown/test.py 18983 +test_distributed_async_insert_batch_recovery/test.py 18960 +test_limit_materialized_view_count/test.py 18946 +test_config_reloader_interval/test.py 18871 +test_distributed_over_distributed/test.py 18752 +test_distributed_broken_files_stat/test.py 18738 +test_storage_iceberg_with_spark/test_types.py 18711 +test_enable_user_name_access_type/test.py 18613 +test_temporary_data/test.py 18556 +test_system_logs_comment/test.py 18554 +test_dictionary_allow_read_expired_keys/test_default_string.py 18546 +test_merge_table_over_distributed/test.py 18530 +test_backward_compatibility/test_memory_bound_aggregation.py 18480 +test_japanese_tokenizer/test.py 18433 +test_plain_rewr_legacy_layout/test.py 18405 +test_alter_update_cast_keep_nullable/test.py 18278 +test_remote_storage_engine_attach/test.py 18183 +test_storage_iceberg_no_spark/test_writes_with_compression_metadata.py 18106 +test_storage_s3_queue/test_file_iterator_lost_lock.py 18092 +test_config_decryption/test_zk_secure.py 18054 +test_check_table_name_length/test.py 17999 +test_reader_executor_page_cache/test.py 17933 +test_scram_sha256_password_with_replicated_zookeeper_replicator/test.py 17896 +test_prometheus_protocols/test_prometheus_query_log.py 17887 +test_scheduler_cached_disk/test.py 17821 +test_timezone_config/test.py 17666 +test_storage_iceberg_no_spark/test_expire_snapshots_history.py 17664 +test_prometheus_protocols/test_async_insert.py 17603 +test_reload_max_table_size_to_drop/test.py 17588 +test_async_logger_metrics/test.py 17567 +test_storage_gcp_auth/test.py 17549 +test_attach_backup_from_s3_plain/test.py 17498 +test_replicated_parse_zk_metadata/test.py 17468 +test_storage_iceberg_with_spark/test_restart_broken_s3.py 17331 +test_fetch_memory_usage/test.py 17268 +test_tcp_hello_string_limits/test.py 17211 +test_drop_no_local_path/test.py 17087 +test_metdata_cache_memory_leak/test.py 17002 +test_codec_encrypted/test.py 16996 +test_arrowflight_interface/test_prepared_statement_ttl.py 16983 +test_git_import/test.py 16980 +test_multiple_authentication_methods/test.py 16978 +test_config_decryption/test_zk.py 16855 +test_cluster_all_replicas/test.py 16828 +test_max_temporary_data_size_on_disk/test.py 16775 +test_keeper_restore_from_snapshot/test.py 16754 +test_keeper_availability_zone/test.py 16736 +test_backward_compatibility/test_const_node_optimization.py 16732 +test_disks_app_other_disk_types/test.py 16714 +test_aggregation_memory_efficient/test.py 16699 +test_trace_log_build_id/test.py 16635 +test_log_levels_update/test.py 16625 +test_prometheus_protocols/test_insert_select.py 16514 +test_refreshable_mv_keeper_loss/test.py 16494 +test_password_constraints/test.py 16488 +test_replicating_constants/test.py 16480 +test_failed_async_inserts/test.py 16470 +test_config_not_overriding_args/test.py 16458 +test_keeper_java_client/test.py 16453 +test_storage_iceberg_with_spark/test_partition_pruning_with_subquery_set.py 16398 +test_azure_workload_identity/test.py 16316 +test_distributed_system_query/test.py 16292 +test_storage_s3_intelligent_tier/test.py 16262 +test_prometheus_protocols/test_query_cache.py 16231 +test_kerberos_auth/test.py 16199 +test_keeper_slow_connection_log/test.py 16153 +test_keeper_sanitizer_logs/test.py 16139 +test_storage_delta/test_sts.py 16112 +test_shutdown_static_destructor_failure/test.py 16067 +test_storage_iceberg_with_spark/test_manifest_data_path_security.py 16012 +test_keeper_ipv4_fallback/test.py 15921 +test_block_structure_mismatch/test.py 15818 +test_prometheus_protocols/test_query_api.py 15818 +test_composable_protocols/test.py 15751 +test_input_format_parallel_parsing_memory_tracking/test.py 15704 +test_reload_certificate/test.py 15699 +test_config_decryption/test_wrong_settings_zk.py 15671 +test_mysql_protocol/test_kill_query.py 15671 +test_postgresql_protocol/test_kill_query.py 15639 +test_user_ip_restrictions/test.py 15591 +test_read_temporary_tables_on_failure/test.py 15499 +test_s3_style_link/test.py 15398 +test_insert_query_profile_events/test.py 15393 +test_attach_without_checksums/test.py 15364 +test_build_sets_from_multiple_threads/test.py 15261 +test_user_zero_database_access/test_user_zero_database_access.py 15238 +test_storage_iceberg_with_spark/test_iceberg_history_summary.py 15230 +test_shard_names/test.py 15210 +test_http_connection_drain_before_reuse/test.py 15161 +test_projection_rebuild_with_required_columns/test.py 15142 +test_settings_constraints_config_profiles/test.py 15140 +test_dictionary_custom_settings/test.py 15062 +test_materialized_view_restart_server/test.py 15027 +test_storage_iceberg_with_spark/test_pruning_nullable_bug.py 14987 +test_spark_session_recovery/test.py 14940 +test_grpc_protocol_ssl/test.py 14924 +test_merge_tree_prewarm_cache/test.py 14900 +test_dot_in_user_name/test.py 14837 +test_dotnet_client/test.py 14777 +test_merge_tree_load_marks/test.py 14642 +test_jdbc_bridge_hang/test.py 14634 +test_storage_iceberg_no_spark/test_writes_with_snappy_compression_metadata.py 14556 +test_filesystem/test.py 14551 +test_arrowflight_interface/test_ticket_expiration.py 14530 +test_jemalloc_merge_tree_arenas/test.py 14526 +test_drop_data/test.py 14518 +test_zookeeper_info/test.py 14505 +test_config_xml_full/test.py 14471 +test_tcp_handler_connection_limits/test.py 14464 +test_cache_s3_object_truncation/test.py 14436 +test_concurrent_queries_for_all_users_restriction/test.py 14421 +test_ttl_to_disk_wrapped_by_cache/test.py 14416 +test_prometheus_protocols/test_labels_api.py 14397 +test_storage_iceberg_with_spark/test_partition_by.py 14366 +test_backward_compatibility/test_short_strings_aggregation.py 14303 +test_backward_compatibility/test_normalized_count_comparison.py 14172 +test_tlsv1_3/test.py 14164 +test_disabled_access_control_improvements/test_impersonate_user.py 14104 +test_storage_iceberg_with_spark/test_writes_field_partitioning.py 14072 +test_s3_with_https/test.py 14059 +test_backward_compatibility/test_select_aggregate_alias_column.py 14047 +test_storage_azure_blob_storage/test_check_after_upload.py 14034 +test_user_grants_from_config/test.py 13957 +test_http_header_limits/test.py 13952 +test_ssh_keys_authentication/test.py 13912 +test_enabling_access_management/test.py 13884 +test_prometheus_protocols/test_series_api.py 13867 +test_config_xml_yaml_mix/test.py 13844 +test_paimon_spark_smoke/test.py 13815 +test_users_config_include_from_reload/test.py 13782 +test_allow_plaintext_and_no_password/test.py 13715 +test_keeper_dynamic_log_level/test.py 13711 +test_distributed_plan_cancel/test.py 13706 +test_database_iceberg_seaweedfs_catalog/test.py 13681 +test_zookeeper_info_number_overflow/test.py 13660 +test_user_query_log_disabled/test.py 13650 +test_s3_imds/test_session_token.py 13627 +test_accept_invalid_certificate/test.py 13596 +test_system_reload_async_metrics/test.py 13583 +test_text_log_level/test.py 13553 +test_storage_url_http_headers/test.py 13516 +test_s3_storage_class_multipart/test.py 13500 +test_config_yaml_full/test.py 13437 +test_config_yaml_merge_keys/test.py 13360 +test_config_yaml_main/test.py 13356 +test_skip_local_missing_table/test.py 13349 +test_keeper_availability_zone_quorum_reads/test.py 13326 +test_executable_udf_driver_config_reload/test.py 13324 +test_custom_settings/test.py 13312 +test_keeper_four_word_command/test_allow_list.py 13300 +test_merge_tree_settings_constraints/test.py 13182 +test_naive_bayes_xml_dictionary/test.py 13142 +test_concurrent_queries_for_user_restriction/test.py 13117 +test_storage_url_last_modified/test.py 13070 +test_http_dictionary_named_collection/test.py 13012 +test_parquet_page_index/test.py 12926 +test_dictionaries_null_value/test.py 12925 +test_geoparquet/test.py 12921 +test_aggregating_in_order_transform_kill_query/test.py 12860 +test_keeper_path_acl/test.py 12839 +test_interserver_tables_status_auth/test.py 12729 +test_freeze_table/test.py 12640 +test_keeper_invalid_digest/test.py 12625 +test_system_grants_url_regexp/test.py 12540 +test_inherit_multiple_profiles/test.py 12447 +test_parallel_replicas_skip_shards/test.py 12440 +test_config_xml_main/test.py 12427 +test_structured_logging_json/test.py 12342 +test_reload_query_masking_rules/test.py 12341 +test_buffer_profile/test.py 12321 +test_system_users_predicate_pushdown/test.py 12302 +test_config_hide_in_preprocessed/test.py 12261 +test_prometheus_protocols/test_http_port.py 12260 +test_allow_implicit_no_password/test.py 12240 +test_disk_name_virtual_column/test.py 12236 +test_keeper_profiler/test.py 12220 +test_remap_executable/test.py 12184 +test_keeper_nuraft_streaming/test.py 12176 +test_keeper_secure_client/test.py 12084 +test_backward_compatibility/test_insert_profile_events.py 12080 +test_s3_non_deterministic_partition_by/test.py 12072 +test_keeper_compression/test_without_compression.py 12046 +test_keeper_http_storage_control/test.py 12036 +test_keeper_and_access_storage/test.py 12034 +test_keeper_compression/test_with_compression.py 12032 +test_user_query_log_distributed_backend/test.py 12031 +test_config_decryption/test.py 12030 +test_memory_thread_stacks_metric/test.py 12022 +test_custom_dashboards/test.py 11948 +test_compatibility_readonly_constrained_setting/test.py 11940 +test_dictionaries_with_invalid_structure/test.py 11876 +test_thread_pool_queue_size/test.py 11815 +test_keeper_memory_soft_limit_ratio/test.py 11778 +test_storage_iceberg_with_spark/test_writes_multiple_threads.py 11763 +test_native_incorrect_data_deserialization/test.py 11745 +test_storage_iceberg_with_spark/test_single_iceberg_file.py 11735 +test_disk_types/test.py 11700 +test_remote_function_view/test.py 11688 +test_s3_redirect_remote_host_filter/test.py 11660 +test_s3_storage_class/test.py 11586 +test_passing_max_partitions_to_read_remotely/test.py 11578 +test_remote_prewhere/test.py 11540 +test_storage_iceberg_with_spark/test_minmax_pruning_for_arrays_and_maps_subfields_disabled.py 11512 +test_backup_s3_storage_class/test.py 11494 +test_async_metrics_in_cgroup/test.py 11489 +test_range_hashed_dictionary_types/test.py 11328 +test_settings_randomization/test.py 11319 +test_internal_queries_not_counted/test.py 11233 +test_play_image_preview/test.py 11218 +test_core_dump_size_limit/test.py 11177 +test_server_initialization/test.py 11148 +test_interserver_marker_requires_cluster_secret/test.py 11134 +test_endpoint_macro_substitution/test.py 11122 +test_server_keep_alive/test.py 11117 +test_arrowflight_session_log/test.py 10987 +test_profile_settings_and_constraints_order/test.py 10972 +test_relative_filepath/test.py 10884 +test_storage_url_with_proxy/test.py 10841 +test_composable_protocol_without_global_ssl/test.py 10828 +test_send_crash_reports/test.py 10805 +test_global_overcommit_tracker/test.py 10794 +test_arrowflight_interface/test_prepared_statement_limit.py 10791 +test_cancel_freeze/test.py 10784 +test_overcommit_tracker/test.py 10755 +test_play_chart_helpers/test.py 10724 +test_play_result_shaping/test.py 10712 +test_keeper_memory_soft_limit/test.py 10653 +test_storage_iceberg_with_spark/test_manifest_read_performance.py 10617 +test_trace_collector_serverwide/test.py 10550 +test_backward_compatibility/test_old_client_with_replicated_columns.py 10546 +test_filesystem_cache/test_size_limit_metric.py 10386 +test_delayed_remote_source/test.py 10349 +test_tcp_query_body_oversized_read/test.py 10344 +test_merge_tree_check_part_with_cache/test.py 10340 +test_ssh/test_options_propagation_enabled.py 10332 +test_jemalloc_global_profiler/test.py 10329 +test_format_cannot_allocate_thread/test.py 10315 +test_async_metrics_overload_warning/test.py 10293 +test_memory_limit/test.py 10289 +test_trace_log_memory_context/test.py 10279 +test_dirty_pages_force_purge/test.py 10270 +test_webterminal_startup/test.py 10253 +test_jemalloc_profiler_sampling_rate/test.py 10252 +test_memory_profiler_min_max_borders/test.py 10248 +test_storage_iceberg_with_spark/test_metadata_file_path_security.py 10234 +test_logs_level/test.py 10209 +test_render_log_file_name_templates/test.py 10196 +test_tcp_handler_interserver_listen_host/test_case.py 10150 +test_system_reload_async_metrics/test_async_metrics_invalid_settings.py 10136 +test_storage_iceberg_no_spark/test_iceberg_history_operation_summary.py 10104 +test_host_regexp_hosts_file_resolution/test.py 10093 +test_tcp_handler_http_responses/test_case.py 10042 +test_cgroup_metrics/test.py 9966 +test_http_auth_config_credentials/test.py 9941 +test_arrowflight_interface/test_prepared_statement_malformed_params.py 9912 +test_filesystem_cache_uninitialized/test.py 9896 +test_storage_iceberg_with_spark/test_delete_manifest_decode_concurrency.py 9887 +test_storage_iceberg_with_spark/test_relevant_iceberg_schema_chosen.py 9802 +test_userspace_page_cache/test_incorrect_limits.py 9750 +test_custom_http_handlers_per_protocol/test.py 9723 +test_validate_threadpool_writer_pool_size/test.py 9633 +test_storage_iceberg_no_spark/test_writes_nullable_bugs2.py 9506 +test_storage_iceberg_no_spark/test_time_travel_bug_fix_validation.py 9494 +test_storage_iceberg_no_spark/test_iceberg_history_missing_optional_summary_metrics.py 9375 +test_config_corresponding_root/test.py 9320 +test_storage_iceberg_with_spark/test_compressed_metadata.py 9138 +test_keeper_watch_profile_events/test.py 9124 +test_database_disk/test.py 8476 +test_storage_iceberg_with_spark/test_dates.py 8114 +test_storage_iceberg_with_spark/test_writes_drop_table.py 8078 +test_keeper_http_control_readiness/test.py 8039 +test_keeper_https_control_standalone_cluster/test.py 7326 +test_storage_iceberg_no_spark/test_writes_create_table_bugs.py 7102 +test_storage_iceberg_no_spark/test_graceful_error_not_configured_iceberg_metadata_log.py 6892 +test_storage_iceberg_with_spark/test_writes_create_version_hint.py 6862 +test_storage_iceberg_with_spark/test_cluster_table_function_with_partition_pruning.py 6658 +test_keeper_http_control_cli/test.py 6134 +test_keeper_request_total_with_subrequests/test.py 5929 +test_keeper_http_jemalloc/test.py 5764 +test_keeper_https_control_cli/test.py 5696 +test_cgroup_limit/test.py 5303 +test_storage_iceberg_with_spark/test_multiple_partitions_on_one_column.py 5044 +test_kafka_bad_messages/test_delete_topic_helper.py 5006 +test_storage_iceberg_with_spark/test_writes_different_path_format_error.py 4703 +test_disks_app_interactive/test.py 2422 +test_storage_iceberg_with_spark/test_local_table_safety.py 1382 +test_jemalloc_percpu_arena/test.py 1303 +test_cluster_waiters/test_rabbitmq_start_retry.py 677 +test_concurrent_backups_s3/test.py 1 +test_storage_rabbitmq/test_shutdown.py 1 diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index e6f80ff9ca018..6d5a0b2f418f0 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,15 +2,14 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54540) +SET(VERSION_REVISION 54560) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 3) -SET(VERSION_PATCH 33) -SET(VERSION_GITHASH b7f898857106bcc1f37ea1b805d53cbc95715bc1) -SET(VERSION_DESCRIBE v26.3.33.10001.altinitytest) -SET(VERSION_STRING 26.3.33.10001.altinitytest) +SET(VERSION_PATCH 53) +SET(VERSION_GITHASH fe8ff5d492ab7325e974b19a619b5c3ab3c095d8) +SET(VERSION_DESCRIBE v26.3.53.10001.altinitytest) +SET(VERSION_STRING 26.3.53.10001.altinitytest) # end of autochange SET(VERSION_TWEAK 10001) SET(VERSION_FLAVOUR altinitytest) - diff --git a/contrib/cctz b/contrib/cctz index c2ba12b73531a..fa5fe114a87a3 160000 --- a/contrib/cctz +++ b/contrib/cctz @@ -1 +1 @@ -Subproject commit c2ba12b73531a7dbc3ac45b5007649e35760f6d8 +Subproject commit fa5fe114a87a34c055853adf54dde305979acea0 diff --git a/contrib/croaring b/contrib/croaring index 025ae3f7add16..f2b1a55bcfa49 160000 --- a/contrib/croaring +++ b/contrib/croaring @@ -1 +1 @@ -Subproject commit 025ae3f7add169bc820dcfd46fa9304f382ec40a +Subproject commit f2b1a55bcfa49ec986539b05b603e26a3adcc95c diff --git a/contrib/croaring-cmake/CMakeLists.txt b/contrib/croaring-cmake/CMakeLists.txt index 1002b6a568d5a..56000e4d8b9d9 100644 --- a/contrib/croaring-cmake/CMakeLists.txt +++ b/contrib/croaring-cmake/CMakeLists.txt @@ -32,6 +32,13 @@ if (OS_FREEBSD) target_compile_definitions(_roaring PRIVATE bswap64=__bswap64) endif() +# clang-21 MemorySanitizer checks the value instead of the shadow of the index operand of AVX-512 +# `vpermi2var`/`vpermt2var` intrinsics (https://github.com/llvm/llvm-project/pull/148785, fixed in clang-22), +# which produces false positives in croaring's AVX-512 kernels (e.g. `avx512_emit_unique16`). +if (SANITIZE STREQUAL "memory") + target_compile_definitions(_roaring PUBLIC CROARING_COMPILER_SUPPORTS_AVX512=0) +endif() + target_include_directories(_roaring PRIVATE "${LIBRARY_DIR}/include/roaring") target_include_directories(_roaring SYSTEM BEFORE PUBLIC "${LIBRARY_DIR}/include") target_include_directories(_roaring SYSTEM BEFORE PUBLIC "${LIBRARY_DIR}/cpp") diff --git a/contrib/libfiu b/contrib/libfiu index 94f7114c4249c..74bc382193df8 160000 --- a/contrib/libfiu +++ b/contrib/libfiu @@ -1 +1 @@ -Subproject commit 94f7114c4249c8d115d57796c86a78a603d35ba0 +Subproject commit 74bc382193df83d829f9e5b99d7f17a841c5cf74 diff --git a/contrib/libllvmlibc-cmake/CMakeLists.txt b/contrib/libllvmlibc-cmake/CMakeLists.txt index e11d93d9aa852..c65511e889b8e 100644 --- a/contrib/libllvmlibc-cmake/CMakeLists.txt +++ b/contrib/libllvmlibc-cmake/CMakeLists.txt @@ -6,8 +6,6 @@ if (NOT (ARCH_AMD64 OR ARCH_AARCH64)) endif() set(SRCS - "${LIBLLVMLIBC_SOURCE_DIR}/src/math/generic/common_constants.cpp" - "${LIBLLVMLIBC_SOURCE_DIR}/src/math/generic/acos.cpp" "${LIBLLVMLIBC_SOURCE_DIR}/src/math/generic/asin.cpp" "${LIBLLVMLIBC_SOURCE_DIR}/src/math/generic/asinf.cpp" @@ -19,7 +17,6 @@ set(SRCS "${LIBLLVMLIBC_SOURCE_DIR}/src/math/generic/cos.cpp" "${LIBLLVMLIBC_SOURCE_DIR}/src/math/generic/exp.cpp" "${LIBLLVMLIBC_SOURCE_DIR}/src/math/generic/exp2.cpp" - "${LIBLLVMLIBC_SOURCE_DIR}/src/math/generic/explogxf.cpp" "${LIBLLVMLIBC_SOURCE_DIR}/src/math/generic/expm1.cpp" "${LIBLLVMLIBC_SOURCE_DIR}/src/math/generic/fabs.cpp" "${LIBLLVMLIBC_SOURCE_DIR}/src/math/generic/fabsl.cpp" diff --git a/contrib/librdkafka-cmake/CMakeLists.txt b/contrib/librdkafka-cmake/CMakeLists.txt index 877b76d9276d9..043f05204d6b6 100644 --- a/contrib/librdkafka-cmake/CMakeLists.txt +++ b/contrib/librdkafka-cmake/CMakeLists.txt @@ -32,7 +32,7 @@ set(SRCS "${RDKAFKA_SOURCE_DIR}/rdkafka.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_cert.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_cgrp.c" - "${RDKAFKA_SOURCE_DIR}/rdkafka_conf.c" +# "${RDKAFKA_SOURCE_DIR}/rdkafka_conf.c" # compiled via the rdkafka_conf_sensitive.c wrapper below "${RDKAFKA_SOURCE_DIR}/rdkafka_coord.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_error.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_event.c" @@ -84,6 +84,7 @@ set(SRCS "${RDKAFKA_SOURCE_DIR}/rdkafka_transport.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_txnmgr.c" "${RDKAFKA_SOURCE_DIR}/rdkafka_zstd.c" # WITH_ZSTD + "${CMAKE_CURRENT_SOURCE_DIR}/rdkafka_conf_sensitive.c" # ClickHouse wrapper compiling rdkafka_conf.c, see the comment in the file "${RDKAFKA_SOURCE_DIR}/rdlist.c" "${RDKAFKA_SOURCE_DIR}/rdlog.c" "${RDKAFKA_SOURCE_DIR}/rdmap.c" diff --git a/contrib/librdkafka-cmake/include/chrdkafka_conf_sensitive.h b/contrib/librdkafka-cmake/include/chrdkafka_conf_sensitive.h new file mode 100644 index 0000000000000..83b99dce955ca --- /dev/null +++ b/contrib/librdkafka-cmake/include/chrdkafka_conf_sensitive.h @@ -0,0 +1,14 @@ +/// See contrib/librdkafka-cmake/rdkafka_conf_sensitive.c +#pragma once + +#ifdef __cplusplus +extern "C" { +#endif + +/// The names of the configuration properties that librdkafka marks with the _RK_SENSITIVE flag, +/// i.e. whose values must not appear in logs, as a NULL-terminated array of static strings. +const char * const * chrd_kafka_conf_sensitive_properties(void); + +#ifdef __cplusplus +} +#endif diff --git a/contrib/librdkafka-cmake/rdkafka_conf_sensitive.c b/contrib/librdkafka-cmake/rdkafka_conf_sensitive.c new file mode 100644 index 0000000000000..740f444ad0a17 --- /dev/null +++ b/contrib/librdkafka-cmake/rdkafka_conf_sensitive.c @@ -0,0 +1,31 @@ +/// Wrapper around librdkafka's rdkafka_conf.c: compiles the file (this is the only place it is +/// compiled, see the SRCS list in contrib/librdkafka-cmake/CMakeLists.txt) and adds an accessor +/// for the static rd_kafka_properties table, which is the source of truth for the _RK_SENSITIVE +/// flag. ClickHouse uses the flag to hide the values of sensitive properties (e.g. sasl.password) +/// in its logs. No public librdkafka API exposes the flag: the redacting dump +/// (rd_kafka_anyconf_dump with redact_sensitive) is static, and the public rd_kafka_conf_dump +/// does not redact. + +#include "rdkafka_conf.c" + +#include + +/* The table ends with a terminator entry, so the count is an upper bound and the + * array is always NULL-terminated. The names point into the static table. */ +static const char * chrd_sensitive_names[sizeof(rd_kafka_properties) / sizeof(*rd_kafka_properties)]; +static pthread_once_t chrd_sensitive_names_once = PTHREAD_ONCE_INIT; + +static void chrd_fill_sensitive_names(void) +{ + const struct rd_kafka_property * prop = NULL; + size_t n = 0; + for (prop = rd_kafka_properties; prop->name; prop++) + if (prop->scope & _RK_SENSITIVE) + chrd_sensitive_names[n++] = prop->name; +} + +const char * const * chrd_kafka_conf_sensitive_properties(void) +{ + pthread_once(&chrd_sensitive_names_once, chrd_fill_sensitive_names); + return chrd_sensitive_names; +} diff --git a/contrib/libssh b/contrib/libssh index 50313883f3a07..c0083428b0c7f 160000 --- a/contrib/libssh +++ b/contrib/libssh @@ -1 +1 @@ -Subproject commit 50313883f3a077458cde4ea95bf46bfeb0771b34 +Subproject commit c0083428b0c7f97f7b6a9c212d630c49b72a2d7a diff --git a/contrib/libssh-cmake/CMakeLists.txt b/contrib/libssh-cmake/CMakeLists.txt index e6d58878f8408..ed1ecdcef0764 100644 --- a/contrib/libssh-cmake/CMakeLists.txt +++ b/contrib/libssh-cmake/CMakeLists.txt @@ -8,7 +8,7 @@ endif() # CMake variables needed by libssh_version.h.cmake, update them when you update libssh set(libssh_VERSION_MAJOR 0) set(libssh_VERSION_MINOR 12) -set(libssh_VERSION_PATCH 0) +set(libssh_VERSION_PATCH 2) set(LIB_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libssh") set(LIB_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/libssh") diff --git a/contrib/libxml2 b/contrib/libxml2 index c94eb0210183b..96498992efa48 160000 --- a/contrib/libxml2 +++ b/contrib/libxml2 @@ -1 +1 @@ -Subproject commit c94eb0210183b9d7cb43f8e7fddc6be55843ef49 +Subproject commit 96498992efa48d52b0e8b83058bd88dbdaf153c1 diff --git a/contrib/libxml2-cmake/linux_x86_64/include/config.h b/contrib/libxml2-cmake/linux_x86_64/include/config.h index 8abacf912b825..5aa68ac31c8d3 100644 --- a/contrib/libxml2-cmake/linux_x86_64/include/config.h +++ b/contrib/libxml2-cmake/linux_x86_64/include/config.h @@ -77,7 +77,7 @@ #define PACKAGE_NAME "libxml2" /* Define to the full name and version of this package. */ -#define PACKAGE_STRING "libxml2 2.15.3" +#define PACKAGE_STRING "libxml2 2.15.4" /* Define to the one symbol short name of this package. */ #define PACKAGE_TARNAME "libxml2" @@ -86,7 +86,7 @@ #define PACKAGE_URL "" /* Define to the version of this package. */ -#define PACKAGE_VERSION "2.15.3" +#define PACKAGE_VERSION "2.15.4" /* Define to 1 if all of the C90 standard headers exist (not just the ones required in a freestanding environment). This macro is provided for @@ -94,7 +94,7 @@ #define STDC_HEADERS 1 /* Version number of package */ -#define VERSION "2.15.3" +#define VERSION "2.15.4" /* System configuration directory (/etc) */ #define XML_SYSCONFDIR "/usr/local/etc" diff --git a/contrib/libxml2-cmake/linux_x86_64/include/libxml/xmlversion.h b/contrib/libxml2-cmake/linux_x86_64/include/libxml/xmlversion.h index e63ad461ce922..6f6a06e8830cf 100644 --- a/contrib/libxml2-cmake/linux_x86_64/include/libxml/xmlversion.h +++ b/contrib/libxml2-cmake/linux_x86_64/include/libxml/xmlversion.h @@ -16,17 +16,17 @@ /** * the version string like "1.2.3" */ -#define LIBXML_DOTTED_VERSION "2.15.3" +#define LIBXML_DOTTED_VERSION "2.15.4" /** * the version number: 1.2.3 value is 10203 */ -#define LIBXML_VERSION 21503 +#define LIBXML_VERSION 21504 /** * the version number string, 1.2.3 value is "10203" */ -#define LIBXML_VERSION_STRING "21503" +#define LIBXML_VERSION_STRING "21504" /** * extra version information, used to show a git commit description @@ -37,7 +37,7 @@ * Macro to check that the libxml version in use is compatible with * the version the software has been compiled against */ -#define LIBXML_TEST_VERSION xmlCheckVersion(21503); +#define LIBXML_TEST_VERSION xmlCheckVersion(21504); #if 1 /** diff --git a/contrib/llvm-project b/contrib/llvm-project index e8e0c06c2dd39..1e8c7547d529e 160000 --- a/contrib/llvm-project +++ b/contrib/llvm-project @@ -1 +1 @@ -Subproject commit e8e0c06c2dd39eab6227839c29c97e643f558474 +Subproject commit 1e8c7547d529e162040b73827333f4ce501f4d36 diff --git a/contrib/llvm-project-cmake/CMakeLists.txt b/contrib/llvm-project-cmake/CMakeLists.txt index 5a2c28daafed7..a8ae44a5dbedf 100644 --- a/contrib/llvm-project-cmake/CMakeLists.txt +++ b/contrib/llvm-project-cmake/CMakeLists.txt @@ -19,7 +19,7 @@ if (NOT ENABLE_EMBEDDED_COMPILER AND NOT ENABLE_DWARF_PARSER AND NOT ENABLE_BLAK return() endif() -set (LLVM_VERSION "21.1.5bundled") +set (LLVM_VERSION "22.1.1bundled") set (LLVM_INCLUDE_DIRS "${ClickHouse_SOURCE_DIR}/contrib/llvm-project/llvm/include" "${ClickHouse_BINARY_DIR}/contrib/llvm-project/llvm/include" diff --git a/contrib/mariadb-connector-c b/contrib/mariadb-connector-c index 95d6264bb43d3..b3c09ab333c2e 160000 --- a/contrib/mariadb-connector-c +++ b/contrib/mariadb-connector-c @@ -1 +1 @@ -Subproject commit 95d6264bb43d31f825cf2f85ce1928557457b62d +Subproject commit b3c09ab333c2ef9277291e721871a2bafca6e8bf diff --git a/contrib/mariadb-connector-c-cmake/CMakeLists.txt b/contrib/mariadb-connector-c-cmake/CMakeLists.txt index 93c8469b41b5a..1f0e9a8695b20 100644 --- a/contrib/mariadb-connector-c-cmake/CMakeLists.txt +++ b/contrib/mariadb-connector-c-cmake/CMakeLists.txt @@ -44,8 +44,8 @@ MATH(EXPR MARIADB_VERSION_ID "${MARIADB_CLIENT_VERSION_MAJOR} * 10000 + ${MARIADB_CLIENT_VERSION_PATCH}") set(CPACK_PACKAGE_VERSION_MAJOR 3) -set(CPACK_PACKAGE_VERSION_MINOR 1) -set(CPACK_PACKAGE_VERSION_PATCH 29) +set(CPACK_PACKAGE_VERSION_MINOR 4) +set(CPACK_PACKAGE_VERSION_PATCH 10) set(CPACK_PACKAGE_VERSION "${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}") MATH(EXPR MARIADB_PACKAGE_VERSION_ID "${CPACK_PACKAGE_VERSION_MAJOR} * 10000 + ${CPACK_PACKAGE_VERSION_MINOR} * 100 + @@ -191,10 +191,14 @@ REGISTER_PLUGIN(TARGET mysql_native_password TYPE MARIADB_CLIENT_PLUGIN_AUTH SOURCES "${CC_SOURCE_DIR}/plugins/auth/my_auth.c") -REGISTER_PLUGIN(TARGET aurora - TYPE MARIADB_CLIENT_PLUGIN_CONNECTION - SOURCES "${CC_SOURCE_DIR}/plugins/connection/aurora.c") - +# zlib protocol compression (CLIENT_COMPRESS). Since connector 3.4 the zlib +# compression moved behind a client compression plugin, so it must be +# registered as a static builtin before ma_client_plugin.c is generated; +# otherwise compression_plugin(net) stays null and mysql_real_connect fails +# with CR_ERR_LOAD_PLUGIN when MYSQL_OPT_COMPRESS is requested. +REGISTER_PLUGIN(TARGET zlib + TYPE MARIADB_CLIENT_PLUGIN_COMPRESSION + SOURCES "${CC_SOURCE_DIR}/plugins/compress/c_zlib.c") add_definitions(-D HAVE_COMPRESS) add_definitions(-D LIBMARIADB) @@ -230,7 +234,7 @@ ${CC_SOURCE_DIR}/libmariadb/ma_compress.c ${CC_SOURCE_DIR}/libmariadb/ma_init.c ${CC_SOURCE_DIR}/libmariadb/ma_password.c ${CC_SOURCE_DIR}/libmariadb/ma_ll2str.c -${CC_SOURCE_DIR}/libmariadb/ma_sha1.c +${CC_SOURCE_DIR}/libmariadb/ma_decimal.c ${CC_SOURCE_DIR}/libmariadb/mariadb_stmt.c ${CC_SOURCE_DIR}/libmariadb/ma_loaddata.c ${CC_SOURCE_DIR}/libmariadb/ma_stmt_codec.c diff --git a/contrib/mongo-c-driver b/contrib/mongo-c-driver index a26c8e251fec1..70de9e880072b 160000 --- a/contrib/mongo-c-driver +++ b/contrib/mongo-c-driver @@ -1 +1 @@ -Subproject commit a26c8e251fec147258e89b7e85e9adfa259fc42f +Subproject commit 70de9e880072b120ccbb48c77de334775747d7a9 diff --git a/contrib/mongo-c-driver-cmake/CMakeLists.txt b/contrib/mongo-c-driver-cmake/CMakeLists.txt index fa856376c93f3..7a270790f1409 100644 --- a/contrib/mongo-c-driver-cmake/CMakeLists.txt +++ b/contrib/mongo-c-driver-cmake/CMakeLists.txt @@ -5,13 +5,13 @@ if(NOT USE_MONGODB) endif() set(libbson_VERSION_MAJOR 2) -set(libbson_VERSION_MINOR 3) -set(libbson_VERSION_PATCH 0) -set(libbson_VERSION 2.3.0) +set(libbson_VERSION_MINOR 5) +set(libbson_VERSION_PATCH 5) +set(libbson_VERSION 2.5.5) set(libmongoc_VERSION_MAJOR 2) -set(libmongoc_VERSION_MINOR 3) -set(libmongoc_VERSION_PATCH 0) -set(libmongoc_VERSION 2.3.0) +set(libmongoc_VERSION_MINOR 5) +set(libmongoc_VERSION_PATCH 5) +set(libmongoc_VERSION 2.5.5) set(LIBBSON_SOURCES_ROOT "${ClickHouse_SOURCE_DIR}/contrib/mongo-c-driver/src") set(LIBBSON_SOURCE_DIR "${LIBBSON_SOURCES_ROOT}/libbson/src") @@ -132,7 +132,7 @@ endif() set(LIBMONGOC_SOURCE_DIR "${LIBBSON_SOURCES_ROOT}/libmongoc/src") set(LIBMONGOC_BINARY_DIR "${LIBBSON_BINARY_ROOT}/libmongoc/src") file(GLOB_RECURSE LIBMONGOC_SOURCES "${LIBMONGOC_SOURCE_DIR}/*.c") -set(UTF8PROC_SOURCE_DIR "${LIBBSON_SOURCES_ROOT}/utf8proc-2.8.0") +set(UTF8PROC_SOURCE_DIR "${LIBBSON_SOURCES_ROOT}/utf8proc-2.11.3") set(UTF8PROC_SOURCES "${UTF8PROC_SOURCE_DIR}/utf8proc.c") set(UTHASH_SOURCE_DIR "${LIBBSON_SOURCES_ROOT}/uthash") diff --git a/contrib/mongo-cxx-driver b/contrib/mongo-cxx-driver index 4f5273939b5cd..fa9ee6ed008e8 160000 --- a/contrib/mongo-cxx-driver +++ b/contrib/mongo-cxx-driver @@ -1 +1 @@ -Subproject commit 4f5273939b5cde587b34719f7c26364e502c00f4 +Subproject commit fa9ee6ed008e84b9562a7a55d70681de8a6bc1c6 diff --git a/contrib/mongo-cxx-driver-cmake/CMakeLists.txt b/contrib/mongo-cxx-driver-cmake/CMakeLists.txt index 8c750e1082a40..4fcb6ce38c226 100644 --- a/contrib/mongo-cxx-driver-cmake/CMakeLists.txt +++ b/contrib/mongo-cxx-driver-cmake/CMakeLists.txt @@ -13,11 +13,22 @@ include(GenerateExportHeader) set(BSONCXX_SOURCES ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/private/itoa.cpp ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/private/version.cpp + ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v1/array/value.cpp + ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v1/array/view.cpp ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v1/config/config.cpp ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v1/config/export.cpp ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v1/config/version.cpp + ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v1/decimal128.cpp ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v1/detail/postlude.cpp ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v1/detail/prelude.cpp + ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v1/document/value.cpp + ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v1/document/view.cpp + ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v1/element/view.cpp + ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v1/exception.cpp + ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v1/oid.cpp + ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v1/types/id.cpp + ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v1/types/value.cpp + ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v1/types/view.cpp ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi/bsoncxx/array/element.cpp ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi/bsoncxx/array/value.cpp ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi/bsoncxx/array/view.cpp @@ -43,10 +54,17 @@ set(BSONCXX_SOURCES set(BSONCXX_POLY_USE_IMPLS ON) +# ClickHouse uses the `v_noabi` API, whose symbols are gated behind the "unstable ABI" switch since r4.4.0. +set(BSONCXX_ENABLE_UNSTABLE_ABI ON) + configure_file( ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v1/config/config.hpp.in ${BSONCXX_BINARY_DIR}/lib/bsoncxx/v1/config/config.hpp ) +configure_file( + ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi/bsoncxx/config/config.hpp.in + ${BSONCXX_BINARY_DIR}/lib/bsoncxx/v_noabi/bsoncxx/config/config.hpp +) configure_file( ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v1/config/version.hpp.in ${BSONCXX_BINARY_DIR}/lib/bsoncxx/v1/config/version.hpp @@ -56,6 +74,40 @@ configure_file( ${BSONCXX_BINARY_DIR}/lib/bsoncxx/private/config/config.hh ) +# Aggregate forward-declaration headers. Upstream generates these by globbing +# all `*-fwd.hpp` files and concatenating them into a single `fwd.hpp` umbrella. +file(GLOB_RECURSE bsoncxx_v_noabi_fwd_headers + RELATIVE "${BSONCXX_SOURCES_DIR}/include/bsoncxx/v_noabi" + "${BSONCXX_SOURCES_DIR}/include/bsoncxx/v_noabi/bsoncxx/*-fwd.hpp" +) +list(SORT bsoncxx_v_noabi_fwd_headers) +set(BSONCXX_V_NOABI_FWD_INCLUDES "") +foreach(header IN LISTS bsoncxx_v_noabi_fwd_headers) + string(APPEND BSONCXX_V_NOABI_FWD_INCLUDES "#include <${header}> // IWYU pragma: export\n") +endforeach() +string(STRIP "${BSONCXX_V_NOABI_FWD_INCLUDES}" BSONCXX_V_NOABI_FWD_INCLUDES) +configure_file( + ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v_noabi/bsoncxx/fwd.hpp.in + ${BSONCXX_BINARY_DIR}/lib/bsoncxx/v_noabi/bsoncxx/fwd.hpp + @ONLY +) + +file(GLOB_RECURSE bsoncxx_v1_fwd_headers + RELATIVE "${BSONCXX_SOURCES_DIR}/include" + "${BSONCXX_SOURCES_DIR}/include/bsoncxx/v1/*-fwd.hpp" +) +list(SORT bsoncxx_v1_fwd_headers) +set(BSONCXX_V1_FWD_INCLUDES "") +foreach(header IN LISTS bsoncxx_v1_fwd_headers) + string(APPEND BSONCXX_V1_FWD_INCLUDES "#include <${header}> // IWYU pragma: export\n") +endforeach() +string(STRIP "${BSONCXX_V1_FWD_INCLUDES}" BSONCXX_V1_FWD_INCLUDES) +configure_file( + ${BSONCXX_SOURCES_DIR}/lib/bsoncxx/v1/fwd.hpp.in + ${BSONCXX_BINARY_DIR}/lib/bsoncxx/v1/fwd.hpp + @ONLY +) + add_library(_bsoncxx ${BSONCXX_SOURCES}) add_library(ch_contrib::bsoncxx ALIAS _bsoncxx) target_include_directories(_bsoncxx SYSTEM PUBLIC @@ -99,15 +151,92 @@ generate_export_header(_bsoncxx set(MONGOCXX_SOURCES_DIR "${ClickHouse_SOURCE_DIR}/contrib/mongo-cxx-driver/src/mongocxx") set(MONGOCXX_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/mongo-cxx-driver/src/mongocxx") set(MONGOCXX_SOURCES - ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/private/bson.cpp - ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/private/conversions.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/private/mongoc.cpp - ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/private/numeric_casting.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/private/scoped_bson.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/aggregate_options.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/apm.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/auto_encryption_options.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/bulk_write.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/change_stream.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/client.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/client_bulk_write.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/client_encryption.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/client_session.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/collection.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/config/config.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/config/export.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/config/version.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/count_options.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/cursor.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/data_key_options.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/database.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/delete_many_options.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/delete_many_result.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/delete_one_options.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/delete_one_result.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/detail/postlude.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/detail/prelude.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/distinct_options.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/encrypt_options.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/estimated_document_count_options.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/events/command_failed.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/events/command_started.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/events/command_succeeded.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/events/server_closed.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/events/server_description.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/events/server_description_changed.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/events/server_heartbeat_failed.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/events/server_heartbeat_started.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/events/server_heartbeat_succeeded.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/events/server_opening.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/events/topology_closed.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/events/topology_description.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/events/topology_description_changed.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/events/topology_opening.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/exception.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/find_one_and_delete_options.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/find_one_and_replace_options.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/find_one_and_update_options.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/find_options.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/gridfs/bucket.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/gridfs/downloader.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/gridfs/upload_options.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/gridfs/upload_result.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/gridfs/uploader.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/hint.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/indexes.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/insert_many_options.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/insert_many_result.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/insert_one_options.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/insert_one_result.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/instance.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/logger.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/oidc_callback.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/oidc_callback_params.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/oidc_credential.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/pipeline.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/pool.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/range_options.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/read_concern.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/read_preference.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/replace_one_options.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/replace_one_result.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/return_document.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/rewrap_many_datakey_options.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/rewrap_many_datakey_result.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/search_indexes.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/server_api.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/server_error.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/string_options.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/text_options.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/tls.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/transaction_options.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/update_many_options.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/update_many_result.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/update_one_options.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/update_one_result.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/uri.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/write_concern.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/bulk_write.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/change_stream.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/client.cpp @@ -117,6 +246,7 @@ set(MONGOCXX_SOURCES ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/config/config.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/config/export.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/config/version.cpp + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/conversions.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/cursor.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/database.cpp ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/events/command_failed_event.cpp @@ -218,10 +348,17 @@ endif() set(BSONCXX_STATIC 1) set(MONGOCXX_STATIC 1) +# ClickHouse uses the `v_noabi` API, whose symbols are gated behind the "unstable ABI" switch since r4.4.0. +set(MONGOCXX_ENABLE_UNSTABLE_ABI ON) + configure_file( ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/config/config.hpp.in ${MONGOCXX_BINARY_DIR}/lib/mongocxx/v1/config/config.hpp ) +configure_file( + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/config/config.hpp.in + ${MONGOCXX_BINARY_DIR}/lib/mongocxx/v_noabi/mongocxx/config/config.hpp +) configure_file( ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/config/version.hpp.in ${MONGOCXX_BINARY_DIR}/lib/mongocxx/v1/config/version.hpp @@ -231,6 +368,39 @@ configure_file( ${MONGOCXX_BINARY_DIR}/lib/mongocxx/private/config/config.hh ) +# Aggregate forward-declaration headers, mirroring upstream's umbrella `fwd.hpp`. +file(GLOB_RECURSE mongocxx_v_noabi_fwd_headers + RELATIVE "${MONGOCXX_SOURCES_DIR}/include/mongocxx/v_noabi" + "${MONGOCXX_SOURCES_DIR}/include/mongocxx/v_noabi/mongocxx/*-fwd.hpp" +) +list(SORT mongocxx_v_noabi_fwd_headers) +set(MONGOCXX_V_NOABI_FWD_INCLUDES "") +foreach(header IN LISTS mongocxx_v_noabi_fwd_headers) + string(APPEND MONGOCXX_V_NOABI_FWD_INCLUDES "#include <${header}> // IWYU pragma: export\n") +endforeach() +string(STRIP "${MONGOCXX_V_NOABI_FWD_INCLUDES}" MONGOCXX_V_NOABI_FWD_INCLUDES) +configure_file( + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v_noabi/mongocxx/fwd.hpp.in + ${MONGOCXX_BINARY_DIR}/lib/mongocxx/v_noabi/mongocxx/fwd.hpp + @ONLY +) + +file(GLOB_RECURSE mongocxx_v1_fwd_headers + RELATIVE "${MONGOCXX_SOURCES_DIR}/include" + "${MONGOCXX_SOURCES_DIR}/include/mongocxx/v1/*-fwd.hpp" +) +list(SORT mongocxx_v1_fwd_headers) +set(MONGOCXX_V1_FWD_INCLUDES "") +foreach(header IN LISTS mongocxx_v1_fwd_headers) + string(APPEND MONGOCXX_V1_FWD_INCLUDES "#include <${header}> // IWYU pragma: export\n") +endforeach() +string(STRIP "${MONGOCXX_V1_FWD_INCLUDES}" MONGOCXX_V1_FWD_INCLUDES) +configure_file( + ${MONGOCXX_SOURCES_DIR}/lib/mongocxx/v1/fwd.hpp.in + ${MONGOCXX_BINARY_DIR}/lib/mongocxx/v1/fwd.hpp + @ONLY +) + add_library(_mongocxx ${MONGOCXX_SOURCES}) add_library(ch_contrib::mongocxx ALIAS _mongocxx) target_include_directories(_mongocxx SYSTEM PUBLIC diff --git a/contrib/nats-io b/contrib/nats-io index cf441828d30fd..b88704dfeb55e 160000 --- a/contrib/nats-io +++ b/contrib/nats-io @@ -1 +1 @@ -Subproject commit cf441828d30fdd5de12d9da319e88d2586fdeeba +Subproject commit b88704dfeb55e0f1dee165b9c1daccb90d5816d4 diff --git a/contrib/openssl b/contrib/openssl index f3b2f0273ed59..614167b874c61 160000 --- a/contrib/openssl +++ b/contrib/openssl @@ -1 +1 @@ -Subproject commit f3b2f0273ed59cef874db0d6943f8d7d1804bd31 +Subproject commit 614167b874c612973344b49f772079213b3606eb diff --git a/contrib/openssl-cmake/common/include/openssl/opensslv.h b/contrib/openssl-cmake/common/include/openssl/opensslv.h index 0349e7d239758..2ffbf47950ad3 100644 --- a/contrib/openssl-cmake/common/include/openssl/opensslv.h +++ b/contrib/openssl-cmake/common/include/openssl/opensslv.h @@ -29,7 +29,7 @@ extern "C" { */ # define OPENSSL_VERSION_MAJOR 3 # define OPENSSL_VERSION_MINOR 5 -# define OPENSSL_VERSION_PATCH 7 +# define OPENSSL_VERSION_PATCH 8 /* * Additional version information @@ -74,21 +74,21 @@ extern "C" { * longer variant with OPENSSL_VERSION_PRE_RELEASE_STR and * OPENSSL_VERSION_BUILD_METADATA_STR appended. */ -# define OPENSSL_VERSION_STR "3.5.7" -# define OPENSSL_FULL_VERSION_STR "3.5.7" +# define OPENSSL_VERSION_STR "3.5.8" +# define OPENSSL_FULL_VERSION_STR "3.5.8" /* * SECTION 3: ADDITIONAL METADATA * * These strings are defined separately to allow them to be parsable. */ -# define OPENSSL_RELEASE_DATE "9 Jun 2026" +# define OPENSSL_RELEASE_DATE "25 Aug 2026" /* * SECTION 4: BACKWARD COMPATIBILITY */ -# define OPENSSL_VERSION_TEXT "OpenSSL 3.5.7 9 Jun 2026" +# define OPENSSL_VERSION_TEXT "OpenSSL 3.5.8 25 Aug 2026" /* Synthesize OPENSSL_VERSION_NUMBER with the layout 0xMNN00PPSL */ # ifdef OPENSSL_VERSION_PRE_RELEASE diff --git a/contrib/openssl-cmake/common/include/openssl/ssl.h b/contrib/openssl-cmake/common/include/openssl/ssl.h index aeb28d2b55d53..cb18794cc7ace 100644 --- a/contrib/openssl-cmake/common/include/openssl/ssl.h +++ b/contrib/openssl-cmake/common/include/openssl/ssl.h @@ -2480,6 +2480,7 @@ __owur int SSL_get_conn_close_info(SSL *ssl, # define SSL_VALUE_STREAM_WRITE_BUF_SIZE 7 # define SSL_VALUE_STREAM_WRITE_BUF_USED 8 # define SSL_VALUE_STREAM_WRITE_BUF_AVAIL 9 +# define SSL_VALUE_QUIC_MAX_PENDING_CONNS 16 # define SSL_VALUE_EVENT_HANDLING_MODE_INHERIT 0 # define SSL_VALUE_EVENT_HANDLING_MODE_IMPLICIT 1 @@ -2727,8 +2728,18 @@ const CTLOG_STORE *SSL_CTX_get0_ctlog_store(const SSL_CTX *ctx); # define SSL_SECOP_OTHER_SIGALG (5 << 16) # define SSL_SECOP_OTHER_CERT (6 << 16) -/* Indicated operation refers to peer key or certificate */ +/* + * Unused values - these do nothing and are never set. + * They are retained because of API. They should + * be removed next major + */ # define SSL_SECOP_PEER 0x1000 +/* Peer EE key in certificate */ +# define SSL_SECOP_PEER_EE_KEY (SSL_SECOP_EE_KEY | SSL_SECOP_PEER) +/* Peer CA key in certificate */ +# define SSL_SECOP_PEER_CA_KEY (SSL_SECOP_CA_KEY | SSL_SECOP_PEER) +/* Peer CA digest algorithm in certificate */ +# define SSL_SECOP_PEER_CA_MD (SSL_SECOP_CA_MD | SSL_SECOP_PEER) /* Values for "op" parameter in security callback */ @@ -2767,12 +2778,6 @@ const CTLOG_STORE *SSL_CTX_get0_ctlog_store(const SSL_CTX *ctx); # define SSL_SECOP_CA_KEY (17 | SSL_SECOP_OTHER_CERT) /* CA digest algorithm in certificate */ # define SSL_SECOP_CA_MD (18 | SSL_SECOP_OTHER_CERT) -/* Peer EE key in certificate */ -# define SSL_SECOP_PEER_EE_KEY (SSL_SECOP_EE_KEY | SSL_SECOP_PEER) -/* Peer CA key in certificate */ -# define SSL_SECOP_PEER_CA_KEY (SSL_SECOP_CA_KEY | SSL_SECOP_PEER) -/* Peer CA digest algorithm in certificate */ -# define SSL_SECOP_PEER_CA_MD (SSL_SECOP_CA_MD | SSL_SECOP_PEER) void SSL_set_security_level(SSL *s, int level); __owur int SSL_get_security_level(const SSL *s); diff --git a/contrib/thrift b/contrib/thrift index 13d30d6a6cdfb..26bf3f0c3cade 160000 --- a/contrib/thrift +++ b/contrib/thrift @@ -1 +1 @@ -Subproject commit 13d30d6a6cdfbd44c07977b8b762e88d9d5eb6df +Subproject commit 26bf3f0c3cade6f2c9fac82d4962751466282c23 diff --git a/contrib/usearch b/contrib/usearch index 1ae009a6aa573..6bd13d4c69a1b 160000 --- a/contrib/usearch +++ b/contrib/usearch @@ -1 +1 @@ -Subproject commit 1ae009a6aa573629c8871ddb97dd6e1a9c03dd5f +Subproject commit 6bd13d4c69a1baef1f668e652c3ca26028970f12 diff --git a/contrib/vectorscan b/contrib/vectorscan index e6993b7003a19..4f863adbc2aa1 160000 --- a/contrib/vectorscan +++ b/contrib/vectorscan @@ -1 +1 @@ -Subproject commit e6993b7003a19806902fa69ac57dcc2df826b946 +Subproject commit 4f863adbc2aa1129d473483d1a70134da07b5074 diff --git a/contrib/vectorscan-cmake/CMakeLists.txt b/contrib/vectorscan-cmake/CMakeLists.txt index 2d8a05af15024..ab06d8a8ac601 100644 --- a/contrib/vectorscan-cmake/CMakeLists.txt +++ b/contrib/vectorscan-cmake/CMakeLists.txt @@ -37,8 +37,8 @@ set (SRCS "${LIBRARY_DIR}/src/fdr/fdr_confirm_compile.cpp" "${LIBRARY_DIR}/src/fdr/fdr_engine_description.cpp" "${LIBRARY_DIR}/src/fdr/flood_compile.cpp" - "${LIBRARY_DIR}/src/fdr/teddy.c" - "${LIBRARY_DIR}/src/fdr/teddy_avx2.c" + "${LIBRARY_DIR}/src/fdr/teddy.cpp" + "${LIBRARY_DIR}/src/fdr/teddy_fat.cpp" "${LIBRARY_DIR}/src/fdr/teddy_compile.cpp" "${LIBRARY_DIR}/src/fdr/teddy_engine_description.cpp" @@ -238,11 +238,20 @@ set (SRCS # weird reasons different constants in the output. # # Also, please use upstream versions of colm and ragel, the packages in Ubuntu 22.04 seem to produce wrong output on ARM. +# +# The generated scanners read the input through `const char *` but compare it against `unsigned char` tables (the .rl files declare +# `alphtype unsigned char`), so they must be compiled with `-funsigned-char`, as the original build system does, otherwise bytes >= 0x80 +# never match on platforms where `char` is signed. if (ARCH_AMD64) list (APPEND SRCS "${LIBRARY_DIR}/../vectorscan-cmake/rageled_files/amd64/Parser.cpp" "${LIBRARY_DIR}/../vectorscan-cmake/rageled_files/amd64/control_verbs.cpp" ) + set_source_files_properties( + "${LIBRARY_DIR}/../vectorscan-cmake/rageled_files/amd64/Parser.cpp" + "${LIBRARY_DIR}/../vectorscan-cmake/rageled_files/amd64/control_verbs.cpp" + COMPILE_FLAGS -funsigned-char + ) elseif (ARCH_AARCH64) list (APPEND SRCS "${LIBRARY_DIR}/../vectorscan-cmake/rageled_files/aarch64/Parser.cpp" @@ -251,7 +260,7 @@ elseif (ARCH_AARCH64) set_source_files_properties( "${LIBRARY_DIR}/../vectorscan-cmake/rageled_files/aarch64/Parser.cpp" "${LIBRARY_DIR}/../vectorscan-cmake/rageled_files/aarch64/control_verbs.cpp" - COMPILE_FLAGS -Wno-c++11-narrowing + COMPILE_FLAGS "-funsigned-char -Wno-c++11-narrowing" ) endif() diff --git a/contrib/vectorscan-cmake/common/hs_version.h b/contrib/vectorscan-cmake/common/hs_version.h index 3d266484095e4..6b69038d3f09c 100644 --- a/contrib/vectorscan-cmake/common/hs_version.h +++ b/contrib/vectorscan-cmake/common/hs_version.h @@ -32,12 +32,12 @@ /** * A version string to identify this release of Hyperscan. */ -#define HS_VERSION_STRING "5.4.11 2024-07-04" +#define HS_VERSION_STRING "5.4.13 2026-08-23" #define HS_VERSION_32BIT ((5 << 24) | (1 << 16) | (7 << 8) | 0) #define HS_MAJOR 5 #define HS_MINOR 4 -#define HS_PATCH 11 +#define HS_PATCH 13 #endif /* HS_VERSION_H_C6428FAF8E3713 */ diff --git a/contrib/vectorscan-cmake/rageled_files/aarch64/Parser.cpp b/contrib/vectorscan-cmake/rageled_files/aarch64/Parser.cpp index fd58ead412837..05494b959c23a 100644 --- a/contrib/vectorscan-cmake/rageled_files/aarch64/Parser.cpp +++ b/contrib/vectorscan-cmake/rageled_files/aarch64/Parser.cpp @@ -164,8 +164,8 @@ namespace ue2 { assert(child); ComponentSequence *seq = child.get(); - parent->addComponent(move(child)); - return seq; + parent->addComponent(std::move(child)); + return seq; // cppcheck-suppress returnDanglingLifetime } static @@ -176,7 +176,7 @@ namespace ue2 { assert(cc); cc->add(c); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } else { currentSeq->addComponent(getLiteralComponentClass(c, mode.caseless)); } @@ -191,7 +191,7 @@ namespace ue2 { assert(cc); cc->add(accum); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } else { if (accum > 255) { throw LocatedParseError(err_msg); @@ -225,7 +225,7 @@ namespace ue2 { static unichar readUtf8CodePoint2c(const char *s) { - auto *ts = (const u8 *)s; + auto *ts = reinterpret_cast(s); assert(ts[0] >= 0xc0 && ts[0] < 0xe0); assert(ts[1] >= 0x80 && ts[1] < 0xc0); unichar val = ts[0] & 0x1f; @@ -238,7 +238,7 @@ namespace ue2 { static unichar readUtf8CodePoint3c(const char *s) { - auto *ts = (const u8 *)s; + auto *ts = reinterpret_cast(s); assert(ts[0] >= 0xe0 && ts[0] < 0xf0); assert(ts[1] >= 0x80 && ts[1] < 0xc0); assert(ts[2] >= 0x80 && ts[2] < 0xc0); @@ -254,7 +254,7 @@ namespace ue2 { static unichar readUtf8CodePoint4c(const char *s) { - auto *ts = (const u8 *)s; + auto *ts = reinterpret_cast(s); assert(ts[0] >= 0xf0 && ts[0] < 0xf8); assert(ts[1] >= 0x80 && ts[1] < 0xc0); assert(ts[2] >= 0x80 && ts[2] < 0xc0); @@ -272,7 +272,7 @@ namespace ue2 { } -#line 1909 "Parser.rl" +#line 1916 "Parser.rl" @@ -352,18 +352,18 @@ static const short _regex_actions[] = { 289, 1, 290, 1, 291, 1, 292, 1, 293, 1, 294, 1, 295, 1, 296, 1, 297, 1, 298, 1, 299, 1, 300, 1, - 301, 1, 302, 1, 303, 1, 307, 1, - 308, 1, 309, 1, 310, 1, 311, 1, + 301, 1, 302, 1, 303, 1, 304, 1, + 305, 1, 309, 1, 310, 1, 311, 1, 312, 1, 313, 1, 314, 1, 315, 1, 316, 1, 317, 1, 318, 1, 319, 1, 320, 1, 321, 1, 322, 1, 323, 1, 324, 1, 325, 1, 326, 1, 327, 1, 328, 1, 329, 1, 330, 1, 331, 1, 332, 1, 333, 1, 334, 1, 335, 1, - 336, 1, 337, 1, 338, 1, 342, 1, - 343, 1, 344, 1, 345, 1, 346, 1, + 336, 1, 337, 1, 338, 1, 339, 1, + 340, 1, 344, 1, 345, 1, 346, 1, 347, 1, 348, 1, 349, 1, 350, 1, - 352, 1, 353, 1, 354, 1, 355, 1, + 351, 1, 352, 1, 354, 1, 355, 1, 356, 1, 357, 1, 358, 1, 359, 1, 360, 1, 361, 1, 362, 1, 363, 1, 364, 1, 365, 1, 366, 1, 367, 1, @@ -384,17 +384,18 @@ static const short _regex_actions[] = { 424, 1, 425, 1, 426, 1, 427, 1, 428, 1, 429, 1, 430, 1, 431, 1, 432, 1, 433, 1, 434, 1, 435, 1, - 436, 2, 3, 0, 2, 4, 5, 2, - 5, 1, 2, 9, 10, 2, 9, 238, - 2, 9, 239, 2, 9, 339, 2, 10, - 1, 2, 10, 340, 2, 10, 341, 2, - 11, 241, 2, 11, 351, 2, 12, 241, - 2, 12, 351, 2, 13, 241, 2, 13, - 351, 2, 14, 375, 2, 14, 376, 2, - 25, 0, 2, 25, 3, 2, 25, 6, - 2, 25, 14, 3, 25, 5, 306, 3, - 25, 10, 305, 3, 25, 14, 15, 4, - 25, 9, 304, 10, 0 + 436, 1, 437, 1, 438, 2, 3, 0, + 2, 4, 5, 2, 5, 1, 2, 9, + 10, 2, 9, 238, 2, 9, 239, 2, + 9, 341, 2, 10, 1, 2, 10, 342, + 2, 10, 343, 2, 11, 241, 2, 11, + 353, 2, 12, 241, 2, 12, 353, 2, + 13, 241, 2, 13, 353, 2, 14, 377, + 2, 14, 378, 2, 25, 0, 2, 25, + 3, 2, 25, 6, 2, 25, 14, 3, + 25, 5, 308, 3, 25, 10, 307, 3, + 25, 14, 15, 4, 25, 9, 306, 10, + 0 }; static const short _regex_key_offsets[] = { @@ -500,14 +501,14 @@ static const short _regex_actions[] = { 1790, 1795, 1804, 1806, 1807, 1809, 1814, 1817, 1819, 1821, 1822, 1824, 1834, 1840, 1841, 1846, 1850, 1858, 1860, 1869, 1873, 1874, 1875, 1879, - 1880, 1883, 1883, 1890, 1904, 1907, 1946, 1948, - 1950, 1952, 1954, 1955, 1955, 1956, 1957, 1964, - 1970, 1976, 1978, 1980, 1982, 1985, 1987, 1998, - 1999, 2001, 2003, 2005, 2016, 2017, 2019, 2021, - 2023, 2024, 0 + 1880, 1883, 1883, 1890, 1905, 1906, 1909, 1948, + 1950, 1952, 1954, 1956, 1957, 1957, 1958, 1959, + 1966, 1972, 1978, 1980, 1982, 1984, 1987, 1989, + 2000, 2001, 2003, 2005, 2007, 2018, 2019, 2021, + 2023, 2025, 2026, 0 }; - static const char _regex_trans_keys[] = { + static const unsigned char _regex_trans_keys[] = { 41u, 33u, 35u, 38u, 39u, 40u, 41u, 43u, 45u, 58u, 60u, 61u, 62u, 63u, 67u, 80u, 105u, 109u, 115u, 120u, 123u, 48u, 57u, 41u, @@ -744,24 +745,24 @@ static const short _regex_actions[] = { 107u, 109u, 111u, 117u, 121u, 97u, 101u, 104u, 105u, 103u, 97u, 97u, 112u, 115u, 119u, 105u, 108u, 112u, 115u, 67u, 76u, 77u, 78u, 80u, - 83u, 90u, 45u, 91u, 92u, 93u, 128u, 191u, - 192u, 223u, 224u, 239u, 240u, 247u, 248u, 255u, - 46u, 58u, 61u, 48u, 68u, 69u, 72u, 76u, - 78u, 80u, 81u, 83u, 85u, 86u, 87u, 97u, - 98u, 99u, 100u, 101u, 102u, 103u, 104u, 108u, - 110u, 111u, 112u, 114u, 115u, 116u, 117u, 118u, - 119u, 120u, 49u, 55u, 56u, 57u, 65u, 90u, - 105u, 122u, 48u, 55u, 48u, 55u, 48u, 55u, - 48u, 55u, 123u, 123u, 123u, 123u, 48u, 57u, + 83u, 90u, 38u, 45u, 91u, 92u, 93u, 128u, + 191u, 192u, 223u, 224u, 239u, 240u, 247u, 248u, + 255u, 38u, 46u, 58u, 61u, 48u, 68u, 69u, + 72u, 76u, 78u, 80u, 81u, 83u, 85u, 86u, + 87u, 97u, 98u, 99u, 100u, 101u, 102u, 103u, + 104u, 108u, 110u, 111u, 112u, 114u, 115u, 116u, + 117u, 118u, 119u, 120u, 49u, 55u, 56u, 57u, + 65u, 90u, 105u, 122u, 48u, 55u, 48u, 55u, + 48u, 55u, 48u, 55u, 123u, 123u, 123u, 123u, + 48u, 57u, 65u, 70u, 97u, 102u, 48u, 57u, 65u, 70u, 97u, 102u, 48u, 57u, 65u, 70u, - 97u, 102u, 48u, 57u, 65u, 70u, 97u, 102u, - 128u, 191u, 128u, 191u, 128u, 191u, 92u, 93u, - 94u, 69u, 81u, 92u, 128u, 191u, 192u, 223u, - 224u, 239u, 240u, 247u, 248u, 255u, 69u, 128u, - 191u, 128u, 191u, 128u, 191u, 92u, 128u, 191u, + 97u, 102u, 128u, 191u, 128u, 191u, 128u, 191u, + 92u, 93u, 94u, 69u, 81u, 92u, 128u, 191u, 192u, 223u, 224u, 239u, 240u, 247u, 248u, 255u, - 69u, 128u, 191u, 128u, 191u, 128u, 191u, 41u, - 10u, 0u + 69u, 128u, 191u, 128u, 191u, 128u, 191u, 92u, + 128u, 191u, 192u, 223u, 224u, 239u, 240u, 247u, + 248u, 255u, 69u, 128u, 191u, 128u, 191u, 128u, + 191u, 41u, 10u, 0u }; static const signed char _regex_single_lengths[] = { @@ -867,11 +868,11 @@ static const short _regex_actions[] = { 5, 9, 2, 1, 2, 5, 3, 2, 2, 1, 2, 10, 6, 1, 5, 4, 8, 2, 9, 4, 1, 1, 4, 1, - 3, 0, 7, 4, 3, 31, 0, 0, - 0, 0, 1, 0, 1, 1, 1, 0, - 0, 0, 0, 0, 3, 2, 1, 1, - 0, 0, 0, 1, 1, 0, 0, 0, - 1, 1, 0 + 3, 0, 7, 5, 1, 3, 31, 0, + 0, 0, 0, 1, 0, 1, 1, 1, + 0, 0, 0, 0, 0, 3, 2, 1, + 1, 0, 0, 0, 1, 1, 0, 0, + 0, 1, 1, 0 }; static const signed char _regex_range_lengths[] = { @@ -977,11 +978,11 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 5, 0, 4, 1, 1, - 1, 1, 0, 0, 0, 0, 3, 3, - 3, 1, 1, 1, 0, 0, 5, 0, - 1, 1, 1, 5, 0, 1, 1, 1, - 0, 0, 0 + 0, 0, 0, 5, 0, 0, 4, 1, + 1, 1, 1, 0, 0, 0, 0, 3, + 3, 3, 1, 1, 1, 0, 0, 5, + 0, 1, 1, 1, 5, 0, 1, 1, + 1, 0, 0, 0 }; static const short _regex_index_offsets[] = { @@ -1087,11 +1088,11 @@ static const short _regex_actions[] = { 2412, 2418, 2428, 2431, 2433, 2436, 2442, 2446, 2449, 2452, 2454, 2457, 2468, 2475, 2477, 2483, 2488, 2497, 2500, 2510, 2515, 2517, 2519, 2524, - 2526, 2530, 2531, 2539, 2549, 2553, 2589, 2591, - 2593, 2595, 2597, 2599, 2600, 2602, 2604, 2609, - 2613, 2617, 2619, 2621, 2623, 2627, 2630, 2637, - 2639, 2641, 2643, 2645, 2652, 2654, 2656, 2658, - 2660, 2662, 0 + 2526, 2530, 2531, 2539, 2550, 2552, 2556, 2592, + 2594, 2596, 2598, 2600, 2602, 2603, 2605, 2607, + 2612, 2616, 2620, 2622, 2624, 2626, 2630, 2633, + 2640, 2642, 2644, 2646, 2648, 2655, 2657, 2659, + 2661, 2663, 2665, 0 }; static const signed char _regex_trans_cond_spaces[] = { @@ -1412,8 +1413,8 @@ static const short _regex_actions[] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 0, - 0, 0, 0, 0, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, + 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, @@ -1422,12 +1423,12 @@ static const short _regex_actions[] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 0, -1, 0, -1, + 0, -1, -1, 2, 2, -1, -1, -1, + -1, -1, 0, 0, 0, 0, 0, -1, + -1, -1, 0, -1, 0, -1, 0, -1, + -1, 0, 0, 0, 0, 0, -1, -1, -1, 0, -1, 0, -1, 0, -1, -1, - 2, 2, -1, -1, -1, -1, -1, 0, - 0, 0, 0, 0, -1, -1, -1, 0, - -1, 0, -1, 0, -1, -1, 0, 0, - 0, 0, 0, -1, -1, -1, 0, -1, - 0, -1, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, @@ -1534,7 +1535,7 @@ static const short _regex_actions[] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 0 + -1, -1, -1, -1, -1, -1, 0 }; static const short _regex_trans_offsets[] = { @@ -1856,7 +1857,7 @@ static const short _regex_actions[] = { 2541, 2542, 2543, 2544, 2545, 2546, 2547, 2548, 2549, 2550, 2551, 2552, 2553, 2554, 2555, 2556, 2557, 2558, 2559, 2560, 2561, 2562, 2563, 2564, - 2566, 2568, 2570, 2572, 2574, 2575, 2576, 2577, + 2565, 2567, 2569, 2571, 2573, 2575, 2576, 2577, 2578, 2579, 2580, 2581, 2582, 2583, 2584, 2585, 2586, 2587, 2588, 2589, 2590, 2591, 2592, 2593, 2594, 2595, 2596, 2597, 2598, 2599, 2600, 2601, @@ -1865,12 +1866,12 @@ static const short _regex_actions[] = { 2618, 2619, 2620, 2621, 2622, 2623, 2624, 2625, 2626, 2627, 2628, 2629, 2630, 2631, 2632, 2633, 2634, 2635, 2636, 2637, 2638, 2639, 2640, 2641, - 2642, 2643, 2645, 2646, 2648, 2649, 2651, 2652, - 2653, 2655, 2657, 2658, 2659, 2660, 2661, 2662, - 2664, 2666, 2668, 2670, 2672, 2673, 2674, 2675, - 2677, 2678, 2680, 2681, 2683, 2684, 2685, 2687, - 2689, 2691, 2693, 2695, 2696, 2697, 2698, 2700, - 2701, 2703, 2704, 2706, 2707, 2708, 2709, 2710, + 2642, 2643, 2644, 2645, 2646, 2648, 2649, 2651, + 2652, 2654, 2655, 2656, 2658, 2660, 2661, 2662, + 2663, 2664, 2665, 2667, 2669, 2671, 2673, 2675, + 2676, 2677, 2678, 2680, 2681, 2683, 2684, 2686, + 2687, 2688, 2690, 2692, 2694, 2696, 2698, 2699, + 2700, 2701, 2703, 2704, 2706, 2707, 2709, 2710, 2711, 2712, 2713, 2714, 2715, 2716, 2717, 2718, 2719, 2720, 2721, 2722, 2723, 2724, 2725, 2726, 2727, 2728, 2729, 2730, 2731, 2732, 2733, 2734, @@ -1977,7 +1978,7 @@ static const short _regex_actions[] = { 3535, 3536, 3537, 3538, 3539, 3540, 3541, 3542, 3543, 3544, 3545, 3546, 3547, 3548, 3549, 3550, 3551, 3552, 3553, 3554, 3555, 3556, 3557, 3558, - 3559, 3560, 0 + 3559, 3560, 3561, 3562, 3563, 3564, 0 }; static const signed char _regex_trans_lengths[] = { @@ -2298,8 +2299,8 @@ static const short _regex_actions[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 2, - 2, 2, 2, 2, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -2308,12 +2309,12 @@ static const short _regex_actions[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 2, 1, 2, 1, + 2, 1, 1, 2, 2, 1, 1, 1, + 1, 1, 2, 2, 2, 2, 2, 1, + 1, 1, 2, 1, 2, 1, 2, 1, + 1, 2, 2, 2, 2, 2, 1, 1, 1, 2, 1, 2, 1, 2, 1, 1, - 2, 2, 1, 1, 1, 1, 1, 2, - 2, 2, 2, 2, 1, 1, 1, 2, - 1, 2, 1, 2, 1, 1, 2, 2, - 2, 2, 2, 1, 1, 1, 2, 1, - 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -2420,7 +2421,7 @@ static const short _regex_actions[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 0 + 1, 1, 1, 1, 1, 1, 0 }; static const signed char _regex_cond_keys[] = { @@ -2744,8 +2745,8 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 1, - 0, 1, 0, 1, 0, 1, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2754,15 +2755,15 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 1, 0, 0, 1, - 0, 0, 1, 0, 0, 0, 1, 0, - 1, 0, 0, 0, 0, 0, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 1, + 0, 0, 1, 0, 0, 1, 0, 0, + 0, 1, 0, 1, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 0, 0, 0, 1, + 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, - 0, 0, 0, 0, 1, 0, 0, 1, - 0, 0, 1, 0, 0, 0, 1, 0, - 1, 0, 1, 0, 1, 0, 1, 0, - 0, 0, 0, 1, 0, 0, 1, 0, - 0, 1, 0, 0, 0, 0, 0, 0, + 0, 1, 0, 0, 0, 0, 1, 0, + 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2869,7 +2870,7 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0 + 0, 0, 0, 0, 0, 0 }; static const short _regex_cond_targs[] = { @@ -3144,9 +3145,9 @@ static const short _regex_actions[] = { 732, 733, 819, 731, 732, 733, 731, 735, 819, 819, 735, 819, 819, 736, 736, 736, 819, 819, 819, 819, 819, 739, 819, 819, - 819, 819, 838, 838, 838, 838, 742, 838, - 838, 838, 838, 843, 843, 843, 843, 745, - 843, 843, 843, 843, 746, 746, 746, 746, + 819, 819, 839, 839, 839, 839, 742, 839, + 839, 839, 839, 844, 844, 844, 844, 745, + 844, 844, 844, 844, 746, 746, 746, 746, 746, 747, 746, 759, 760, 746, 761, 762, 763, 746, 782, 746, 746, 746, 746, 746, 784, 746, 785, 746, 786, 746, 746, 746, @@ -3193,26 +3194,26 @@ static const short _regex_actions[] = { 548, 790, 554, 790, 555, 556, 557, 558, 790, 790, 790, 790, 790, 790, 790, 0, 818, 818, 818, 818, 818, 818, 818, 818, - 819, 820, 821, 819, 819, 819, 819, 833, - 819, 834, 819, 835, 819, 819, 819, 563, - 566, 731, 819, 822, 819, 819, 819, 819, - 819, 826, 819, 819, 819, 819, 819, 819, - 819, 827, 819, 819, 819, 819, 819, 819, - 819, 828, 829, 819, 819, 819, 819, 819, - 819, 830, 824, 819, 819, 819, 819, 823, - 819, 819, 819, 825, 819, 819, 819, 819, - 819, 819, 734, 819, 819, 819, 832, 831, - 831, 831, 819, 819, 819, 819, 819, 736, - 736, 736, 819, 819, 819, 819, 819, 737, - 819, 819, 738, 819, 837, 836, 836, 836, - 836, 836, 836, 836, 836, 839, 838, 838, - 838, 840, 838, 841, 838, 842, 838, 838, - 838, 838, 838, 838, 838, 838, 838, 740, - 838, 838, 741, 838, 844, 843, 843, 843, - 845, 843, 846, 843, 847, 843, 843, 843, - 843, 843, 843, 843, 843, 843, 743, 843, - 843, 744, 843, 848, 848, 849, 849, 0, - 746, 746, 746, 746, 746, 746, 746, 746, + 820, 819, 821, 822, 819, 819, 819, 819, + 834, 819, 835, 819, 836, 819, 819, 819, + 819, 819, 563, 566, 731, 819, 823, 819, + 819, 819, 819, 819, 827, 819, 819, 819, + 819, 819, 819, 819, 828, 819, 819, 819, + 819, 819, 819, 819, 829, 830, 819, 819, + 819, 819, 819, 819, 831, 825, 819, 819, + 819, 819, 824, 819, 819, 819, 826, 819, + 819, 819, 819, 819, 819, 734, 819, 819, + 819, 833, 832, 832, 832, 819, 819, 819, + 819, 819, 736, 736, 736, 819, 819, 819, + 819, 819, 737, 819, 819, 738, 819, 838, + 837, 837, 837, 837, 837, 837, 837, 837, + 840, 839, 839, 839, 841, 839, 842, 839, + 843, 839, 839, 839, 839, 839, 839, 839, + 839, 839, 740, 839, 839, 741, 839, 845, + 844, 844, 844, 846, 844, 847, 844, 848, + 844, 844, 844, 844, 844, 844, 844, 844, + 844, 743, 844, 844, 744, 844, 849, 849, + 850, 850, 0, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, @@ -3220,7 +3221,8 @@ static const short _regex_actions[] = { 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, - 787, 787, 787, 787, 787, 787, 790, 790, + 746, 746, 746, 787, 787, 787, 787, 787, + 787, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, @@ -3281,8 +3283,7 @@ static const short _regex_actions[] = { 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, - 790, 790, 790, 790, 790, 790, 559, 560, - 561, 562, 819, 819, 819, 819, 819, 819, + 790, 559, 560, 561, 562, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, @@ -3304,62 +3305,62 @@ static const short _regex_actions[] = { 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, - 819, 819, 819, 838, 838, 838, 843, 843, - 843, 746, 746, 746, 746, 746, 746, 746, + 819, 819, 819, 819, 819, 819, 839, 839, + 839, 844, 844, 844, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, - 746, 746, 787, 787, 787, 790, 790, 790, + 746, 746, 746, 746, 746, 787, 787, 787, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, - 817, 818, 819, 819, 819, 819, 819, 819, + 790, 790, 790, 817, 818, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, - 819, 819, 819, 836, 836, 838, 838, 838, - 838, 838, 843, 843, 843, 843, 843, 848, - 849, 0 + 819, 819, 819, 819, 819, 819, 819, 837, + 837, 839, 839, 839, 839, 839, 844, 844, + 844, 844, 844, 849, 850, 0 }; static const short _regex_cond_actions[] = { - 827, 631, 731, 723, 45, 903, 903, 897, - 45, 912, 900, 903, 729, 741, 0, 45, - 45, 923, 923, 923, 923, 737, 45, 765, - 747, 0, 0, 0, 0, 841, 743, 3, + 831, 635, 735, 727, 45, 907, 907, 901, + 45, 916, 904, 907, 733, 745, 0, 45, + 45, 927, 927, 927, 927, 741, 45, 769, + 751, 0, 0, 0, 0, 845, 747, 3, + 3, 3, 3, 845, 3, 3, 3, 3, + 843, 0, 3, 3, 3, 3, 843, 765, + 843, 765, 3, 3, 3, 3, 843, 763, + 3, 874, 3, 3, 843, 765, 3, 874, + 3, 3, 843, 3, 3, 3, 3, 843, + 0, 3, 3, 3, 3, 843, 755, 0, + 753, 843, 759, 757, 843, 0, 761, 3, + 3, 3, 3, 843, 0, 0, 0, 0, + 843, 761, 0, 0, 0, 0, 843, 761, + 3, 3, 3, 3, 843, 749, 0, 845, + 729, 731, 27, 27, 27, 27, 845, 747, + 3, 3, 3, 3, 845, 767, 0, 845, + 3, 3, 3, 3, 845, 3, 3, 3, + 3, 845, 677, 3, 3, 3, 3, 845, + 0, 0, 0, 0, 845, 27, 27, 27, + 27, 845, 729, 0, 731, 25, 25, 25, + 25, 845, 0, 0, 833, 0, 0, 0, + 607, 0, 0, 0, 0, 0, 0, 833, + 0, 0, 0, 605, 0, 0, 0, 0, + 0, 0, 833, 0, 0, 0, 607, 0, + 0, 0, 0, 679, 0, 17, 835, 679, + 0, 7, 3, 874, 3, 3, 835, 17, + 835, 667, 17, 835, 667, 17, 835, 667, + 835, 3, 665, 874, 3, 3, 835, 3, + 665, 874, 3, 3, 835, 3, 665, 3, + 3, 3, 835, 3, 669, 3, 3, 3, + 835, 3, 3, 3, 3, 841, 675, 3, 3, 3, 3, 841, 3, 3, 3, 3, - 839, 0, 3, 3, 3, 3, 839, 761, - 839, 761, 3, 3, 3, 3, 839, 759, - 3, 870, 3, 3, 839, 761, 3, 870, - 3, 3, 839, 3, 3, 3, 3, 839, - 0, 3, 3, 3, 3, 839, 751, 0, - 749, 839, 755, 753, 839, 0, 757, 3, - 3, 3, 3, 839, 0, 0, 0, 0, - 839, 757, 0, 0, 0, 0, 839, 757, - 3, 3, 3, 3, 839, 745, 0, 841, - 725, 727, 27, 27, 27, 27, 841, 743, - 3, 3, 3, 3, 841, 763, 0, 841, - 3, 3, 3, 3, 841, 3, 3, 3, - 3, 841, 673, 3, 3, 3, 3, 841, - 0, 0, 0, 0, 841, 27, 27, 27, - 27, 841, 725, 0, 727, 25, 25, 25, - 25, 841, 0, 0, 829, 0, 0, 0, - 603, 0, 0, 0, 0, 0, 0, 829, - 0, 0, 0, 601, 0, 0, 0, 0, - 0, 0, 829, 0, 0, 0, 603, 0, - 0, 0, 0, 675, 0, 17, 831, 675, - 0, 7, 3, 870, 3, 3, 831, 17, - 831, 663, 17, 831, 663, 17, 831, 663, - 831, 3, 661, 870, 3, 3, 831, 3, - 661, 870, 3, 3, 831, 3, 661, 3, - 3, 3, 831, 3, 665, 3, 3, 3, - 831, 3, 3, 3, 3, 837, 671, 3, - 3, 3, 3, 837, 3, 3, 3, 3, - 837, 669, 3, 3, 3, 3, 837, 3, - 3, 3, 3, 837, 3, 667, 3, 3, - 3, 837, 0, 833, 677, 0, 833, 679, - 0, 0, 0, 835, 0, 29, 11, 845, - 31, 13, 845, 0, 13, 845, 843, 769, - 843, 843, 0, 843, 843, 771, 843, 53, + 841, 673, 3, 3, 3, 3, 841, 3, + 3, 3, 3, 841, 3, 671, 3, 3, + 3, 841, 0, 837, 681, 0, 837, 683, + 0, 0, 0, 839, 0, 29, 11, 849, + 31, 13, 849, 0, 13, 849, 847, 773, + 847, 847, 0, 847, 847, 775, 847, 53, 0, 53, 0, 0, 51, 0, 53, 0, 0, 49, 0, 0, 47, 0, 315, 359, 0, 0, 359, 0, 359, 0, 359, 127, @@ -3489,139 +3490,139 @@ static const short _regex_actions[] = { 359, 0, 359, 0, 359, 309, 359, 311, 359, 119, 359, 121, 359, 123, 359, 125, 359, 0, 39, 33, 35, 37, 39, 39, - 35, 0, 0, 539, 0, 0, 0, 377, - 0, 0, 0, 0, 0, 0, 539, 0, + 35, 0, 0, 543, 0, 0, 0, 377, + 0, 0, 0, 0, 0, 0, 543, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 539, 0, + 0, 0, 0, 0, 0, 0, 543, 0, 0, 0, 435, 0, 0, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 381, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 385, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 389, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 393, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 397, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 401, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 405, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 409, - 0, 0, 0, 539, 0, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 413, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 417, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 421, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 425, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 429, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 433, 0, 0, 0, 539, - 0, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 379, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 383, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 387, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 391, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 395, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 399, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 403, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 407, 0, - 0, 0, 539, 0, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 411, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 415, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 419, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 423, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 427, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 431, 0, 0, 0, 539, 0, + 0, 543, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 381, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 385, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 389, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 393, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 397, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 401, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 405, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 409, + 0, 0, 0, 543, 0, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 413, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 417, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 421, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 425, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 429, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 433, 0, 0, 0, 543, + 0, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 379, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 383, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 387, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 391, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 395, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 399, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 403, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 407, 0, + 0, 0, 543, 0, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 411, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 415, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 419, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 423, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 427, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 431, 0, 0, 0, 543, 0, 0, 0, 377, 0, 0, 0, 0, 0, - 533, 471, 0, 533, 475, 0, 0, 0, - 535, 537, 503, 537, 537, 0, 537, 537, - 505, 537, 569, 557, 569, 569, 0, 569, - 569, 559, 569, 587, 575, 587, 587, 0, - 587, 587, 577, 587, 633, 775, 777, 629, - 637, 45, 597, 0, 0, 609, 0, 45, - 0, 635, 909, 599, 775, 777, 773, 777, - 0, 777, 45, 777, 45, 777, 773, 777, - 0, 0, 779, 0, 0, 0, 0, 821, - 3, 3, 3, 3, 821, 1, 3, 1, - 0, 3, 3, 855, 855, 3, 3, 819, - 0, 821, 27, 27, 27, 27, 0, 821, - 745, 0, 821, 735, 733, 3, 3, 3, - 3, 821, 739, 821, 763, 0, 821, 1, - 1, 0, 821, 725, 0, 727, 25, 25, - 25, 25, 821, 615, 613, 783, 619, 617, - 785, 623, 621, 787, 0, 0, 0, 781, - 5, 639, 647, 611, 695, 607, 717, 699, - 715, 683, 683, 0, 605, 713, 691, 683, - 703, 687, 719, 641, 657, 645, 0, 693, - 659, 655, 906, 697, 45, 683, 651, 45, - 0, 653, 689, 649, 683, 701, 685, 7, - 643, 852, 915, 721, 15, 791, 867, 791, - 858, 919, 795, 927, 919, 793, 919, 847, - 709, 711, 681, 0, 7, 0, 849, 17, - 801, 17, 799, 876, 799, 17, 797, 873, - 797, 1, 1, 1, 815, 0, 803, 705, - 707, 45, 19, 23, 21, 805, 882, 894, - 888, 805, 0, 0, 0, 807, 11, 825, - 627, 625, 789, 823, 767, 823, 823, 0, - 823, 823, 0, 823, 55, 45, 45, 53, + 537, 471, 0, 537, 475, 0, 0, 0, + 539, 541, 503, 541, 541, 0, 541, 541, + 505, 541, 573, 561, 573, 573, 0, 573, + 573, 563, 573, 591, 579, 591, 591, 0, + 591, 591, 581, 591, 637, 779, 781, 633, + 641, 45, 601, 0, 0, 613, 0, 45, + 0, 639, 913, 603, 779, 781, 777, 781, + 0, 781, 45, 781, 45, 781, 777, 781, + 0, 0, 783, 0, 0, 0, 0, 825, + 3, 3, 3, 3, 825, 1, 3, 1, + 0, 3, 3, 859, 859, 3, 3, 823, + 0, 825, 27, 27, 27, 27, 0, 825, + 749, 0, 825, 739, 737, 3, 3, 3, + 3, 825, 743, 825, 767, 0, 825, 1, + 1, 0, 825, 729, 0, 731, 25, 25, + 25, 25, 825, 619, 617, 787, 623, 621, + 789, 627, 625, 791, 0, 0, 0, 785, + 5, 643, 651, 615, 699, 611, 721, 703, + 719, 687, 687, 0, 609, 717, 695, 687, + 707, 691, 723, 645, 661, 649, 0, 697, + 663, 659, 910, 701, 45, 687, 655, 45, + 0, 657, 693, 653, 687, 705, 689, 7, + 647, 856, 919, 725, 15, 795, 871, 795, + 862, 923, 799, 931, 923, 797, 923, 851, + 713, 715, 685, 0, 7, 0, 853, 17, + 805, 17, 803, 880, 803, 17, 801, 877, + 801, 1, 1, 1, 819, 0, 807, 709, + 711, 45, 19, 23, 21, 809, 886, 898, + 892, 809, 0, 0, 0, 811, 11, 829, + 631, 629, 793, 827, 771, 827, 827, 0, + 827, 827, 0, 827, 55, 45, 45, 53, 0, 53, 0, 0, 0, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, @@ -3642,47 +3643,48 @@ static const short _regex_actions[] = { 0, 339, 0, 339, 0, 0, 0, 0, 339, 313, 339, 113, 115, 117, 335, 39, 361, 363, 365, 367, 369, 371, 373, 375, - 491, 45, 0, 511, 509, 507, 509, 0, - 509, 45, 509, 45, 509, 507, 509, 0, - 0, 0, 531, 5, 489, 439, 457, 493, - 493, 0, 437, 485, 493, 461, 481, 451, - 441, 0, 487, 453, 449, 495, 455, 493, - 445, 45, 0, 447, 483, 443, 493, 459, - 479, 7, 9, 473, 497, 497, 499, 15, - 517, 861, 517, 15, 519, 864, 519, 467, - 469, 477, 0, 521, 463, 465, 45, 19, - 23, 21, 523, 879, 891, 885, 523, 0, - 0, 0, 525, 529, 501, 529, 529, 0, - 529, 529, 0, 529, 0, 549, 543, 549, - 541, 549, 547, 545, 551, 0, 563, 561, - 563, 0, 563, 45, 563, 45, 563, 561, - 563, 553, 567, 565, 555, 565, 565, 0, - 565, 565, 0, 565, 0, 581, 579, 581, - 0, 581, 45, 581, 45, 581, 579, 581, - 571, 585, 583, 573, 583, 583, 0, 583, - 583, 0, 583, 589, 591, 593, 595, 0, - 827, 827, 841, 841, 839, 839, 839, 839, - 839, 839, 839, 839, 839, 839, 839, 839, - 839, 839, 841, 841, 841, 841, 841, 841, - 841, 841, 841, 841, 829, 829, 829, 829, - 829, 829, 829, 829, 829, 831, 831, 831, - 831, 831, 831, 831, 831, 831, 831, 831, - 831, 837, 837, 837, 837, 837, 837, 833, - 833, 835, 845, 845, 845, 843, 843, 843, - 59, 59, 59, 59, 59, 59, 359, 359, + 0, 491, 45, 0, 513, 511, 507, 511, + 0, 511, 45, 511, 45, 511, 507, 511, + 509, 535, 0, 0, 0, 533, 5, 489, + 439, 457, 493, 493, 0, 437, 485, 493, + 461, 481, 451, 441, 0, 487, 453, 449, + 495, 455, 493, 445, 45, 0, 447, 483, + 443, 493, 459, 479, 7, 9, 473, 497, + 497, 499, 15, 519, 865, 519, 15, 521, + 868, 521, 467, 469, 477, 0, 523, 463, + 465, 45, 19, 23, 21, 525, 883, 895, + 889, 525, 0, 0, 0, 527, 531, 501, + 531, 531, 0, 531, 531, 0, 531, 0, + 553, 547, 553, 545, 553, 551, 549, 555, + 0, 567, 565, 567, 0, 567, 45, 567, + 45, 567, 565, 567, 557, 571, 569, 559, + 569, 569, 0, 569, 569, 0, 569, 0, + 585, 583, 585, 0, 585, 45, 585, 45, + 585, 583, 585, 575, 589, 587, 577, 587, + 587, 0, 587, 587, 0, 587, 593, 595, + 597, 599, 0, 831, 831, 845, 845, 843, + 843, 843, 843, 843, 843, 843, 843, 843, + 843, 843, 843, 843, 843, 845, 845, 845, + 845, 845, 845, 845, 845, 845, 845, 833, + 833, 833, 833, 833, 833, 833, 833, 833, + 835, 835, 835, 835, 835, 835, 835, 835, + 835, 835, 835, 835, 841, 841, 841, 841, + 841, 841, 837, 837, 839, 849, 849, 849, + 847, 847, 847, 59, 59, 59, 59, 59, + 59, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, - 359, 359, 359, 359, 359, 359, 359, 359, - 359, 341, 341, 341, 341, 341, 341, 341, + 359, 359, 359, 359, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, - 341, 341, 341, 341, 343, 343, 343, 343, - 343, 343, 341, 341, 341, 341, 341, 341, + 341, 341, 341, 341, 341, 341, 341, 343, + 343, 343, 343, 343, 343, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, - 341, 341, 341, 359, 359, 359, 359, 359, + 341, 341, 341, 341, 341, 341, 359, 359, + 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, @@ -3690,8 +3692,8 @@ static const short _regex_actions[] = { 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, + 359, 359, 359, 357, 357, 357, 357, 357, 359, 359, 359, 359, 359, 359, 359, 359, - 357, 357, 357, 357, 357, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, @@ -3702,72 +3704,71 @@ static const short _regex_actions[] = { 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, - 359, 359, 359, 359, 359, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, - 345, 345, 345, 345, 347, 347, 347, 347, - 347, 347, 347, 347, 347, 347, 347, 349, - 349, 349, 349, 349, 349, 349, 349, 349, + 345, 345, 345, 345, 345, 345, 345, 347, 347, 347, 347, 347, 347, 347, 347, 347, - 347, 347, 347, 347, 351, 351, 351, 351, - 351, 351, 351, 351, 351, 351, 359, 359, + 347, 347, 349, 349, 349, 349, 349, 349, + 349, 349, 349, 347, 347, 347, 347, 347, + 347, 347, 347, 347, 347, 347, 347, 351, + 351, 351, 351, 351, 351, 351, 351, 351, + 351, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, - 359, 359, 359, 359, 359, 353, 353, 353, 353, 353, 353, 353, 353, 353, 353, 353, - 353, 353, 359, 359, 359, 359, 359, 359, - 359, 355, 355, 355, 355, 355, 355, 355, + 353, 353, 353, 353, 353, 359, 359, 359, + 359, 359, 359, 359, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, - 355, 355, 355, 355, 355, 359, 359, 359, + 355, 355, 355, 355, 355, 355, 355, 355, + 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, - 359, 359, 359, 359, 359, 359, 39, 39, - 39, 39, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 533, 533, 535, - 537, 537, 537, 569, 569, 569, 587, 587, - 587, 0, 779, 821, 821, 819, 821, 821, - 821, 821, 821, 821, 821, 821, 783, 785, - 787, 781, 817, 791, 791, 795, 793, 847, - 811, 809, 801, 799, 799, 797, 797, 815, - 803, 813, 805, 805, 807, 825, 789, 823, - 823, 823, 0, 57, 57, 0, 339, 339, - 319, 321, 339, 339, 339, 339, 337, 339, - 339, 339, 323, 325, 327, 329, 339, 331, - 339, 333, 339, 339, 339, 339, 339, 335, - 0, 0, 0, 531, 531, 517, 517, 519, - 519, 513, 527, 521, 515, 523, 523, 525, - 529, 529, 529, 0, 551, 0, 567, 565, - 565, 565, 0, 585, 583, 583, 583, 0, - 0, 0 + 359, 39, 39, 39, 39, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 537, 537, 539, 541, 541, 541, 573, 573, + 573, 591, 591, 591, 0, 783, 825, 825, + 823, 825, 825, 825, 825, 825, 825, 825, + 825, 787, 789, 791, 785, 821, 795, 795, + 799, 797, 851, 815, 813, 805, 803, 803, + 801, 801, 819, 807, 817, 809, 809, 811, + 829, 793, 827, 827, 827, 0, 57, 57, + 0, 339, 339, 319, 321, 339, 339, 339, + 339, 337, 339, 339, 339, 323, 325, 327, + 329, 339, 331, 339, 333, 339, 339, 339, + 339, 339, 335, 0, 0, 0, 535, 533, + 535, 519, 519, 521, 521, 515, 529, 523, + 517, 525, 525, 527, 531, 531, 531, 0, + 555, 0, 571, 569, 569, 569, 0, 589, + 587, 587, 587, 0, 0, 0 }; static const signed char _regex_to_state_actions[] = { @@ -3875,9 +3876,9 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 41, 0, 41, 0, - 0, 0, 0, 41, 0, 0, 0, 0, - 41, 41, 0 + 0, 0, 0, 0, 0, 41, 0, 41, + 0, 0, 0, 0, 41, 0, 0, 0, + 0, 41, 41, 0 }; static const signed char _regex_from_state_actions[] = { @@ -3985,119 +3986,119 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 43, 0, 43, 0, - 0, 0, 0, 43, 0, 0, 0, 0, - 43, 43, 0 + 0, 0, 0, 0, 0, 43, 0, 43, + 0, 0, 0, 0, 43, 0, 0, 0, + 0, 43, 43, 0 }; static const short _regex_eof_trans[] = { - 2665, 2666, 2667, 2668, 2669, 2670, 2671, 2672, - 2673, 2674, 2675, 2676, 2677, 2678, 2679, 2680, - 2681, 2682, 2683, 2684, 2685, 2686, 2687, 2688, - 2689, 2690, 2691, 2692, 2693, 2694, 2695, 2696, - 2697, 2698, 2699, 2700, 2701, 2702, 2703, 2704, - 2705, 2706, 2707, 2708, 2709, 2710, 2711, 2712, - 2713, 2714, 2715, 2716, 2717, 2718, 2719, 2720, - 2721, 2722, 2723, 2724, 2725, 2726, 2727, 2728, - 2729, 2730, 2731, 2732, 2733, 2734, 2735, 2736, - 2737, 2738, 2739, 2740, 2741, 2742, 2743, 2744, - 2745, 2746, 2747, 2748, 2749, 2750, 2751, 2752, - 2753, 2754, 2755, 2756, 2757, 2758, 2759, 2760, - 2761, 2762, 2763, 2764, 2765, 2766, 2767, 2768, - 2769, 2770, 2771, 2772, 2773, 2774, 2775, 2776, - 2777, 2778, 2779, 2780, 2781, 2782, 2783, 2784, - 2785, 2786, 2787, 2788, 2789, 2790, 2791, 2792, - 2793, 2794, 2795, 2796, 2797, 2798, 2799, 2800, - 2801, 2802, 2803, 2804, 2805, 2806, 2807, 2808, - 2809, 2810, 2811, 2812, 2813, 2814, 2815, 2816, - 2817, 2818, 2819, 2820, 2821, 2822, 2823, 2824, - 2825, 2826, 2827, 2828, 2829, 2830, 2831, 2832, - 2833, 2834, 2835, 2836, 2837, 2838, 2839, 2840, - 2841, 2842, 2843, 2844, 2845, 2846, 2847, 2848, - 2849, 2850, 2851, 2852, 2853, 2854, 2855, 2856, - 2857, 2858, 2859, 2860, 2861, 2862, 2863, 2864, - 2865, 2866, 2867, 2868, 2869, 2870, 2871, 2872, - 2873, 2874, 2875, 2876, 2877, 2878, 2879, 2880, - 2881, 2882, 2883, 2884, 2885, 2886, 2887, 2888, - 2889, 2890, 2891, 2892, 2893, 2894, 2895, 2896, - 2897, 2898, 2899, 2900, 2901, 2902, 2903, 2904, - 2905, 2906, 2907, 2908, 2909, 2910, 2911, 2912, - 2913, 2914, 2915, 2916, 2917, 2918, 2919, 2920, - 2921, 2922, 2923, 2924, 2925, 2926, 2927, 2928, - 2929, 2930, 2931, 2932, 2933, 2934, 2935, 2936, - 2937, 2938, 2939, 2940, 2941, 2942, 2943, 2944, - 2945, 2946, 2947, 2948, 2949, 2950, 2951, 2952, - 2953, 2954, 2955, 2956, 2957, 2958, 2959, 2960, - 2961, 2962, 2963, 2964, 2965, 2966, 2967, 2968, - 2969, 2970, 2971, 2972, 2973, 2974, 2975, 2976, - 2977, 2978, 2979, 2980, 2981, 2982, 2983, 2984, - 2985, 2986, 2987, 2988, 2989, 2990, 2991, 2992, - 2993, 2994, 2995, 2996, 2997, 2998, 2999, 3000, - 3001, 3002, 3003, 3004, 3005, 3006, 3007, 3008, - 3009, 3010, 3011, 3012, 3013, 3014, 3015, 3016, - 3017, 3018, 3019, 3020, 3021, 3022, 3023, 3024, - 3025, 3026, 3027, 3028, 3029, 3030, 3031, 3032, - 3033, 3034, 3035, 3036, 3037, 3038, 3039, 3040, - 3041, 3042, 3043, 3044, 3045, 3046, 3047, 3048, - 3049, 3050, 3051, 3052, 3053, 3054, 3055, 3056, - 3057, 3058, 3059, 3060, 3061, 3062, 3063, 3064, - 3065, 3066, 3067, 3068, 3069, 3070, 3071, 3072, - 3073, 3074, 3075, 3076, 3077, 3078, 3079, 3080, - 3081, 3082, 3083, 3084, 3085, 3086, 3087, 3088, - 3089, 3090, 3091, 3092, 3093, 3094, 3095, 3096, - 3097, 3098, 3099, 3100, 3101, 3102, 3103, 3104, - 3105, 3106, 3107, 3108, 3109, 3110, 3111, 3112, - 3113, 3114, 3115, 3116, 3117, 3118, 3119, 3120, - 3121, 3122, 3123, 3124, 3125, 3126, 3127, 3128, - 3129, 3130, 3131, 3132, 3133, 3134, 3135, 3136, - 3137, 3138, 3139, 3140, 3141, 3142, 3143, 3144, - 3145, 3146, 3147, 3148, 3149, 3150, 3151, 3152, - 3153, 3154, 3155, 3156, 3157, 3158, 3159, 3160, - 3161, 3162, 3163, 3164, 3165, 3166, 3167, 3168, - 3169, 3170, 3171, 3172, 3173, 3174, 3175, 3176, - 3177, 3178, 3179, 3180, 3181, 3182, 3183, 3184, - 3185, 3186, 3187, 3188, 3189, 3190, 3191, 3192, - 3193, 3194, 3195, 3196, 3197, 3198, 3199, 3200, - 3201, 3202, 3203, 3204, 3205, 3206, 3207, 3208, - 3209, 3210, 3211, 3212, 3213, 3214, 3215, 3216, - 3217, 3218, 3219, 3220, 3221, 3222, 3223, 3224, - 3225, 3226, 3227, 3228, 3229, 3230, 3231, 3232, - 3233, 3234, 3235, 3236, 3237, 3238, 3239, 3240, - 3241, 3242, 3243, 3244, 3245, 3246, 3247, 3248, - 3249, 3250, 3251, 3252, 3253, 3254, 3255, 3256, - 3257, 3258, 3259, 3260, 3261, 3262, 3263, 3264, - 3265, 3266, 3267, 3268, 3269, 3270, 3271, 3272, - 3273, 3274, 3275, 3276, 3277, 3278, 3279, 3280, - 3281, 3282, 3283, 3284, 3285, 3286, 3287, 3288, - 3289, 3290, 3291, 3292, 3293, 3294, 3295, 3296, - 3297, 3298, 3299, 3300, 3301, 3302, 3303, 3304, - 3305, 3306, 3307, 3308, 3309, 3310, 3311, 3312, - 3313, 3314, 3315, 3316, 3317, 3318, 3319, 3320, - 3321, 3322, 3323, 3324, 3325, 3326, 3327, 3328, - 3329, 3330, 3331, 3332, 3333, 3334, 3335, 3336, - 3337, 3338, 3339, 3340, 3341, 3342, 3343, 3344, - 3345, 3346, 3347, 3348, 3349, 3350, 3351, 3352, - 3353, 3354, 3355, 3356, 3357, 3358, 3359, 3360, - 3361, 3362, 3363, 3364, 3365, 3366, 3367, 3368, - 3369, 3370, 3371, 3372, 3373, 3374, 3375, 3376, - 3377, 3378, 3379, 3380, 3381, 3382, 3383, 3384, - 3385, 3386, 3387, 3388, 3389, 3390, 3391, 3392, - 3393, 3394, 3395, 3396, 3397, 3398, 3399, 3400, - 3401, 3402, 3403, 3404, 3405, 3406, 3407, 3408, - 3409, 3410, 3411, 3412, 3413, 3414, 3415, 3416, - 3417, 3418, 3419, 3420, 3421, 3422, 3423, 3424, - 3425, 3426, 3427, 3428, 3429, 3430, 3431, 3432, - 3433, 3434, 3435, 3436, 3437, 3438, 3439, 3440, - 3441, 3442, 3443, 3444, 3445, 3446, 3447, 3448, - 3449, 3450, 3451, 3452, 3453, 3454, 3455, 3456, - 3457, 3458, 3459, 3460, 3461, 3462, 3463, 3464, - 3465, 3466, 3467, 3468, 3469, 3470, 3471, 3472, - 3473, 3474, 3475, 3476, 3477, 3478, 3479, 3480, - 3481, 3482, 3483, 3484, 3485, 3486, 3487, 3488, - 3489, 3490, 3491, 3492, 3493, 3494, 3495, 3496, - 3497, 3498, 3499, 3500, 3501, 3502, 3503, 3504, - 3505, 3506, 3507, 3508, 3509, 3510, 3511, 3512, - 3513, 3514, 0 + 2668, 2669, 2670, 2671, 2672, 2673, 2674, 2675, + 2676, 2677, 2678, 2679, 2680, 2681, 2682, 2683, + 2684, 2685, 2686, 2687, 2688, 2689, 2690, 2691, + 2692, 2693, 2694, 2695, 2696, 2697, 2698, 2699, + 2700, 2701, 2702, 2703, 2704, 2705, 2706, 2707, + 2708, 2709, 2710, 2711, 2712, 2713, 2714, 2715, + 2716, 2717, 2718, 2719, 2720, 2721, 2722, 2723, + 2724, 2725, 2726, 2727, 2728, 2729, 2730, 2731, + 2732, 2733, 2734, 2735, 2736, 2737, 2738, 2739, + 2740, 2741, 2742, 2743, 2744, 2745, 2746, 2747, + 2748, 2749, 2750, 2751, 2752, 2753, 2754, 2755, + 2756, 2757, 2758, 2759, 2760, 2761, 2762, 2763, + 2764, 2765, 2766, 2767, 2768, 2769, 2770, 2771, + 2772, 2773, 2774, 2775, 2776, 2777, 2778, 2779, + 2780, 2781, 2782, 2783, 2784, 2785, 2786, 2787, + 2788, 2789, 2790, 2791, 2792, 2793, 2794, 2795, + 2796, 2797, 2798, 2799, 2800, 2801, 2802, 2803, + 2804, 2805, 2806, 2807, 2808, 2809, 2810, 2811, + 2812, 2813, 2814, 2815, 2816, 2817, 2818, 2819, + 2820, 2821, 2822, 2823, 2824, 2825, 2826, 2827, + 2828, 2829, 2830, 2831, 2832, 2833, 2834, 2835, + 2836, 2837, 2838, 2839, 2840, 2841, 2842, 2843, + 2844, 2845, 2846, 2847, 2848, 2849, 2850, 2851, + 2852, 2853, 2854, 2855, 2856, 2857, 2858, 2859, + 2860, 2861, 2862, 2863, 2864, 2865, 2866, 2867, + 2868, 2869, 2870, 2871, 2872, 2873, 2874, 2875, + 2876, 2877, 2878, 2879, 2880, 2881, 2882, 2883, + 2884, 2885, 2886, 2887, 2888, 2889, 2890, 2891, + 2892, 2893, 2894, 2895, 2896, 2897, 2898, 2899, + 2900, 2901, 2902, 2903, 2904, 2905, 2906, 2907, + 2908, 2909, 2910, 2911, 2912, 2913, 2914, 2915, + 2916, 2917, 2918, 2919, 2920, 2921, 2922, 2923, + 2924, 2925, 2926, 2927, 2928, 2929, 2930, 2931, + 2932, 2933, 2934, 2935, 2936, 2937, 2938, 2939, + 2940, 2941, 2942, 2943, 2944, 2945, 2946, 2947, + 2948, 2949, 2950, 2951, 2952, 2953, 2954, 2955, + 2956, 2957, 2958, 2959, 2960, 2961, 2962, 2963, + 2964, 2965, 2966, 2967, 2968, 2969, 2970, 2971, + 2972, 2973, 2974, 2975, 2976, 2977, 2978, 2979, + 2980, 2981, 2982, 2983, 2984, 2985, 2986, 2987, + 2988, 2989, 2990, 2991, 2992, 2993, 2994, 2995, + 2996, 2997, 2998, 2999, 3000, 3001, 3002, 3003, + 3004, 3005, 3006, 3007, 3008, 3009, 3010, 3011, + 3012, 3013, 3014, 3015, 3016, 3017, 3018, 3019, + 3020, 3021, 3022, 3023, 3024, 3025, 3026, 3027, + 3028, 3029, 3030, 3031, 3032, 3033, 3034, 3035, + 3036, 3037, 3038, 3039, 3040, 3041, 3042, 3043, + 3044, 3045, 3046, 3047, 3048, 3049, 3050, 3051, + 3052, 3053, 3054, 3055, 3056, 3057, 3058, 3059, + 3060, 3061, 3062, 3063, 3064, 3065, 3066, 3067, + 3068, 3069, 3070, 3071, 3072, 3073, 3074, 3075, + 3076, 3077, 3078, 3079, 3080, 3081, 3082, 3083, + 3084, 3085, 3086, 3087, 3088, 3089, 3090, 3091, + 3092, 3093, 3094, 3095, 3096, 3097, 3098, 3099, + 3100, 3101, 3102, 3103, 3104, 3105, 3106, 3107, + 3108, 3109, 3110, 3111, 3112, 3113, 3114, 3115, + 3116, 3117, 3118, 3119, 3120, 3121, 3122, 3123, + 3124, 3125, 3126, 3127, 3128, 3129, 3130, 3131, + 3132, 3133, 3134, 3135, 3136, 3137, 3138, 3139, + 3140, 3141, 3142, 3143, 3144, 3145, 3146, 3147, + 3148, 3149, 3150, 3151, 3152, 3153, 3154, 3155, + 3156, 3157, 3158, 3159, 3160, 3161, 3162, 3163, + 3164, 3165, 3166, 3167, 3168, 3169, 3170, 3171, + 3172, 3173, 3174, 3175, 3176, 3177, 3178, 3179, + 3180, 3181, 3182, 3183, 3184, 3185, 3186, 3187, + 3188, 3189, 3190, 3191, 3192, 3193, 3194, 3195, + 3196, 3197, 3198, 3199, 3200, 3201, 3202, 3203, + 3204, 3205, 3206, 3207, 3208, 3209, 3210, 3211, + 3212, 3213, 3214, 3215, 3216, 3217, 3218, 3219, + 3220, 3221, 3222, 3223, 3224, 3225, 3226, 3227, + 3228, 3229, 3230, 3231, 3232, 3233, 3234, 3235, + 3236, 3237, 3238, 3239, 3240, 3241, 3242, 3243, + 3244, 3245, 3246, 3247, 3248, 3249, 3250, 3251, + 3252, 3253, 3254, 3255, 3256, 3257, 3258, 3259, + 3260, 3261, 3262, 3263, 3264, 3265, 3266, 3267, + 3268, 3269, 3270, 3271, 3272, 3273, 3274, 3275, + 3276, 3277, 3278, 3279, 3280, 3281, 3282, 3283, + 3284, 3285, 3286, 3287, 3288, 3289, 3290, 3291, + 3292, 3293, 3294, 3295, 3296, 3297, 3298, 3299, + 3300, 3301, 3302, 3303, 3304, 3305, 3306, 3307, + 3308, 3309, 3310, 3311, 3312, 3313, 3314, 3315, + 3316, 3317, 3318, 3319, 3320, 3321, 3322, 3323, + 3324, 3325, 3326, 3327, 3328, 3329, 3330, 3331, + 3332, 3333, 3334, 3335, 3336, 3337, 3338, 3339, + 3340, 3341, 3342, 3343, 3344, 3345, 3346, 3347, + 3348, 3349, 3350, 3351, 3352, 3353, 3354, 3355, + 3356, 3357, 3358, 3359, 3360, 3361, 3362, 3363, + 3364, 3365, 3366, 3367, 3368, 3369, 3370, 3371, + 3372, 3373, 3374, 3375, 3376, 3377, 3378, 3379, + 3380, 3381, 3382, 3383, 3384, 3385, 3386, 3387, + 3388, 3389, 3390, 3391, 3392, 3393, 3394, 3395, + 3396, 3397, 3398, 3399, 3400, 3401, 3402, 3403, + 3404, 3405, 3406, 3407, 3408, 3409, 3410, 3411, + 3412, 3413, 3414, 3415, 3416, 3417, 3418, 3419, + 3420, 3421, 3422, 3423, 3424, 3425, 3426, 3427, + 3428, 3429, 3430, 3431, 3432, 3433, 3434, 3435, + 3436, 3437, 3438, 3439, 3440, 3441, 3442, 3443, + 3444, 3445, 3446, 3447, 3448, 3449, 3450, 3451, + 3452, 3453, 3454, 3455, 3456, 3457, 3458, 3459, + 3460, 3461, 3462, 3463, 3464, 3465, 3466, 3467, + 3468, 3469, 3470, 3471, 3472, 3473, 3474, 3475, + 3476, 3477, 3478, 3479, 3480, 3481, 3482, 3483, + 3484, 3485, 3486, 3487, 3488, 3489, 3490, 3491, + 3492, 3493, 3494, 3495, 3496, 3497, 3498, 3499, + 3500, 3501, 3502, 3503, 3504, 3505, 3506, 3507, + 3508, 3509, 3510, 3511, 3512, 3513, 3514, 3515, + 3516, 3517, 3518, 0 }; static const int regex_start = 746; @@ -4108,15 +4109,15 @@ static const short _regex_actions[] = { static const int regex_en_readBracedUCP = 559; static const int regex_en_readUCPSingle = 818; static const int regex_en_charClassGuts = 819; - static const int regex_en_readClass = 836; - static const int regex_en_readQuotedLiteral = 838; - static const int regex_en_readQuotedClass = 843; - static const int regex_en_readComment = 848; - static const int regex_en_readNewlineTerminatedComment = 849; + static const int regex_en_readClass = 837; + static const int regex_en_readQuotedLiteral = 839; + static const int regex_en_readQuotedClass = 844; + static const int regex_en_readComment = 849; + static const int regex_en_readNewlineTerminatedComment = 850; static const int regex_en_main = 746; -#line 1911 "Parser.rl" +#line 1918 "Parser.rl" /** \brief Main parser call, returns root Component or nullptr. */ @@ -4189,7 +4190,7 @@ static const short _regex_actions[] = { try { // Embed the Ragel machine here -#line 4187 "Parser.cpp" +#line 4188 "Parser.cpp" { cs = (int)regex_start; top = 0; @@ -4198,17 +4199,17 @@ static const short _regex_actions[] = { act = 0; } -#line 1982 "Parser.rl" +#line 1989 "Parser.rl" -#line 4196 "Parser.cpp" +#line 4197 "Parser.cpp" { int _cpc; int _klen; const signed char * _ckeys; unsigned int _trans = 0; unsigned int _cond = 0; - const char * _keys; + const unsigned char * _keys; const short * _acts; unsigned int _nacts; _resume: {} @@ -4224,7 +4225,7 @@ static const short _regex_actions[] = { #line 1 "NONE" {ts = p;}} -#line 4218 "Parser.cpp" +#line 4219 "Parser.cpp" break; } @@ -4244,9 +4245,9 @@ static const short _regex_actions[] = { _klen = (int)_regex_single_lengths[cs]; if ( _klen > 0 ) { - const char *_lower = _keys; - const char *_upper = _keys + _klen - 1; - const char *_mid; + const unsigned char *_lower = _keys; + const unsigned char *_upper = _keys + _klen - 1; + const unsigned char *_mid; while ( 1 ) { if ( _upper < _lower ) { _keys += _klen; @@ -4268,9 +4269,9 @@ static const short _regex_actions[] = { _klen = (int)_regex_range_lengths[cs]; if ( _klen > 0 ) { - const char *_lower = _keys; - const char *_upper = _keys + (_klen<<1) - 2; - const char *_mid; + const unsigned char *_lower = _keys; + const unsigned char *_upper = _keys + (_klen<<1) - 2; + const unsigned char *_mid; while ( 1 ) { if ( _upper < _lower ) { _trans += (unsigned int)_klen; @@ -4300,19 +4301,19 @@ static const short _regex_actions[] = { case 0: { if ( ( mode.utf8 ) -#line 4293 "Parser.cpp" +#line 4294 "Parser.cpp" ) _cpc += 1; break; } case 1: { if ( ( mode.ignore_space ) -#line 4298 "Parser.cpp" +#line 4299 "Parser.cpp" ) _cpc += 1; break; } case 2: { if ( ( inCharClassEarly ) -#line 4303 "Parser.cpp" +#line 4304 "Parser.cpp" ) _cpc += 1; break; } @@ -4323,7 +4324,7 @@ static const short _regex_actions[] = { const signed char *_mid; while ( 1 ) { if ( _upper < _lower ) { - _cond = 3561; + _cond = 3565; break; } @@ -4352,159 +4353,159 @@ static const short _regex_actions[] = { { case 0: { { -#line 285 "Parser.rl" +#line 286 "Parser.rl" label.clear();} -#line 4345 "Parser.cpp" +#line 4346 "Parser.cpp" break; } case 1: { { -#line 286 "Parser.rl" +#line 287 "Parser.rl" label.push_back((( (*( p)))));} -#line 4353 "Parser.cpp" +#line 4354 "Parser.cpp" break; } case 2: { { -#line 287 "Parser.rl" +#line 288 "Parser.rl" octAccumulator = 0;} -#line 4361 "Parser.cpp" +#line 4362 "Parser.cpp" break; } case 3: { { -#line 288 "Parser.rl" +#line 289 "Parser.rl" accumulator = 0;} -#line 4369 "Parser.cpp" +#line 4370 "Parser.cpp" break; } case 4: { { -#line 289 "Parser.rl" +#line 290 "Parser.rl" octAccumulator = 0; pushOct(&octAccumulator, (( (*( p))))); } -#line 4380 "Parser.cpp" +#line 4381 "Parser.cpp" break; } case 5: { { -#line 293 "Parser.rl" +#line 294 "Parser.rl" accumulator = 0; pushDec(&accumulator, (( (*( p))))); } -#line 4391 "Parser.cpp" +#line 4392 "Parser.cpp" break; } case 6: { { -#line 297 "Parser.rl" +#line 298 "Parser.rl" repeatN = 0; repeatM = 0; } -#line 4399 "Parser.cpp" +#line 4400 "Parser.cpp" break; } case 7: { { -#line 298 "Parser.rl" +#line 299 "Parser.rl" pushDec(&repeatN, (( (*( p))))); } -#line 4407 "Parser.cpp" +#line 4408 "Parser.cpp" break; } case 8: { { -#line 299 "Parser.rl" +#line 300 "Parser.rl" pushDec(&repeatM, (( (*( p))))); } -#line 4415 "Parser.cpp" +#line 4416 "Parser.cpp" break; } case 9: { { -#line 300 "Parser.rl" +#line 301 "Parser.rl" pushOct(&octAccumulator, (( (*( p))))); } -#line 4423 "Parser.cpp" +#line 4424 "Parser.cpp" break; } case 10: { { -#line 301 "Parser.rl" +#line 302 "Parser.rl" pushDec(&accumulator, (( (*( p))))); } -#line 4431 "Parser.cpp" +#line 4432 "Parser.cpp" break; } case 11: { { -#line 302 "Parser.rl" +#line 303 "Parser.rl" accumulator *= 16; accumulator += (( (*( p)))) - '0'; } -#line 4442 "Parser.cpp" +#line 4443 "Parser.cpp" break; } case 12: { { -#line 306 "Parser.rl" +#line 307 "Parser.rl" accumulator *= 16; accumulator += 10 + (( (*( p)))) - 'a'; } -#line 4453 "Parser.cpp" +#line 4454 "Parser.cpp" break; } case 13: { { -#line 310 "Parser.rl" +#line 311 "Parser.rl" accumulator *= 16; accumulator += 10 + (( (*( p)))) - 'A'; } -#line 4464 "Parser.cpp" +#line 4465 "Parser.cpp" break; } case 14: { { -#line 430 "Parser.rl" +#line 431 "Parser.rl" newMode = mode; } -#line 4474 "Parser.cpp" +#line 4475 "Parser.cpp" break; } case 15: { { -#line 437 "Parser.rl" +#line 438 "Parser.rl" switch ((( (*( p))))) { case 'i': @@ -4525,13 +4526,13 @@ static const short _regex_actions[] = { } } -#line 4500 "Parser.cpp" +#line 4501 "Parser.cpp" break; } case 16: { { -#line 456 "Parser.rl" +#line 457 "Parser.rl" switch ((( (*( p))))) { case 'i': @@ -4552,42 +4553,42 @@ static const short _regex_actions[] = { } } -#line 4526 "Parser.cpp" +#line 4527 "Parser.cpp" break; } case 17: { { -#line 510 "Parser.rl" +#line 511 "Parser.rl" repeatM = repeatN;} -#line 4534 "Parser.cpp" +#line 4535 "Parser.cpp" break; } case 18: { { -#line 510 "Parser.rl" +#line 511 "Parser.rl" repeatM = ComponentRepeat::NoLimit;} -#line 4542 "Parser.cpp" +#line 4543 "Parser.cpp" break; } case 19: { { -#line 722 "Parser.rl" +#line 723 "Parser.rl" negated = !negated; } -#line 4550 "Parser.cpp" +#line 4551 "Parser.cpp" break; } case 20: { { -#line 723 "Parser.rl" +#line 724 "Parser.rl" {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -4596,30 +4597,30 @@ static const short _regex_actions[] = { } stack[top] = cs; top += 1;cs = 790;goto _again;}} -#line 4566 "Parser.cpp" +#line 4567 "Parser.cpp" break; } case 21: { { -#line 724 "Parser.rl" +#line 725 "Parser.rl" if (!inCharClass) { // not inside [..] currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } -#line 4579 "Parser.cpp" +#line 4580 "Parser.cpp" break; } case 22: { { -#line 730 "Parser.rl" +#line 731 "Parser.rl" throw LocatedParseError("Malformed property"); } -#line 4587 "Parser.cpp" +#line 4588 "Parser.cpp" break; } @@ -4628,60 +4629,60 @@ static const short _regex_actions[] = { #line 1 "NONE" {te = p+1;}} -#line 4595 "Parser.cpp" +#line 4596 "Parser.cpp" break; } case 26: { { -#line 550 "Parser.rl" +#line 551 "Parser.rl" {te = p+1;{ -#line 550 "Parser.rl" +#line 551 "Parser.rl" throw LocatedParseError("(*UTF8) must be at start of " "expression, encountered"); } }} -#line 4609 "Parser.cpp" +#line 4610 "Parser.cpp" break; } case 27: { { -#line 554 "Parser.rl" +#line 555 "Parser.rl" {te = p+1;{ -#line 554 "Parser.rl" +#line 555 "Parser.rl" throw LocatedParseError("(*UTF) must be at start of " "expression, encountered"); } }} -#line 4623 "Parser.cpp" +#line 4624 "Parser.cpp" break; } case 28: { { -#line 558 "Parser.rl" +#line 559 "Parser.rl" {te = p+1;{ -#line 558 "Parser.rl" +#line 559 "Parser.rl" throw LocatedParseError("(*UCP) must be at start of " "expression, encountered"); } }} -#line 4637 "Parser.cpp" +#line 4638 "Parser.cpp" break; } case 29: { { -#line 564 "Parser.rl" +#line 565 "Parser.rl" {te = p+1;{ -#line 564 "Parser.rl" +#line 565 "Parser.rl" ParseMode temp_mode; assert(ts - 2 >= ptr); // parser needs the '(*' at the start too. @@ -4691,2626 +4692,2626 @@ static const short _regex_actions[] = { } }} -#line 4654 "Parser.cpp" +#line 4655 "Parser.cpp" break; } case 30: { { -#line 571 "Parser.rl" +#line 572 "Parser.rl" {te = p+1;{ -#line 571 "Parser.rl" +#line 572 "Parser.rl" throw LocatedParseError("Unknown control verb"); } }} -#line 4667 "Parser.cpp" +#line 4668 "Parser.cpp" break; } case 31: { { -#line 571 "Parser.rl" +#line 572 "Parser.rl" {te = p;p = p - 1;{ -#line 571 "Parser.rl" +#line 572 "Parser.rl" throw LocatedParseError("Unknown control verb"); } }} -#line 4680 "Parser.cpp" +#line 4681 "Parser.cpp" break; } case 32: { { -#line 571 "Parser.rl" +#line 572 "Parser.rl" {p = ((te))-1; { -#line 571 "Parser.rl" +#line 572 "Parser.rl" throw LocatedParseError("Unknown control verb"); } }} -#line 4694 "Parser.cpp" +#line 4695 "Parser.cpp" break; } case 33: { { -#line 581 "Parser.rl" +#line 582 "Parser.rl" {te = p+1;{ -#line 581 "Parser.rl" +#line 582 "Parser.rl" currentCls->add(CLASS_UCP_CC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4705 "Parser.cpp" +#line 4706 "Parser.cpp" break; } case 34: { { -#line 582 "Parser.rl" +#line 583 "Parser.rl" {te = p+1;{ -#line 582 "Parser.rl" +#line 583 "Parser.rl" currentCls->add(CLASS_UCP_CF, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4716 "Parser.cpp" +#line 4717 "Parser.cpp" break; } case 35: { { -#line 583 "Parser.rl" +#line 584 "Parser.rl" {te = p+1;{ -#line 583 "Parser.rl" +#line 584 "Parser.rl" currentCls->add(CLASS_UCP_CN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4727 "Parser.cpp" +#line 4728 "Parser.cpp" break; } case 36: { { -#line 585 "Parser.rl" +#line 586 "Parser.rl" {te = p+1;{ -#line 585 "Parser.rl" +#line 586 "Parser.rl" currentCls->add(CLASS_UCP_CS, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4738 "Parser.cpp" +#line 4739 "Parser.cpp" break; } case 37: { { -#line 587 "Parser.rl" +#line 588 "Parser.rl" {te = p+1;{ -#line 587 "Parser.rl" +#line 588 "Parser.rl" currentCls->add(CLASS_UCP_LL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4749 "Parser.cpp" +#line 4750 "Parser.cpp" break; } case 38: { { -#line 588 "Parser.rl" +#line 589 "Parser.rl" {te = p+1;{ -#line 588 "Parser.rl" +#line 589 "Parser.rl" currentCls->add(CLASS_UCP_LM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4760 "Parser.cpp" +#line 4761 "Parser.cpp" break; } case 39: { { -#line 589 "Parser.rl" +#line 590 "Parser.rl" {te = p+1;{ -#line 589 "Parser.rl" +#line 590 "Parser.rl" currentCls->add(CLASS_UCP_LO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4771 "Parser.cpp" +#line 4772 "Parser.cpp" break; } case 40: { { -#line 590 "Parser.rl" +#line 591 "Parser.rl" {te = p+1;{ -#line 590 "Parser.rl" +#line 591 "Parser.rl" currentCls->add(CLASS_UCP_LT, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4782 "Parser.cpp" +#line 4783 "Parser.cpp" break; } case 41: { { -#line 591 "Parser.rl" +#line 592 "Parser.rl" {te = p+1;{ -#line 591 "Parser.rl" +#line 592 "Parser.rl" currentCls->add(CLASS_UCP_LU, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4793 "Parser.cpp" +#line 4794 "Parser.cpp" break; } case 42: { { -#line 592 "Parser.rl" +#line 593 "Parser.rl" {te = p+1;{ -#line 592 "Parser.rl" +#line 593 "Parser.rl" currentCls->add(CLASS_UCP_L_AND, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4804 "Parser.cpp" +#line 4805 "Parser.cpp" break; } case 43: { { -#line 594 "Parser.rl" +#line 595 "Parser.rl" {te = p+1;{ -#line 594 "Parser.rl" +#line 595 "Parser.rl" currentCls->add(CLASS_UCP_MC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4815 "Parser.cpp" +#line 4816 "Parser.cpp" break; } case 44: { { -#line 596 "Parser.rl" +#line 597 "Parser.rl" {te = p+1;{ -#line 596 "Parser.rl" +#line 597 "Parser.rl" currentCls->add(CLASS_UCP_MN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4826 "Parser.cpp" +#line 4827 "Parser.cpp" break; } case 45: { { -#line 598 "Parser.rl" +#line 599 "Parser.rl" {te = p+1;{ -#line 598 "Parser.rl" +#line 599 "Parser.rl" currentCls->add(CLASS_UCP_ND, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4837 "Parser.cpp" +#line 4838 "Parser.cpp" break; } case 46: { { -#line 599 "Parser.rl" +#line 600 "Parser.rl" {te = p+1;{ -#line 599 "Parser.rl" +#line 600 "Parser.rl" currentCls->add(CLASS_UCP_NL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4848 "Parser.cpp" +#line 4849 "Parser.cpp" break; } case 47: { { -#line 600 "Parser.rl" +#line 601 "Parser.rl" {te = p+1;{ -#line 600 "Parser.rl" +#line 601 "Parser.rl" currentCls->add(CLASS_UCP_NO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4859 "Parser.cpp" +#line 4860 "Parser.cpp" break; } case 48: { { -#line 602 "Parser.rl" +#line 603 "Parser.rl" {te = p+1;{ -#line 602 "Parser.rl" +#line 603 "Parser.rl" currentCls->add(CLASS_UCP_PC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4870 "Parser.cpp" +#line 4871 "Parser.cpp" break; } case 49: { { -#line 603 "Parser.rl" +#line 604 "Parser.rl" {te = p+1;{ -#line 603 "Parser.rl" +#line 604 "Parser.rl" currentCls->add(CLASS_UCP_PD, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4881 "Parser.cpp" +#line 4882 "Parser.cpp" break; } case 50: { { -#line 604 "Parser.rl" +#line 605 "Parser.rl" {te = p+1;{ -#line 604 "Parser.rl" +#line 605 "Parser.rl" currentCls->add(CLASS_UCP_PE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4892 "Parser.cpp" +#line 4893 "Parser.cpp" break; } case 51: { { -#line 605 "Parser.rl" +#line 606 "Parser.rl" {te = p+1;{ -#line 605 "Parser.rl" +#line 606 "Parser.rl" currentCls->add(CLASS_UCP_PF, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4903 "Parser.cpp" +#line 4904 "Parser.cpp" break; } case 52: { { -#line 606 "Parser.rl" +#line 607 "Parser.rl" {te = p+1;{ -#line 606 "Parser.rl" +#line 607 "Parser.rl" currentCls->add(CLASS_UCP_PI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4914 "Parser.cpp" +#line 4915 "Parser.cpp" break; } case 53: { { -#line 607 "Parser.rl" +#line 608 "Parser.rl" {te = p+1;{ -#line 607 "Parser.rl" +#line 608 "Parser.rl" currentCls->add(CLASS_UCP_PO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4925 "Parser.cpp" +#line 4926 "Parser.cpp" break; } case 54: { { -#line 608 "Parser.rl" +#line 609 "Parser.rl" {te = p+1;{ -#line 608 "Parser.rl" +#line 609 "Parser.rl" currentCls->add(CLASS_UCP_PS, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4936 "Parser.cpp" +#line 4937 "Parser.cpp" break; } case 55: { { -#line 610 "Parser.rl" +#line 611 "Parser.rl" {te = p+1;{ -#line 610 "Parser.rl" +#line 611 "Parser.rl" currentCls->add(CLASS_UCP_SC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4947 "Parser.cpp" +#line 4948 "Parser.cpp" break; } case 56: { { -#line 611 "Parser.rl" +#line 612 "Parser.rl" {te = p+1;{ -#line 611 "Parser.rl" +#line 612 "Parser.rl" currentCls->add(CLASS_UCP_SK, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4958 "Parser.cpp" +#line 4959 "Parser.cpp" break; } case 57: { { -#line 612 "Parser.rl" +#line 613 "Parser.rl" {te = p+1;{ -#line 612 "Parser.rl" +#line 613 "Parser.rl" currentCls->add(CLASS_UCP_SM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4969 "Parser.cpp" +#line 4970 "Parser.cpp" break; } case 58: { { -#line 613 "Parser.rl" +#line 614 "Parser.rl" {te = p+1;{ -#line 613 "Parser.rl" +#line 614 "Parser.rl" currentCls->add(CLASS_UCP_SO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4980 "Parser.cpp" +#line 4981 "Parser.cpp" break; } case 59: { { -#line 615 "Parser.rl" +#line 616 "Parser.rl" {te = p+1;{ -#line 615 "Parser.rl" +#line 616 "Parser.rl" currentCls->add(CLASS_UCP_ZL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4991 "Parser.cpp" +#line 4992 "Parser.cpp" break; } case 60: { { -#line 616 "Parser.rl" +#line 617 "Parser.rl" {te = p+1;{ -#line 616 "Parser.rl" +#line 617 "Parser.rl" currentCls->add(CLASS_UCP_ZP, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5002 "Parser.cpp" +#line 5003 "Parser.cpp" break; } case 61: { { -#line 617 "Parser.rl" +#line 618 "Parser.rl" {te = p+1;{ -#line 617 "Parser.rl" +#line 618 "Parser.rl" currentCls->add(CLASS_UCP_ZS, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5013 "Parser.cpp" +#line 5014 "Parser.cpp" break; } case 62: { { -#line 618 "Parser.rl" +#line 619 "Parser.rl" {te = p+1;{ -#line 618 "Parser.rl" +#line 619 "Parser.rl" currentCls->add(CLASS_UCP_XAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5024 "Parser.cpp" +#line 5025 "Parser.cpp" break; } case 63: { { -#line 619 "Parser.rl" +#line 620 "Parser.rl" {te = p+1;{ -#line 619 "Parser.rl" +#line 620 "Parser.rl" currentCls->add(CLASS_UCP_XPS, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5035 "Parser.cpp" +#line 5036 "Parser.cpp" break; } case 64: { { -#line 620 "Parser.rl" +#line 621 "Parser.rl" {te = p+1;{ -#line 620 "Parser.rl" +#line 621 "Parser.rl" currentCls->add(CLASS_UCP_XSP, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5046 "Parser.cpp" +#line 5047 "Parser.cpp" break; } case 65: { { -#line 621 "Parser.rl" +#line 622 "Parser.rl" {te = p+1;{ -#line 621 "Parser.rl" +#line 622 "Parser.rl" currentCls->add(CLASS_UCP_XWD, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5057 "Parser.cpp" +#line 5058 "Parser.cpp" break; } case 66: { { -#line 622 "Parser.rl" +#line 623 "Parser.rl" {te = p+1;{ -#line 622 "Parser.rl" +#line 623 "Parser.rl" currentCls->add(CLASS_SCRIPT_ARABIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5068 "Parser.cpp" +#line 5069 "Parser.cpp" break; } case 67: { { -#line 623 "Parser.rl" +#line 624 "Parser.rl" {te = p+1;{ -#line 623 "Parser.rl" +#line 624 "Parser.rl" currentCls->add(CLASS_SCRIPT_ARMENIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5079 "Parser.cpp" +#line 5080 "Parser.cpp" break; } case 68: { { -#line 624 "Parser.rl" +#line 625 "Parser.rl" {te = p+1;{ -#line 624 "Parser.rl" +#line 625 "Parser.rl" currentCls->add(CLASS_SCRIPT_AVESTAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5090 "Parser.cpp" +#line 5091 "Parser.cpp" break; } case 69: { { -#line 625 "Parser.rl" +#line 626 "Parser.rl" {te = p+1;{ -#line 625 "Parser.rl" +#line 626 "Parser.rl" currentCls->add(CLASS_SCRIPT_BALINESE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5101 "Parser.cpp" +#line 5102 "Parser.cpp" break; } case 70: { { -#line 626 "Parser.rl" +#line 627 "Parser.rl" {te = p+1;{ -#line 626 "Parser.rl" +#line 627 "Parser.rl" currentCls->add(CLASS_SCRIPT_BAMUM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5112 "Parser.cpp" +#line 5113 "Parser.cpp" break; } case 71: { { -#line 627 "Parser.rl" +#line 628 "Parser.rl" {te = p+1;{ -#line 627 "Parser.rl" +#line 628 "Parser.rl" currentCls->add(CLASS_SCRIPT_BATAK, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5123 "Parser.cpp" +#line 5124 "Parser.cpp" break; } case 72: { { -#line 628 "Parser.rl" +#line 629 "Parser.rl" {te = p+1;{ -#line 628 "Parser.rl" +#line 629 "Parser.rl" currentCls->add(CLASS_SCRIPT_BENGALI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5134 "Parser.cpp" +#line 5135 "Parser.cpp" break; } case 73: { { -#line 629 "Parser.rl" +#line 630 "Parser.rl" {te = p+1;{ -#line 629 "Parser.rl" +#line 630 "Parser.rl" currentCls->add(CLASS_SCRIPT_BOPOMOFO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5145 "Parser.cpp" +#line 5146 "Parser.cpp" break; } case 74: { { -#line 630 "Parser.rl" +#line 631 "Parser.rl" {te = p+1;{ -#line 630 "Parser.rl" +#line 631 "Parser.rl" currentCls->add(CLASS_SCRIPT_BRAHMI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5156 "Parser.cpp" +#line 5157 "Parser.cpp" break; } case 75: { { -#line 631 "Parser.rl" +#line 632 "Parser.rl" {te = p+1;{ -#line 631 "Parser.rl" +#line 632 "Parser.rl" currentCls->add(CLASS_SCRIPT_BRAILLE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5167 "Parser.cpp" +#line 5168 "Parser.cpp" break; } case 76: { { -#line 632 "Parser.rl" +#line 633 "Parser.rl" {te = p+1;{ -#line 632 "Parser.rl" +#line 633 "Parser.rl" currentCls->add(CLASS_SCRIPT_BUGINESE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5178 "Parser.cpp" +#line 5179 "Parser.cpp" break; } case 77: { { -#line 633 "Parser.rl" +#line 634 "Parser.rl" {te = p+1;{ -#line 633 "Parser.rl" +#line 634 "Parser.rl" currentCls->add(CLASS_SCRIPT_BUHID, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5189 "Parser.cpp" +#line 5190 "Parser.cpp" break; } case 78: { { -#line 634 "Parser.rl" +#line 635 "Parser.rl" {te = p+1;{ -#line 634 "Parser.rl" +#line 635 "Parser.rl" currentCls->add(CLASS_SCRIPT_CANADIAN_ABORIGINAL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5200 "Parser.cpp" +#line 5201 "Parser.cpp" break; } case 79: { { -#line 635 "Parser.rl" +#line 636 "Parser.rl" {te = p+1;{ -#line 635 "Parser.rl" +#line 636 "Parser.rl" currentCls->add(CLASS_SCRIPT_CARIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5211 "Parser.cpp" +#line 5212 "Parser.cpp" break; } case 80: { { -#line 636 "Parser.rl" +#line 637 "Parser.rl" {te = p+1;{ -#line 636 "Parser.rl" +#line 637 "Parser.rl" currentCls->add(CLASS_SCRIPT_CHAM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5222 "Parser.cpp" +#line 5223 "Parser.cpp" break; } case 81: { { -#line 637 "Parser.rl" +#line 638 "Parser.rl" {te = p+1;{ -#line 637 "Parser.rl" +#line 638 "Parser.rl" currentCls->add(CLASS_SCRIPT_CHEROKEE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5233 "Parser.cpp" +#line 5234 "Parser.cpp" break; } case 82: { { -#line 638 "Parser.rl" +#line 639 "Parser.rl" {te = p+1;{ -#line 638 "Parser.rl" +#line 639 "Parser.rl" currentCls->add(CLASS_SCRIPT_COMMON, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5244 "Parser.cpp" +#line 5245 "Parser.cpp" break; } case 83: { { -#line 639 "Parser.rl" +#line 640 "Parser.rl" {te = p+1;{ -#line 639 "Parser.rl" +#line 640 "Parser.rl" currentCls->add(CLASS_SCRIPT_COPTIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5255 "Parser.cpp" +#line 5256 "Parser.cpp" break; } case 84: { { -#line 640 "Parser.rl" +#line 641 "Parser.rl" {te = p+1;{ -#line 640 "Parser.rl" +#line 641 "Parser.rl" currentCls->add(CLASS_SCRIPT_CUNEIFORM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5266 "Parser.cpp" +#line 5267 "Parser.cpp" break; } case 85: { { -#line 641 "Parser.rl" +#line 642 "Parser.rl" {te = p+1;{ -#line 641 "Parser.rl" +#line 642 "Parser.rl" currentCls->add(CLASS_SCRIPT_CYPRIOT, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5277 "Parser.cpp" +#line 5278 "Parser.cpp" break; } case 86: { { -#line 642 "Parser.rl" +#line 643 "Parser.rl" {te = p+1;{ -#line 642 "Parser.rl" +#line 643 "Parser.rl" currentCls->add(CLASS_SCRIPT_CYRILLIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5288 "Parser.cpp" +#line 5289 "Parser.cpp" break; } case 87: { { -#line 643 "Parser.rl" +#line 644 "Parser.rl" {te = p+1;{ -#line 643 "Parser.rl" +#line 644 "Parser.rl" currentCls->add(CLASS_SCRIPT_DESERET, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5299 "Parser.cpp" +#line 5300 "Parser.cpp" break; } case 88: { { -#line 644 "Parser.rl" +#line 645 "Parser.rl" {te = p+1;{ -#line 644 "Parser.rl" +#line 645 "Parser.rl" currentCls->add(CLASS_SCRIPT_DEVANAGARI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5310 "Parser.cpp" +#line 5311 "Parser.cpp" break; } case 89: { { -#line 645 "Parser.rl" +#line 646 "Parser.rl" {te = p+1;{ -#line 645 "Parser.rl" +#line 646 "Parser.rl" currentCls->add(CLASS_SCRIPT_EGYPTIAN_HIEROGLYPHS, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5321 "Parser.cpp" +#line 5322 "Parser.cpp" break; } case 90: { { -#line 646 "Parser.rl" +#line 647 "Parser.rl" {te = p+1;{ -#line 646 "Parser.rl" +#line 647 "Parser.rl" currentCls->add(CLASS_SCRIPT_ETHIOPIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5332 "Parser.cpp" +#line 5333 "Parser.cpp" break; } case 91: { { -#line 647 "Parser.rl" +#line 648 "Parser.rl" {te = p+1;{ -#line 647 "Parser.rl" +#line 648 "Parser.rl" currentCls->add(CLASS_SCRIPT_GEORGIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5343 "Parser.cpp" +#line 5344 "Parser.cpp" break; } case 92: { { -#line 648 "Parser.rl" +#line 649 "Parser.rl" {te = p+1;{ -#line 648 "Parser.rl" +#line 649 "Parser.rl" currentCls->add(CLASS_SCRIPT_GLAGOLITIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5354 "Parser.cpp" +#line 5355 "Parser.cpp" break; } case 93: { { -#line 649 "Parser.rl" +#line 650 "Parser.rl" {te = p+1;{ -#line 649 "Parser.rl" +#line 650 "Parser.rl" currentCls->add(CLASS_SCRIPT_GOTHIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5365 "Parser.cpp" +#line 5366 "Parser.cpp" break; } case 94: { { -#line 650 "Parser.rl" +#line 651 "Parser.rl" {te = p+1;{ -#line 650 "Parser.rl" +#line 651 "Parser.rl" currentCls->add(CLASS_SCRIPT_GREEK, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5376 "Parser.cpp" +#line 5377 "Parser.cpp" break; } case 95: { { -#line 651 "Parser.rl" +#line 652 "Parser.rl" {te = p+1;{ -#line 651 "Parser.rl" +#line 652 "Parser.rl" currentCls->add(CLASS_SCRIPT_GUJARATI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5387 "Parser.cpp" +#line 5388 "Parser.cpp" break; } case 96: { { -#line 652 "Parser.rl" +#line 653 "Parser.rl" {te = p+1;{ -#line 652 "Parser.rl" +#line 653 "Parser.rl" currentCls->add(CLASS_SCRIPT_GURMUKHI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5398 "Parser.cpp" +#line 5399 "Parser.cpp" break; } case 97: { { -#line 654 "Parser.rl" +#line 655 "Parser.rl" {te = p+1;{ -#line 654 "Parser.rl" +#line 655 "Parser.rl" currentCls->add(CLASS_SCRIPT_HANGUL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5409 "Parser.cpp" +#line 5410 "Parser.cpp" break; } case 98: { { -#line 655 "Parser.rl" +#line 656 "Parser.rl" {te = p+1;{ -#line 655 "Parser.rl" +#line 656 "Parser.rl" currentCls->add(CLASS_SCRIPT_HANUNOO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5420 "Parser.cpp" +#line 5421 "Parser.cpp" break; } case 99: { { -#line 656 "Parser.rl" +#line 657 "Parser.rl" {te = p+1;{ -#line 656 "Parser.rl" +#line 657 "Parser.rl" currentCls->add(CLASS_SCRIPT_HEBREW, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5431 "Parser.cpp" +#line 5432 "Parser.cpp" break; } case 100: { { -#line 657 "Parser.rl" +#line 658 "Parser.rl" {te = p+1;{ -#line 657 "Parser.rl" +#line 658 "Parser.rl" currentCls->add(CLASS_SCRIPT_HIRAGANA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5442 "Parser.cpp" +#line 5443 "Parser.cpp" break; } case 101: { { -#line 658 "Parser.rl" +#line 659 "Parser.rl" {te = p+1;{ -#line 658 "Parser.rl" +#line 659 "Parser.rl" currentCls->add(CLASS_SCRIPT_IMPERIAL_ARAMAIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5453 "Parser.cpp" +#line 5454 "Parser.cpp" break; } case 102: { { -#line 659 "Parser.rl" +#line 660 "Parser.rl" {te = p+1;{ -#line 659 "Parser.rl" +#line 660 "Parser.rl" currentCls->add(CLASS_SCRIPT_INHERITED, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5464 "Parser.cpp" +#line 5465 "Parser.cpp" break; } case 103: { { -#line 660 "Parser.rl" +#line 661 "Parser.rl" {te = p+1;{ -#line 660 "Parser.rl" +#line 661 "Parser.rl" currentCls->add(CLASS_SCRIPT_INSCRIPTIONAL_PAHLAVI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5475 "Parser.cpp" +#line 5476 "Parser.cpp" break; } case 104: { { -#line 661 "Parser.rl" +#line 662 "Parser.rl" {te = p+1;{ -#line 661 "Parser.rl" +#line 662 "Parser.rl" currentCls->add(CLASS_SCRIPT_INSCRIPTIONAL_PARTHIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5486 "Parser.cpp" +#line 5487 "Parser.cpp" break; } case 105: { { -#line 662 "Parser.rl" +#line 663 "Parser.rl" {te = p+1;{ -#line 662 "Parser.rl" +#line 663 "Parser.rl" currentCls->add(CLASS_SCRIPT_JAVANESE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5497 "Parser.cpp" +#line 5498 "Parser.cpp" break; } case 106: { { -#line 663 "Parser.rl" +#line 664 "Parser.rl" {te = p+1;{ -#line 663 "Parser.rl" +#line 664 "Parser.rl" currentCls->add(CLASS_SCRIPT_KAITHI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5508 "Parser.cpp" +#line 5509 "Parser.cpp" break; } case 107: { { -#line 664 "Parser.rl" +#line 665 "Parser.rl" {te = p+1;{ -#line 664 "Parser.rl" +#line 665 "Parser.rl" currentCls->add(CLASS_SCRIPT_KANNADA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5519 "Parser.cpp" +#line 5520 "Parser.cpp" break; } case 108: { { -#line 665 "Parser.rl" +#line 666 "Parser.rl" {te = p+1;{ -#line 665 "Parser.rl" +#line 666 "Parser.rl" currentCls->add(CLASS_SCRIPT_KATAKANA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5530 "Parser.cpp" +#line 5531 "Parser.cpp" break; } case 109: { { -#line 666 "Parser.rl" +#line 667 "Parser.rl" {te = p+1;{ -#line 666 "Parser.rl" +#line 667 "Parser.rl" currentCls->add(CLASS_SCRIPT_KAYAH_LI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5541 "Parser.cpp" +#line 5542 "Parser.cpp" break; } case 110: { { -#line 667 "Parser.rl" +#line 668 "Parser.rl" {te = p+1;{ -#line 667 "Parser.rl" +#line 668 "Parser.rl" currentCls->add(CLASS_SCRIPT_KHAROSHTHI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5552 "Parser.cpp" +#line 5553 "Parser.cpp" break; } case 111: { { -#line 668 "Parser.rl" +#line 669 "Parser.rl" {te = p+1;{ -#line 668 "Parser.rl" +#line 669 "Parser.rl" currentCls->add(CLASS_SCRIPT_KHMER, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5563 "Parser.cpp" +#line 5564 "Parser.cpp" break; } case 112: { { -#line 669 "Parser.rl" +#line 670 "Parser.rl" {te = p+1;{ -#line 669 "Parser.rl" +#line 670 "Parser.rl" currentCls->add(CLASS_SCRIPT_LAO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5574 "Parser.cpp" +#line 5575 "Parser.cpp" break; } case 113: { { -#line 670 "Parser.rl" +#line 671 "Parser.rl" {te = p+1;{ -#line 670 "Parser.rl" +#line 671 "Parser.rl" currentCls->add(CLASS_SCRIPT_LATIN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5585 "Parser.cpp" +#line 5586 "Parser.cpp" break; } case 114: { { -#line 671 "Parser.rl" +#line 672 "Parser.rl" {te = p+1;{ -#line 671 "Parser.rl" +#line 672 "Parser.rl" currentCls->add(CLASS_SCRIPT_LEPCHA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5596 "Parser.cpp" +#line 5597 "Parser.cpp" break; } case 115: { { -#line 672 "Parser.rl" +#line 673 "Parser.rl" {te = p+1;{ -#line 672 "Parser.rl" +#line 673 "Parser.rl" currentCls->add(CLASS_SCRIPT_LIMBU, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5607 "Parser.cpp" +#line 5608 "Parser.cpp" break; } case 116: { { -#line 673 "Parser.rl" +#line 674 "Parser.rl" {te = p+1;{ -#line 673 "Parser.rl" +#line 674 "Parser.rl" currentCls->add(CLASS_SCRIPT_LINEAR_B, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5618 "Parser.cpp" +#line 5619 "Parser.cpp" break; } case 117: { { -#line 674 "Parser.rl" +#line 675 "Parser.rl" {te = p+1;{ -#line 674 "Parser.rl" +#line 675 "Parser.rl" currentCls->add(CLASS_SCRIPT_LISU, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5629 "Parser.cpp" +#line 5630 "Parser.cpp" break; } case 118: { { -#line 675 "Parser.rl" +#line 676 "Parser.rl" {te = p+1;{ -#line 675 "Parser.rl" +#line 676 "Parser.rl" currentCls->add(CLASS_SCRIPT_LYCIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5640 "Parser.cpp" +#line 5641 "Parser.cpp" break; } case 119: { { -#line 676 "Parser.rl" +#line 677 "Parser.rl" {te = p+1;{ -#line 676 "Parser.rl" +#line 677 "Parser.rl" currentCls->add(CLASS_SCRIPT_LYDIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5651 "Parser.cpp" +#line 5652 "Parser.cpp" break; } case 120: { { -#line 677 "Parser.rl" +#line 678 "Parser.rl" {te = p+1;{ -#line 677 "Parser.rl" +#line 678 "Parser.rl" currentCls->add(CLASS_SCRIPT_MALAYALAM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5662 "Parser.cpp" +#line 5663 "Parser.cpp" break; } case 121: { { -#line 678 "Parser.rl" +#line 679 "Parser.rl" {te = p+1;{ -#line 678 "Parser.rl" +#line 679 "Parser.rl" currentCls->add(CLASS_SCRIPT_MANDAIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5673 "Parser.cpp" +#line 5674 "Parser.cpp" break; } case 122: { { -#line 679 "Parser.rl" +#line 680 "Parser.rl" {te = p+1;{ -#line 679 "Parser.rl" +#line 680 "Parser.rl" currentCls->add(CLASS_SCRIPT_MEETEI_MAYEK, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5684 "Parser.cpp" +#line 5685 "Parser.cpp" break; } case 123: { { -#line 680 "Parser.rl" +#line 681 "Parser.rl" {te = p+1;{ -#line 680 "Parser.rl" +#line 681 "Parser.rl" currentCls->add(CLASS_SCRIPT_MONGOLIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5695 "Parser.cpp" +#line 5696 "Parser.cpp" break; } case 124: { { -#line 681 "Parser.rl" +#line 682 "Parser.rl" {te = p+1;{ -#line 681 "Parser.rl" +#line 682 "Parser.rl" currentCls->add(CLASS_SCRIPT_MYANMAR, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5706 "Parser.cpp" +#line 5707 "Parser.cpp" break; } case 125: { { -#line 682 "Parser.rl" +#line 683 "Parser.rl" {te = p+1;{ -#line 682 "Parser.rl" +#line 683 "Parser.rl" currentCls->add(CLASS_SCRIPT_NEW_TAI_LUE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5717 "Parser.cpp" +#line 5718 "Parser.cpp" break; } case 126: { { -#line 683 "Parser.rl" +#line 684 "Parser.rl" {te = p+1;{ -#line 683 "Parser.rl" +#line 684 "Parser.rl" currentCls->add(CLASS_SCRIPT_NKO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5728 "Parser.cpp" +#line 5729 "Parser.cpp" break; } case 127: { { -#line 684 "Parser.rl" +#line 685 "Parser.rl" {te = p+1;{ -#line 684 "Parser.rl" +#line 685 "Parser.rl" currentCls->add(CLASS_SCRIPT_OGHAM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5739 "Parser.cpp" +#line 5740 "Parser.cpp" break; } case 128: { { -#line 685 "Parser.rl" +#line 686 "Parser.rl" {te = p+1;{ -#line 685 "Parser.rl" +#line 686 "Parser.rl" currentCls->add(CLASS_SCRIPT_OL_CHIKI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5750 "Parser.cpp" +#line 5751 "Parser.cpp" break; } case 129: { { -#line 686 "Parser.rl" +#line 687 "Parser.rl" {te = p+1;{ -#line 686 "Parser.rl" +#line 687 "Parser.rl" currentCls->add(CLASS_SCRIPT_OLD_ITALIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5761 "Parser.cpp" +#line 5762 "Parser.cpp" break; } case 130: { { -#line 687 "Parser.rl" +#line 688 "Parser.rl" {te = p+1;{ -#line 687 "Parser.rl" +#line 688 "Parser.rl" currentCls->add(CLASS_SCRIPT_OLD_PERSIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5772 "Parser.cpp" +#line 5773 "Parser.cpp" break; } case 131: { { -#line 688 "Parser.rl" +#line 689 "Parser.rl" {te = p+1;{ -#line 688 "Parser.rl" +#line 689 "Parser.rl" currentCls->add(CLASS_SCRIPT_OLD_SOUTH_ARABIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5783 "Parser.cpp" +#line 5784 "Parser.cpp" break; } case 132: { { -#line 689 "Parser.rl" +#line 690 "Parser.rl" {te = p+1;{ -#line 689 "Parser.rl" +#line 690 "Parser.rl" currentCls->add(CLASS_SCRIPT_OLD_TURKIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5794 "Parser.cpp" +#line 5795 "Parser.cpp" break; } case 133: { { -#line 690 "Parser.rl" +#line 691 "Parser.rl" {te = p+1;{ -#line 690 "Parser.rl" +#line 691 "Parser.rl" currentCls->add(CLASS_SCRIPT_ORIYA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5805 "Parser.cpp" +#line 5806 "Parser.cpp" break; } case 134: { { -#line 691 "Parser.rl" +#line 692 "Parser.rl" {te = p+1;{ -#line 691 "Parser.rl" +#line 692 "Parser.rl" currentCls->add(CLASS_SCRIPT_OSMANYA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5816 "Parser.cpp" +#line 5817 "Parser.cpp" break; } case 135: { { -#line 692 "Parser.rl" +#line 693 "Parser.rl" {te = p+1;{ -#line 692 "Parser.rl" +#line 693 "Parser.rl" currentCls->add(CLASS_SCRIPT_PHAGS_PA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5827 "Parser.cpp" +#line 5828 "Parser.cpp" break; } case 136: { { -#line 693 "Parser.rl" +#line 694 "Parser.rl" {te = p+1;{ -#line 693 "Parser.rl" +#line 694 "Parser.rl" currentCls->add(CLASS_SCRIPT_PHOENICIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5838 "Parser.cpp" +#line 5839 "Parser.cpp" break; } case 137: { { -#line 694 "Parser.rl" +#line 695 "Parser.rl" {te = p+1;{ -#line 694 "Parser.rl" +#line 695 "Parser.rl" currentCls->add(CLASS_SCRIPT_REJANG, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5849 "Parser.cpp" +#line 5850 "Parser.cpp" break; } case 138: { { -#line 695 "Parser.rl" +#line 696 "Parser.rl" {te = p+1;{ -#line 695 "Parser.rl" +#line 696 "Parser.rl" currentCls->add(CLASS_SCRIPT_RUNIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5860 "Parser.cpp" +#line 5861 "Parser.cpp" break; } case 139: { { -#line 696 "Parser.rl" +#line 697 "Parser.rl" {te = p+1;{ -#line 696 "Parser.rl" +#line 697 "Parser.rl" currentCls->add(CLASS_SCRIPT_SAMARITAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5871 "Parser.cpp" +#line 5872 "Parser.cpp" break; } case 140: { { -#line 697 "Parser.rl" +#line 698 "Parser.rl" {te = p+1;{ -#line 697 "Parser.rl" +#line 698 "Parser.rl" currentCls->add(CLASS_SCRIPT_SAURASHTRA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5882 "Parser.cpp" +#line 5883 "Parser.cpp" break; } case 141: { { -#line 698 "Parser.rl" +#line 699 "Parser.rl" {te = p+1;{ -#line 698 "Parser.rl" +#line 699 "Parser.rl" currentCls->add(CLASS_SCRIPT_SHAVIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5893 "Parser.cpp" +#line 5894 "Parser.cpp" break; } case 142: { { -#line 699 "Parser.rl" +#line 700 "Parser.rl" {te = p+1;{ -#line 699 "Parser.rl" +#line 700 "Parser.rl" currentCls->add(CLASS_SCRIPT_SINHALA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5904 "Parser.cpp" +#line 5905 "Parser.cpp" break; } case 143: { { -#line 700 "Parser.rl" +#line 701 "Parser.rl" {te = p+1;{ -#line 700 "Parser.rl" +#line 701 "Parser.rl" currentCls->add(CLASS_SCRIPT_SUNDANESE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5915 "Parser.cpp" +#line 5916 "Parser.cpp" break; } case 144: { { -#line 701 "Parser.rl" +#line 702 "Parser.rl" {te = p+1;{ -#line 701 "Parser.rl" +#line 702 "Parser.rl" currentCls->add(CLASS_SCRIPT_SYLOTI_NAGRI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5926 "Parser.cpp" +#line 5927 "Parser.cpp" break; } case 145: { { -#line 702 "Parser.rl" +#line 703 "Parser.rl" {te = p+1;{ -#line 702 "Parser.rl" +#line 703 "Parser.rl" currentCls->add(CLASS_SCRIPT_SYRIAC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5937 "Parser.cpp" +#line 5938 "Parser.cpp" break; } case 146: { { -#line 703 "Parser.rl" +#line 704 "Parser.rl" {te = p+1;{ -#line 703 "Parser.rl" +#line 704 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAGALOG, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5948 "Parser.cpp" +#line 5949 "Parser.cpp" break; } case 147: { { -#line 704 "Parser.rl" +#line 705 "Parser.rl" {te = p+1;{ -#line 704 "Parser.rl" +#line 705 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAGBANWA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5959 "Parser.cpp" +#line 5960 "Parser.cpp" break; } case 148: { { -#line 705 "Parser.rl" +#line 706 "Parser.rl" {te = p+1;{ -#line 705 "Parser.rl" +#line 706 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAI_LE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5970 "Parser.cpp" +#line 5971 "Parser.cpp" break; } case 149: { { -#line 706 "Parser.rl" +#line 707 "Parser.rl" {te = p+1;{ -#line 706 "Parser.rl" +#line 707 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAI_THAM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5981 "Parser.cpp" +#line 5982 "Parser.cpp" break; } case 150: { { -#line 707 "Parser.rl" +#line 708 "Parser.rl" {te = p+1;{ -#line 707 "Parser.rl" +#line 708 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAI_VIET, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5992 "Parser.cpp" +#line 5993 "Parser.cpp" break; } case 151: { { -#line 708 "Parser.rl" +#line 709 "Parser.rl" {te = p+1;{ -#line 708 "Parser.rl" +#line 709 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAMIL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6003 "Parser.cpp" +#line 6004 "Parser.cpp" break; } case 152: { { -#line 709 "Parser.rl" +#line 710 "Parser.rl" {te = p+1;{ -#line 709 "Parser.rl" +#line 710 "Parser.rl" currentCls->add(CLASS_SCRIPT_TELUGU, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6014 "Parser.cpp" +#line 6015 "Parser.cpp" break; } case 153: { { -#line 710 "Parser.rl" +#line 711 "Parser.rl" {te = p+1;{ -#line 710 "Parser.rl" +#line 711 "Parser.rl" currentCls->add(CLASS_SCRIPT_THAANA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6025 "Parser.cpp" +#line 6026 "Parser.cpp" break; } case 154: { { -#line 711 "Parser.rl" +#line 712 "Parser.rl" {te = p+1;{ -#line 711 "Parser.rl" +#line 712 "Parser.rl" currentCls->add(CLASS_SCRIPT_THAI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6036 "Parser.cpp" +#line 6037 "Parser.cpp" break; } case 155: { { -#line 712 "Parser.rl" +#line 713 "Parser.rl" {te = p+1;{ -#line 712 "Parser.rl" +#line 713 "Parser.rl" currentCls->add(CLASS_SCRIPT_TIBETAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6047 "Parser.cpp" +#line 6048 "Parser.cpp" break; } case 156: { { -#line 713 "Parser.rl" +#line 714 "Parser.rl" {te = p+1;{ -#line 713 "Parser.rl" +#line 714 "Parser.rl" currentCls->add(CLASS_SCRIPT_TIFINAGH, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6058 "Parser.cpp" +#line 6059 "Parser.cpp" break; } case 157: { { -#line 714 "Parser.rl" +#line 715 "Parser.rl" {te = p+1;{ -#line 714 "Parser.rl" +#line 715 "Parser.rl" currentCls->add(CLASS_SCRIPT_UGARITIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6069 "Parser.cpp" +#line 6070 "Parser.cpp" break; } case 158: { { -#line 715 "Parser.rl" +#line 716 "Parser.rl" {te = p+1;{ -#line 715 "Parser.rl" +#line 716 "Parser.rl" currentCls->add(CLASS_SCRIPT_VAI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6080 "Parser.cpp" +#line 6081 "Parser.cpp" break; } case 159: { { -#line 716 "Parser.rl" +#line 717 "Parser.rl" {te = p+1;{ -#line 716 "Parser.rl" +#line 717 "Parser.rl" currentCls->add(CLASS_SCRIPT_YI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6091 "Parser.cpp" +#line 6092 "Parser.cpp" break; } case 160: { { -#line 717 "Parser.rl" +#line 718 "Parser.rl" {te = p+1;{ -#line 717 "Parser.rl" +#line 718 "Parser.rl" currentCls->add(CLASS_UCP_ANY, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6102 "Parser.cpp" +#line 6103 "Parser.cpp" break; } case 161: { { -#line 718 "Parser.rl" +#line 719 "Parser.rl" {te = p+1;{ -#line 718 "Parser.rl" +#line 719 "Parser.rl" throw LocatedParseError("Unknown property"); } }} -#line 6113 "Parser.cpp" +#line 6114 "Parser.cpp" break; } case 162: { { -#line 580 "Parser.rl" +#line 581 "Parser.rl" {te = p;p = p - 1;{ -#line 580 "Parser.rl" +#line 581 "Parser.rl" currentCls->add(CLASS_UCP_C, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6124 "Parser.cpp" +#line 6125 "Parser.cpp" break; } case 163: { { -#line 584 "Parser.rl" +#line 585 "Parser.rl" {te = p;p = p - 1;{ -#line 584 "Parser.rl" +#line 585 "Parser.rl" currentCls->add(CLASS_UCP_CO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6135 "Parser.cpp" +#line 6136 "Parser.cpp" break; } case 164: { { -#line 586 "Parser.rl" +#line 587 "Parser.rl" {te = p;p = p - 1;{ -#line 586 "Parser.rl" +#line 587 "Parser.rl" currentCls->add(CLASS_UCP_L, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6146 "Parser.cpp" +#line 6147 "Parser.cpp" break; } case 165: { { -#line 593 "Parser.rl" +#line 594 "Parser.rl" {te = p;p = p - 1;{ -#line 593 "Parser.rl" +#line 594 "Parser.rl" currentCls->add(CLASS_UCP_M, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6157 "Parser.cpp" +#line 6158 "Parser.cpp" break; } case 166: { { -#line 595 "Parser.rl" +#line 596 "Parser.rl" {te = p;p = p - 1;{ -#line 595 "Parser.rl" +#line 596 "Parser.rl" currentCls->add(CLASS_UCP_ME, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6168 "Parser.cpp" +#line 6169 "Parser.cpp" break; } case 167: { { -#line 597 "Parser.rl" +#line 598 "Parser.rl" {te = p;p = p - 1;{ -#line 597 "Parser.rl" +#line 598 "Parser.rl" currentCls->add(CLASS_UCP_N, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6179 "Parser.cpp" +#line 6180 "Parser.cpp" break; } case 168: { { -#line 601 "Parser.rl" +#line 602 "Parser.rl" {te = p;p = p - 1;{ -#line 601 "Parser.rl" +#line 602 "Parser.rl" currentCls->add(CLASS_UCP_P, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6190 "Parser.cpp" +#line 6191 "Parser.cpp" break; } case 169: { { -#line 609 "Parser.rl" +#line 610 "Parser.rl" {te = p;p = p - 1;{ -#line 609 "Parser.rl" +#line 610 "Parser.rl" currentCls->add(CLASS_UCP_S, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6201 "Parser.cpp" +#line 6202 "Parser.cpp" break; } case 170: { { -#line 614 "Parser.rl" +#line 615 "Parser.rl" {te = p;p = p - 1;{ -#line 614 "Parser.rl" +#line 615 "Parser.rl" currentCls->add(CLASS_UCP_Z, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6212 "Parser.cpp" +#line 6213 "Parser.cpp" break; } case 171: { { -#line 653 "Parser.rl" +#line 654 "Parser.rl" {te = p;p = p - 1;{ -#line 653 "Parser.rl" +#line 654 "Parser.rl" currentCls->add(CLASS_SCRIPT_HAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6223 "Parser.cpp" +#line 6224 "Parser.cpp" break; } case 172: { { -#line 718 "Parser.rl" +#line 719 "Parser.rl" {te = p;p = p - 1;{ -#line 718 "Parser.rl" +#line 719 "Parser.rl" throw LocatedParseError("Unknown property"); } }} -#line 6234 "Parser.cpp" +#line 6235 "Parser.cpp" break; } case 173: { { -#line 580 "Parser.rl" +#line 581 "Parser.rl" {p = ((te))-1; { -#line 580 "Parser.rl" +#line 581 "Parser.rl" currentCls->add(CLASS_UCP_C, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6246 "Parser.cpp" +#line 6247 "Parser.cpp" break; } case 174: { { -#line 584 "Parser.rl" +#line 585 "Parser.rl" {p = ((te))-1; { -#line 584 "Parser.rl" +#line 585 "Parser.rl" currentCls->add(CLASS_UCP_CO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6258 "Parser.cpp" +#line 6259 "Parser.cpp" break; } case 175: { { -#line 586 "Parser.rl" +#line 587 "Parser.rl" {p = ((te))-1; { -#line 586 "Parser.rl" +#line 587 "Parser.rl" currentCls->add(CLASS_UCP_L, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6270 "Parser.cpp" +#line 6271 "Parser.cpp" break; } case 176: { { -#line 593 "Parser.rl" +#line 594 "Parser.rl" {p = ((te))-1; { -#line 593 "Parser.rl" +#line 594 "Parser.rl" currentCls->add(CLASS_UCP_M, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6282 "Parser.cpp" +#line 6283 "Parser.cpp" break; } case 177: { { -#line 595 "Parser.rl" +#line 596 "Parser.rl" {p = ((te))-1; { -#line 595 "Parser.rl" +#line 596 "Parser.rl" currentCls->add(CLASS_UCP_ME, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6294 "Parser.cpp" +#line 6295 "Parser.cpp" break; } case 178: { { -#line 597 "Parser.rl" +#line 598 "Parser.rl" {p = ((te))-1; { -#line 597 "Parser.rl" +#line 598 "Parser.rl" currentCls->add(CLASS_UCP_N, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6306 "Parser.cpp" +#line 6307 "Parser.cpp" break; } case 179: { { -#line 601 "Parser.rl" +#line 602 "Parser.rl" {p = ((te))-1; { -#line 601 "Parser.rl" +#line 602 "Parser.rl" currentCls->add(CLASS_UCP_P, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6318 "Parser.cpp" +#line 6319 "Parser.cpp" break; } case 180: { { -#line 609 "Parser.rl" +#line 610 "Parser.rl" {p = ((te))-1; { -#line 609 "Parser.rl" +#line 610 "Parser.rl" currentCls->add(CLASS_UCP_S, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6330 "Parser.cpp" +#line 6331 "Parser.cpp" break; } case 181: { { -#line 653 "Parser.rl" +#line 654 "Parser.rl" {p = ((te))-1; { -#line 653 "Parser.rl" +#line 654 "Parser.rl" currentCls->add(CLASS_SCRIPT_HAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6342 "Parser.cpp" +#line 6343 "Parser.cpp" break; } case 182: { { -#line 718 "Parser.rl" +#line 719 "Parser.rl" {p = ((te))-1; { -#line 718 "Parser.rl" +#line 719 "Parser.rl" throw LocatedParseError("Unknown property"); } }} -#line 6354 "Parser.cpp" +#line 6355 "Parser.cpp" break; } case 183: { { -#line 733 "Parser.rl" +#line 734 "Parser.rl" {te = p+1;{ -#line 733 "Parser.rl" +#line 734 "Parser.rl" currentCls->add(CLASS_UCP_C, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6372 "Parser.cpp" +#line 6373 "Parser.cpp" break; } case 184: { { -#line 741 "Parser.rl" +#line 742 "Parser.rl" {te = p+1;{ -#line 741 "Parser.rl" +#line 742 "Parser.rl" currentCls->add(CLASS_UCP_L, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6390 "Parser.cpp" +#line 6391 "Parser.cpp" break; } case 185: { { -#line 749 "Parser.rl" +#line 750 "Parser.rl" {te = p+1;{ -#line 749 "Parser.rl" +#line 750 "Parser.rl" currentCls->add(CLASS_UCP_M, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6408 "Parser.cpp" +#line 6409 "Parser.cpp" break; } case 186: { { -#line 757 "Parser.rl" +#line 758 "Parser.rl" {te = p+1;{ -#line 757 "Parser.rl" +#line 758 "Parser.rl" currentCls->add(CLASS_UCP_N, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6426 "Parser.cpp" +#line 6427 "Parser.cpp" break; } case 187: { { -#line 765 "Parser.rl" +#line 766 "Parser.rl" {te = p+1;{ -#line 765 "Parser.rl" +#line 766 "Parser.rl" currentCls->add(CLASS_UCP_P, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6444 "Parser.cpp" +#line 6445 "Parser.cpp" break; } case 188: { { -#line 773 "Parser.rl" +#line 774 "Parser.rl" {te = p+1;{ -#line 773 "Parser.rl" +#line 774 "Parser.rl" currentCls->add(CLASS_UCP_S, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6462 "Parser.cpp" +#line 6463 "Parser.cpp" break; } case 189: { { -#line 781 "Parser.rl" +#line 782 "Parser.rl" {te = p+1;{ -#line 781 "Parser.rl" +#line 782 "Parser.rl" currentCls->add(CLASS_UCP_Z, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6480 "Parser.cpp" +#line 6481 "Parser.cpp" break; } case 190: { { -#line 790 "Parser.rl" +#line 791 "Parser.rl" {te = p+1;{ -#line 790 "Parser.rl" +#line 791 "Parser.rl" throw LocatedParseError("Unknown property"); } }} -#line 6491 "Parser.cpp" +#line 6492 "Parser.cpp" break; } case 191: { { -#line 796 "Parser.rl" +#line 797 "Parser.rl" {te = p+1;{ -#line 796 "Parser.rl" +#line 797 "Parser.rl" throw LocatedParseError("Unsupported POSIX collating " "element"); } }} -#line 6505 "Parser.cpp" +#line 6506 "Parser.cpp" break; } case 192: { { -#line 803 "Parser.rl" +#line 804 "Parser.rl" {te = p+1;{ -#line 803 "Parser.rl" +#line 804 "Parser.rl" currentCls->add(CLASS_ALNUM, false); } }} -#line 6518 "Parser.cpp" +#line 6519 "Parser.cpp" break; } case 193: { { -#line 806 "Parser.rl" +#line 807 "Parser.rl" {te = p+1;{ -#line 806 "Parser.rl" +#line 807 "Parser.rl" currentCls->add(CLASS_ALNUM, true); } }} -#line 6531 "Parser.cpp" +#line 6532 "Parser.cpp" break; } case 194: { { -#line 809 "Parser.rl" +#line 810 "Parser.rl" {te = p+1;{ -#line 809 "Parser.rl" +#line 810 "Parser.rl" currentCls->add(CLASS_ALPHA, false); } }} -#line 6544 "Parser.cpp" +#line 6545 "Parser.cpp" break; } case 195: { { -#line 812 "Parser.rl" +#line 813 "Parser.rl" {te = p+1;{ -#line 812 "Parser.rl" +#line 813 "Parser.rl" currentCls->add(CLASS_ALPHA, true); } }} -#line 6557 "Parser.cpp" +#line 6558 "Parser.cpp" break; } case 196: { { -#line 815 "Parser.rl" +#line 816 "Parser.rl" {te = p+1;{ -#line 815 "Parser.rl" +#line 816 "Parser.rl" currentCls->add(CLASS_ASCII, false); } }} -#line 6570 "Parser.cpp" +#line 6571 "Parser.cpp" break; } case 197: { { -#line 818 "Parser.rl" +#line 819 "Parser.rl" {te = p+1;{ -#line 818 "Parser.rl" +#line 819 "Parser.rl" currentCls->add(CLASS_ASCII, true); } }} -#line 6583 "Parser.cpp" +#line 6584 "Parser.cpp" break; } case 198: { { -#line 821 "Parser.rl" +#line 822 "Parser.rl" {te = p+1;{ -#line 821 "Parser.rl" +#line 822 "Parser.rl" currentCls->add(CLASS_BLANK, false); } }} -#line 6596 "Parser.cpp" +#line 6597 "Parser.cpp" break; } case 199: { { -#line 824 "Parser.rl" +#line 825 "Parser.rl" {te = p+1;{ -#line 824 "Parser.rl" +#line 825 "Parser.rl" currentCls->add(CLASS_BLANK, true); } }} -#line 6609 "Parser.cpp" +#line 6610 "Parser.cpp" break; } case 200: { { -#line 827 "Parser.rl" +#line 828 "Parser.rl" {te = p+1;{ -#line 827 "Parser.rl" +#line 828 "Parser.rl" currentCls->add(CLASS_CNTRL, false); } }} -#line 6622 "Parser.cpp" +#line 6623 "Parser.cpp" break; } case 201: { { -#line 830 "Parser.rl" +#line 831 "Parser.rl" {te = p+1;{ -#line 830 "Parser.rl" +#line 831 "Parser.rl" currentCls->add(CLASS_CNTRL, true); } }} -#line 6635 "Parser.cpp" +#line 6636 "Parser.cpp" break; } case 202: { { -#line 833 "Parser.rl" +#line 834 "Parser.rl" {te = p+1;{ -#line 833 "Parser.rl" +#line 834 "Parser.rl" currentCls->add(CLASS_DIGIT, false); } }} -#line 6648 "Parser.cpp" +#line 6649 "Parser.cpp" break; } case 203: { { -#line 836 "Parser.rl" +#line 837 "Parser.rl" {te = p+1;{ -#line 836 "Parser.rl" +#line 837 "Parser.rl" currentCls->add(CLASS_DIGIT, true); } }} -#line 6661 "Parser.cpp" +#line 6662 "Parser.cpp" break; } case 204: { { -#line 839 "Parser.rl" +#line 840 "Parser.rl" {te = p+1;{ -#line 839 "Parser.rl" +#line 840 "Parser.rl" currentCls->add(CLASS_GRAPH, false); } }} -#line 6674 "Parser.cpp" +#line 6675 "Parser.cpp" break; } case 205: { { -#line 842 "Parser.rl" +#line 843 "Parser.rl" {te = p+1;{ -#line 842 "Parser.rl" +#line 843 "Parser.rl" currentCls->add(CLASS_GRAPH, true); } }} -#line 6687 "Parser.cpp" +#line 6688 "Parser.cpp" break; } case 206: { { -#line 845 "Parser.rl" +#line 846 "Parser.rl" {te = p+1;{ -#line 845 "Parser.rl" +#line 846 "Parser.rl" currentCls->add(CLASS_LOWER, false); } }} -#line 6700 "Parser.cpp" +#line 6701 "Parser.cpp" break; } case 207: { { -#line 848 "Parser.rl" +#line 849 "Parser.rl" {te = p+1;{ -#line 848 "Parser.rl" +#line 849 "Parser.rl" currentCls->add(CLASS_LOWER, true); } }} -#line 6713 "Parser.cpp" +#line 6714 "Parser.cpp" break; } case 208: { { -#line 851 "Parser.rl" +#line 852 "Parser.rl" {te = p+1;{ -#line 851 "Parser.rl" +#line 852 "Parser.rl" currentCls->add(CLASS_PRINT, false); } }} -#line 6726 "Parser.cpp" +#line 6727 "Parser.cpp" break; } case 209: { { -#line 854 "Parser.rl" +#line 855 "Parser.rl" {te = p+1;{ -#line 854 "Parser.rl" +#line 855 "Parser.rl" currentCls->add(CLASS_PRINT, true); } }} -#line 6739 "Parser.cpp" +#line 6740 "Parser.cpp" break; } case 210: { { -#line 857 "Parser.rl" +#line 858 "Parser.rl" {te = p+1;{ -#line 857 "Parser.rl" +#line 858 "Parser.rl" currentCls->add(CLASS_PUNCT, false); } }} -#line 6752 "Parser.cpp" +#line 6753 "Parser.cpp" break; } case 211: { { -#line 860 "Parser.rl" +#line 861 "Parser.rl" {te = p+1;{ -#line 860 "Parser.rl" +#line 861 "Parser.rl" currentCls->add(CLASS_PUNCT, true); } }} -#line 6765 "Parser.cpp" +#line 6766 "Parser.cpp" break; } case 212: { { -#line 864 "Parser.rl" +#line 865 "Parser.rl" {te = p+1;{ -#line 864 "Parser.rl" +#line 865 "Parser.rl" currentCls->add(CLASS_SPACE, false); } }} -#line 6778 "Parser.cpp" +#line 6779 "Parser.cpp" break; } case 213: { { -#line 867 "Parser.rl" +#line 868 "Parser.rl" {te = p+1;{ -#line 867 "Parser.rl" +#line 868 "Parser.rl" currentCls->add(CLASS_SPACE, true); } }} -#line 6791 "Parser.cpp" +#line 6792 "Parser.cpp" break; } case 214: { { -#line 870 "Parser.rl" +#line 871 "Parser.rl" {te = p+1;{ -#line 870 "Parser.rl" +#line 871 "Parser.rl" currentCls->add(CLASS_UPPER, false); } }} -#line 6804 "Parser.cpp" +#line 6805 "Parser.cpp" break; } case 215: { { -#line 873 "Parser.rl" +#line 874 "Parser.rl" {te = p+1;{ -#line 873 "Parser.rl" +#line 874 "Parser.rl" currentCls->add(CLASS_UPPER, true); } }} -#line 6817 "Parser.cpp" +#line 6818 "Parser.cpp" break; } case 216: { { -#line 876 "Parser.rl" +#line 877 "Parser.rl" {te = p+1;{ -#line 876 "Parser.rl" +#line 877 "Parser.rl" currentCls->add(CLASS_WORD, false); } }} -#line 6830 "Parser.cpp" +#line 6831 "Parser.cpp" break; } case 217: { { -#line 879 "Parser.rl" +#line 880 "Parser.rl" {te = p+1;{ -#line 879 "Parser.rl" +#line 880 "Parser.rl" currentCls->add(CLASS_WORD, true); } }} -#line 6843 "Parser.cpp" +#line 6844 "Parser.cpp" break; } case 218: { { -#line 882 "Parser.rl" +#line 883 "Parser.rl" {te = p+1;{ -#line 882 "Parser.rl" +#line 883 "Parser.rl" currentCls->add(CLASS_XDIGIT, false); } }} -#line 6856 "Parser.cpp" +#line 6857 "Parser.cpp" break; } case 219: { { -#line 885 "Parser.rl" +#line 886 "Parser.rl" {te = p+1;{ -#line 885 "Parser.rl" +#line 886 "Parser.rl" currentCls->add(CLASS_XDIGIT, true); } }} -#line 6869 "Parser.cpp" +#line 6870 "Parser.cpp" break; } case 220: { { -#line 890 "Parser.rl" +#line 891 "Parser.rl" {te = p+1;{ -#line 890 "Parser.rl" +#line 891 "Parser.rl" throw LocatedParseError("Invalid POSIX named class"); } }} -#line 6882 "Parser.cpp" +#line 6883 "Parser.cpp" break; } case 221: { { -#line 893 "Parser.rl" +#line 894 "Parser.rl" {te = p+1;{ -#line 893 "Parser.rl" +#line 894 "Parser.rl" {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { stack.resize(2 * (top + 1)); } } - stack[top] = cs; top += 1;cs = 843;goto _again;}} + stack[top] = cs; top += 1;cs = 844;goto _again;}} }} -#line 6902 "Parser.cpp" +#line 6903 "Parser.cpp" break; } case 222: { { -#line 896 "Parser.rl" +#line 897 "Parser.rl" {te = p+1;{ -#line 896 "Parser.rl" +#line 897 "Parser.rl" /*noop*/} }} -#line 6913 "Parser.cpp" +#line 6914 "Parser.cpp" break; } case 223: { { -#line 898 "Parser.rl" +#line 899 "Parser.rl" {te = p+1;{ -#line 898 "Parser.rl" +#line 899 "Parser.rl" currentCls->add('\x08'); } }} -#line 6926 "Parser.cpp" +#line 6927 "Parser.cpp" break; } case 224: { { -#line 902 "Parser.rl" +#line 903 "Parser.rl" {te = p+1;{ -#line 902 "Parser.rl" +#line 903 "Parser.rl" currentCls->add('\x09'); } }} -#line 6939 "Parser.cpp" +#line 6940 "Parser.cpp" break; } case 225: { { -#line 906 "Parser.rl" +#line 907 "Parser.rl" {te = p+1;{ -#line 906 "Parser.rl" +#line 907 "Parser.rl" currentCls->add('\x0a'); } }} -#line 6952 "Parser.cpp" +#line 6953 "Parser.cpp" break; } case 226: { { -#line 910 "Parser.rl" +#line 911 "Parser.rl" {te = p+1;{ -#line 910 "Parser.rl" +#line 911 "Parser.rl" currentCls->add('\x0d'); } }} -#line 6965 "Parser.cpp" +#line 6966 "Parser.cpp" break; } case 227: { { -#line 914 "Parser.rl" +#line 915 "Parser.rl" {te = p+1;{ -#line 914 "Parser.rl" +#line 915 "Parser.rl" currentCls->add('\x0c'); } }} -#line 6978 "Parser.cpp" +#line 6979 "Parser.cpp" break; } case 228: { { -#line 918 "Parser.rl" +#line 919 "Parser.rl" {te = p+1;{ -#line 918 "Parser.rl" +#line 919 "Parser.rl" currentCls->add('\x07'); } }} -#line 6991 "Parser.cpp" +#line 6992 "Parser.cpp" break; } case 229: { { -#line 922 "Parser.rl" +#line 923 "Parser.rl" {te = p+1;{ -#line 922 "Parser.rl" +#line 923 "Parser.rl" currentCls->add('\x1b'); } }} -#line 7004 "Parser.cpp" +#line 7005 "Parser.cpp" break; } case 230: { { -#line 926 "Parser.rl" +#line 927 "Parser.rl" {te = p+1;{ -#line 926 "Parser.rl" +#line 927 "Parser.rl" currentCls->add(CLASS_HORZ, false); } }} -#line 7017 "Parser.cpp" +#line 7018 "Parser.cpp" break; } case 231: { { -#line 930 "Parser.rl" +#line 931 "Parser.rl" {te = p+1;{ -#line 930 "Parser.rl" +#line 931 "Parser.rl" currentCls->add(CLASS_HORZ, true); } }} -#line 7030 "Parser.cpp" +#line 7031 "Parser.cpp" break; } case 232: { { -#line 934 "Parser.rl" +#line 935 "Parser.rl" {te = p+1;{ -#line 934 "Parser.rl" +#line 935 "Parser.rl" currentCls->add(CLASS_VERT, false); } }} -#line 7043 "Parser.cpp" +#line 7044 "Parser.cpp" break; } case 233: { { -#line 938 "Parser.rl" +#line 939 "Parser.rl" {te = p+1;{ -#line 938 "Parser.rl" +#line 939 "Parser.rl" currentCls->add(CLASS_VERT, true); } }} -#line 7056 "Parser.cpp" +#line 7057 "Parser.cpp" break; } case 234: { { -#line 942 "Parser.rl" +#line 943 "Parser.rl" {te = p+1;{ -#line 942 "Parser.rl" +#line 943 "Parser.rl" negated = false; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -7320,20 +7321,20 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 559;goto _again;}} }} -#line 7078 "Parser.cpp" +#line 7079 "Parser.cpp" break; } case 235: { { -#line 948 "Parser.rl" +#line 949 "Parser.rl" {te = p+1;{ -#line 948 "Parser.rl" +#line 949 "Parser.rl" negated = false; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -7343,20 +7344,20 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 818;goto _again;}} }} -#line 7100 "Parser.cpp" +#line 7101 "Parser.cpp" break; } case 236: { { -#line 954 "Parser.rl" +#line 955 "Parser.rl" {te = p+1;{ -#line 954 "Parser.rl" +#line 955 "Parser.rl" negated = true; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -7366,20 +7367,20 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 559;goto _again;}} }} -#line 7122 "Parser.cpp" +#line 7123 "Parser.cpp" break; } case 237: { { -#line 960 "Parser.rl" +#line 961 "Parser.rl" {te = p+1;{ -#line 960 "Parser.rl" +#line 961 "Parser.rl" negated = true; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -7389,43 +7390,43 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 818;goto _again;}} }} -#line 7144 "Parser.cpp" +#line 7145 "Parser.cpp" break; } case 238: { { -#line 970 "Parser.rl" +#line 971 "Parser.rl" {te = p+1;{ -#line 970 "Parser.rl" +#line 971 "Parser.rl" currentCls->add(octAccumulator); } }} -#line 7157 "Parser.cpp" +#line 7158 "Parser.cpp" break; } case 239: { { -#line 973 "Parser.rl" +#line 974 "Parser.rl" {te = p+1;{ -#line 973 "Parser.rl" +#line 974 "Parser.rl" currentCls->add(octAccumulator); } }} -#line 7170 "Parser.cpp" +#line 7171 "Parser.cpp" break; } case 240: { { -#line 977 "Parser.rl" +#line 978 "Parser.rl" {te = p+1;{ -#line 977 "Parser.rl" +#line 978 "Parser.rl" string oct(ts + 3, te - ts - 4); unsigned long val; @@ -7441,29 +7442,29 @@ static const short _regex_actions[] = { } }} -#line 7193 "Parser.cpp" +#line 7194 "Parser.cpp" break; } case 241: { { -#line 997 "Parser.rl" +#line 998 "Parser.rl" {te = p+1;{ -#line 997 "Parser.rl" +#line 998 "Parser.rl" currentCls->add(accumulator); } }} -#line 7206 "Parser.cpp" +#line 7207 "Parser.cpp" break; } case 242: { { -#line 1001 "Parser.rl" +#line 1002 "Parser.rl" {te = p+1;{ -#line 1001 "Parser.rl" +#line 1002 "Parser.rl" // whatever we found here currentCls->add(*(ts + 1)); @@ -7471,15 +7472,15 @@ static const short _regex_actions[] = { } }} -#line 7221 "Parser.cpp" +#line 7222 "Parser.cpp" break; } case 243: { { -#line 1007 "Parser.rl" +#line 1008 "Parser.rl" {te = p+1;{ -#line 1007 "Parser.rl" +#line 1008 "Parser.rl" string hex(ts + 3, te - ts - 4); unsigned long val; @@ -7495,15 +7496,15 @@ static const short _regex_actions[] = { } }} -#line 7244 "Parser.cpp" +#line 7245 "Parser.cpp" break; } case 244: { { -#line 1025 "Parser.rl" +#line 1026 "Parser.rl" {te = p+1;{ -#line 1025 "Parser.rl" +#line 1026 "Parser.rl" if (te - ts < 3) { assert(te - ts == 2); @@ -7515,113 +7516,113 @@ static const short _regex_actions[] = { } }} -#line 7263 "Parser.cpp" +#line 7264 "Parser.cpp" break; } case 245: { { -#line 1035 "Parser.rl" +#line 1036 "Parser.rl" {te = p+1;{ -#line 1035 "Parser.rl" +#line 1036 "Parser.rl" currentCls->add(CLASS_WORD, false); } }} -#line 7276 "Parser.cpp" +#line 7277 "Parser.cpp" break; } case 246: { { -#line 1039 "Parser.rl" +#line 1040 "Parser.rl" {te = p+1;{ -#line 1039 "Parser.rl" +#line 1040 "Parser.rl" currentCls->add(CLASS_WORD, true); } }} -#line 7289 "Parser.cpp" +#line 7290 "Parser.cpp" break; } case 247: { { -#line 1043 "Parser.rl" +#line 1044 "Parser.rl" {te = p+1;{ -#line 1043 "Parser.rl" +#line 1044 "Parser.rl" currentCls->add(CLASS_SPACE, false); } }} -#line 7302 "Parser.cpp" +#line 7303 "Parser.cpp" break; } case 248: { { -#line 1047 "Parser.rl" +#line 1048 "Parser.rl" {te = p+1;{ -#line 1047 "Parser.rl" +#line 1048 "Parser.rl" currentCls->add(CLASS_SPACE, true); } }} -#line 7315 "Parser.cpp" +#line 7316 "Parser.cpp" break; } case 249: { { -#line 1051 "Parser.rl" +#line 1052 "Parser.rl" {te = p+1;{ -#line 1051 "Parser.rl" +#line 1052 "Parser.rl" currentCls->add(CLASS_DIGIT, false); } }} -#line 7328 "Parser.cpp" +#line 7329 "Parser.cpp" break; } case 250: { { -#line 1055 "Parser.rl" +#line 1056 "Parser.rl" {te = p+1;{ -#line 1055 "Parser.rl" +#line 1056 "Parser.rl" currentCls->add(CLASS_DIGIT, true); } }} -#line 7341 "Parser.cpp" +#line 7342 "Parser.cpp" break; } case 251: { { -#line 1058 "Parser.rl" +#line 1059 "Parser.rl" {te = p+1;{ -#line 1058 "Parser.rl" +#line 1059 "Parser.rl" currentCls->addDash(); } }} -#line 7354 "Parser.cpp" +#line 7355 "Parser.cpp" break; } case 252: { { -#line 276 "Parser.rl" +#line 277 "Parser.rl" {te = p+1;{ -#line 276 "Parser.rl" +#line 277 "Parser.rl" ostringstream str; str << "'\\" << *(ts + 1) << "' at index " << ts - ptr @@ -7630,15 +7631,15 @@ static const short _regex_actions[] = { } }} -#line 7370 "Parser.cpp" +#line 7371 "Parser.cpp" break; } case 253: { { -#line 276 "Parser.rl" +#line 277 "Parser.rl" {te = p+1;{ -#line 276 "Parser.rl" +#line 277 "Parser.rl" ostringstream str; str << "'\\" << *(ts + 1) << "' at index " << ts - ptr @@ -7647,15 +7648,15 @@ static const short _regex_actions[] = { } }} -#line 7386 "Parser.cpp" +#line 7387 "Parser.cpp" break; } case 254: { { -#line 276 "Parser.rl" +#line 277 "Parser.rl" {te = p+1;{ -#line 276 "Parser.rl" +#line 277 "Parser.rl" ostringstream str; str << "'\\" << *(ts + 1) << "' at index " << ts - ptr @@ -7664,214 +7665,226 @@ static const short _regex_actions[] = { } }} -#line 7402 "Parser.cpp" +#line 7403 "Parser.cpp" break; } case 255: { { -#line 1075 "Parser.rl" +#line 1076 "Parser.rl" {te = p+1;{ -#line 1075 "Parser.rl" +#line 1076 "Parser.rl" // add the literal char currentCls->add(*(ts + 1)); } }} -#line 7416 "Parser.cpp" +#line 7417 "Parser.cpp" break; } case 256: { { -#line 1081 "Parser.rl" +#line 1082 "Parser.rl" {te = p+1;{ -#line 1081 "Parser.rl" +#line 1082 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint2c(ts)); } }} -#line 7430 "Parser.cpp" +#line 7431 "Parser.cpp" break; } case 257: { { -#line 1086 "Parser.rl" +#line 1087 "Parser.rl" {te = p+1;{ -#line 1086 "Parser.rl" +#line 1087 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint3c(ts)); } }} -#line 7444 "Parser.cpp" +#line 7445 "Parser.cpp" break; } case 258: { { -#line 1091 "Parser.rl" +#line 1092 "Parser.rl" {te = p+1;{ -#line 1091 "Parser.rl" +#line 1092 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint4c(ts)); } }} -#line 7458 "Parser.cpp" +#line 7459 "Parser.cpp" break; } case 259: { { -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" {te = p+1;{ -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7472 "Parser.cpp" +#line 7473 "Parser.cpp" break; } case 260: { { -#line 1102 "Parser.rl" +#line 1106 "Parser.rl" + {te = p+1;{ +#line 1106 "Parser.rl" + throw LocatedParseError("Character class intersection/subtraction is not supported"); } + }} + +#line 7484 "Parser.cpp" + + break; + } + case 261: { + { +#line 1109 "Parser.rl" {te = p+1;{ -#line 1102 "Parser.rl" +#line 1109 "Parser.rl" currentCls->add((u8)*ts); } }} -#line 7485 "Parser.cpp" +#line 7497 "Parser.cpp" break; } - case 261: { + case 262: { { -#line 1106 "Parser.rl" +#line 1113 "Parser.rl" {te = p+1;{ -#line 1106 "Parser.rl" +#line 1113 "Parser.rl" currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); inCharClass = false; {cs = 746;goto _again;}} }} -#line 7500 "Parser.cpp" +#line 7512 "Parser.cpp" break; } - case 262: { + case 263: { { -#line 966 "Parser.rl" +#line 967 "Parser.rl" {te = p;p = p - 1;{ -#line 966 "Parser.rl" +#line 967 "Parser.rl" throw LocatedParseError("Malformed property"); } }} -#line 7511 "Parser.cpp" +#line 7523 "Parser.cpp" break; } - case 263: { + case 264: { { -#line 967 "Parser.rl" +#line 968 "Parser.rl" {te = p;p = p - 1;{ -#line 967 "Parser.rl" +#line 968 "Parser.rl" throw LocatedParseError("Malformed property"); } }} -#line 7522 "Parser.cpp" +#line 7534 "Parser.cpp" break; } - case 264: { + case 265: { { -#line 970 "Parser.rl" +#line 971 "Parser.rl" {te = p;p = p - 1;{ -#line 970 "Parser.rl" +#line 971 "Parser.rl" currentCls->add(octAccumulator); } }} -#line 7535 "Parser.cpp" +#line 7547 "Parser.cpp" break; } - case 265: { + case 266: { { -#line 973 "Parser.rl" +#line 974 "Parser.rl" {te = p;p = p - 1;{ -#line 973 "Parser.rl" +#line 974 "Parser.rl" currentCls->add(octAccumulator); } }} -#line 7548 "Parser.cpp" +#line 7560 "Parser.cpp" break; } - case 266: { + case 267: { { -#line 992 "Parser.rl" +#line 993 "Parser.rl" {te = p;p = p - 1;{ -#line 992 "Parser.rl" +#line 993 "Parser.rl" throw LocatedParseError("Value in \\o{...} sequence is non-octal or missing braces"); } }} -#line 7561 "Parser.cpp" +#line 7573 "Parser.cpp" break; } - case 267: { + case 268: { { -#line 997 "Parser.rl" +#line 998 "Parser.rl" {te = p;p = p - 1;{ -#line 997 "Parser.rl" +#line 998 "Parser.rl" currentCls->add(accumulator); } }} -#line 7574 "Parser.cpp" +#line 7586 "Parser.cpp" break; } - case 268: { + case 269: { { -#line 1021 "Parser.rl" +#line 1022 "Parser.rl" {te = p;p = p - 1;{ -#line 1021 "Parser.rl" +#line 1022 "Parser.rl" throw LocatedParseError("Value in \\x{...} sequence is non-hex or missing }"); } }} -#line 7587 "Parser.cpp" +#line 7599 "Parser.cpp" break; } - case 269: { + case 270: { { -#line 1025 "Parser.rl" +#line 1026 "Parser.rl" {te = p;p = p - 1;{ -#line 1025 "Parser.rl" +#line 1026 "Parser.rl" if (te - ts < 3) { assert(te - ts == 2); @@ -7883,105 +7896,115 @@ static const short _regex_actions[] = { } }} -#line 7606 "Parser.cpp" +#line 7618 "Parser.cpp" break; } - case 270: { + case 271: { { -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" {te = p;p = p - 1;{ -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7620 "Parser.cpp" +#line 7632 "Parser.cpp" break; } - case 271: { + case 272: { + { +#line 1103 "Parser.rl" + {te = p;p = p - 1;{ +#line 1103 "Parser.rl" + throw LocatedParseError("Nested character classes are not supported"); } + }} + +#line 7643 "Parser.cpp" + + break; + } + case 273: { { -#line 1102 "Parser.rl" +#line 1109 "Parser.rl" {te = p;p = p - 1;{ -#line 1102 "Parser.rl" +#line 1109 "Parser.rl" currentCls->add((u8)*ts); } }} -#line 7633 "Parser.cpp" +#line 7656 "Parser.cpp" break; } - case 272: { + case 274: { { -#line 992 "Parser.rl" +#line 993 "Parser.rl" {p = ((te))-1; { -#line 992 "Parser.rl" +#line 993 "Parser.rl" throw LocatedParseError("Value in \\o{...} sequence is non-octal or missing braces"); } }} -#line 7647 "Parser.cpp" +#line 7670 "Parser.cpp" break; } - case 273: { + case 275: { { -#line 1021 "Parser.rl" +#line 1022 "Parser.rl" {p = ((te))-1; { -#line 1021 "Parser.rl" +#line 1022 "Parser.rl" throw LocatedParseError("Value in \\x{...} sequence is non-hex or missing }"); } }} -#line 7661 "Parser.cpp" +#line 7684 "Parser.cpp" break; } - case 274: { + case 276: { { -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" {p = ((te))-1; { -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7676 "Parser.cpp" +#line 7699 "Parser.cpp" break; } - case 275: { + case 277: { { -#line 1102 "Parser.rl" +#line 1103 "Parser.rl" {p = ((te))-1; { -#line 1102 "Parser.rl" - - currentCls->add((u8)*ts); - } +#line 1103 "Parser.rl" + throw LocatedParseError("Nested character classes are not supported"); } }} -#line 7690 "Parser.cpp" +#line 7711 "Parser.cpp" break; } - case 276: { + case 278: { { -#line 1120 "Parser.rl" +#line 1127 "Parser.rl" {te = p+1;{ -#line 1120 "Parser.rl" +#line 1127 "Parser.rl" if (currentCls->isNegated()) { // Already seen a caret; the second one is not a meta-character. @@ -7994,250 +8017,250 @@ static const short _regex_actions[] = { } }} -#line 7710 "Parser.cpp" +#line 7731 "Parser.cpp" break; } - case 277: { + case 279: { { -#line 1133 "Parser.rl" +#line 1140 "Parser.rl" {te = p+1;{ -#line 1133 "Parser.rl" +#line 1140 "Parser.rl" currentCls->add(']'); inCharClassEarly = false; } }} -#line 7724 "Parser.cpp" +#line 7745 "Parser.cpp" break; } - case 278: { + case 280: { { -#line 1138 "Parser.rl" +#line 1145 "Parser.rl" {te = p+1;{ -#line 1138 "Parser.rl" +#line 1145 "Parser.rl" {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { stack.resize(2 * (top + 1)); } } - stack[top] = cs; top += 1;cs = 843;goto _again;}} + stack[top] = cs; top += 1;cs = 844;goto _again;}} }} -#line 7743 "Parser.cpp" +#line 7764 "Parser.cpp" break; } - case 279: { + case 281: { { -#line 1139 "Parser.rl" +#line 1146 "Parser.rl" {te = p+1;{ -#line 1139 "Parser.rl" +#line 1146 "Parser.rl" /*noop*/} }} -#line 7754 "Parser.cpp" +#line 7775 "Parser.cpp" break; } - case 280: { + case 282: { { -#line 1142 "Parser.rl" +#line 1149 "Parser.rl" {te = p+1;{ -#line 1142 "Parser.rl" +#line 1149 "Parser.rl" inCharClassEarly = false; {p = p - 1; } {cs = 819;goto _again;}} }} -#line 7768 "Parser.cpp" +#line 7789 "Parser.cpp" break; } - case 281: { + case 283: { { -#line 1142 "Parser.rl" +#line 1149 "Parser.rl" {te = p;p = p - 1;{ -#line 1142 "Parser.rl" +#line 1149 "Parser.rl" inCharClassEarly = false; {p = p - 1; } {cs = 819;goto _again;}} }} -#line 7782 "Parser.cpp" +#line 7803 "Parser.cpp" break; } - case 282: { + case 284: { { -#line 1154 "Parser.rl" +#line 1161 "Parser.rl" {te = p+1;{ -#line 1154 "Parser.rl" +#line 1161 "Parser.rl" {cs = 746;goto _again;}} }} -#line 7794 "Parser.cpp" +#line 7815 "Parser.cpp" break; } - case 283: { + case 285: { { -#line 1159 "Parser.rl" +#line 1166 "Parser.rl" {te = p+1;{ -#line 1159 "Parser.rl" +#line 1166 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint2c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 7812 "Parser.cpp" +#line 7833 "Parser.cpp" break; } - case 284: { + case 286: { { -#line 1168 "Parser.rl" +#line 1175 "Parser.rl" {te = p+1;{ -#line 1168 "Parser.rl" +#line 1175 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint3c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 7830 "Parser.cpp" +#line 7851 "Parser.cpp" break; } - case 285: { + case 287: { { -#line 1177 "Parser.rl" +#line 1184 "Parser.rl" {te = p+1;{ -#line 1177 "Parser.rl" +#line 1184 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint4c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 7848 "Parser.cpp" +#line 7869 "Parser.cpp" break; } - case 286: { + case 288: { { -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" {te = p+1;{ -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7862 "Parser.cpp" +#line 7883 "Parser.cpp" break; } - case 287: { + case 289: { { -#line 1192 "Parser.rl" +#line 1199 "Parser.rl" {te = p+1;{ -#line 1192 "Parser.rl" +#line 1199 "Parser.rl" addLiteral(currentSeq, *ts, mode); } }} -#line 7875 "Parser.cpp" +#line 7896 "Parser.cpp" break; } - case 288: { + case 290: { { -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" {te = p;p = p - 1;{ -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7889 "Parser.cpp" +#line 7910 "Parser.cpp" break; } - case 289: { + case 291: { { -#line 1192 "Parser.rl" +#line 1199 "Parser.rl" {te = p;p = p - 1;{ -#line 1192 "Parser.rl" +#line 1199 "Parser.rl" addLiteral(currentSeq, *ts, mode); } }} -#line 7902 "Parser.cpp" +#line 7923 "Parser.cpp" break; } - case 290: { + case 292: { { -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" {p = ((te))-1; { -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7917 "Parser.cpp" +#line 7938 "Parser.cpp" break; } - case 291: { + case 293: { { -#line 1202 "Parser.rl" +#line 1209 "Parser.rl" {te = p+1;{ -#line 1202 "Parser.rl" +#line 1209 "Parser.rl" {top -= 1;cs = stack[top];goto _again;} } }} -#line 7930 "Parser.cpp" +#line 7951 "Parser.cpp" break; } - case 292: { + case 294: { { -#line 1207 "Parser.rl" +#line 1214 "Parser.rl" {te = p+1;{ -#line 1207 "Parser.rl" +#line 1214 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint2c(ts)); @@ -8245,15 +8268,15 @@ static const short _regex_actions[] = { } }} -#line 7945 "Parser.cpp" +#line 7966 "Parser.cpp" break; } - case 293: { + case 295: { { -#line 1213 "Parser.rl" +#line 1220 "Parser.rl" {te = p+1;{ -#line 1213 "Parser.rl" +#line 1220 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint3c(ts)); @@ -8261,15 +8284,15 @@ static const short _regex_actions[] = { } }} -#line 7960 "Parser.cpp" +#line 7981 "Parser.cpp" break; } - case 294: { + case 296: { { -#line 1219 "Parser.rl" +#line 1226 "Parser.rl" {te = p+1;{ -#line 1219 "Parser.rl" +#line 1226 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint4c(ts)); @@ -8277,160 +8300,160 @@ static const short _regex_actions[] = { } }} -#line 7975 "Parser.cpp" +#line 7996 "Parser.cpp" break; } - case 295: { + case 297: { { -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" {te = p+1;{ -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7989 "Parser.cpp" +#line 8010 "Parser.cpp" break; } - case 296: { + case 298: { { -#line 1231 "Parser.rl" +#line 1238 "Parser.rl" {te = p+1;{ -#line 1231 "Parser.rl" +#line 1238 "Parser.rl" currentCls->add(*ts); inCharClassEarly = false; } }} -#line 8003 "Parser.cpp" +#line 8024 "Parser.cpp" break; } - case 297: { + case 299: { { -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" {te = p;p = p - 1;{ -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 8017 "Parser.cpp" +#line 8038 "Parser.cpp" break; } - case 298: { + case 300: { { -#line 1231 "Parser.rl" +#line 1238 "Parser.rl" {te = p;p = p - 1;{ -#line 1231 "Parser.rl" +#line 1238 "Parser.rl" currentCls->add(*ts); inCharClassEarly = false; } }} -#line 8031 "Parser.cpp" +#line 8052 "Parser.cpp" break; } - case 299: { + case 301: { { -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" {p = ((te))-1; { -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 8046 "Parser.cpp" +#line 8067 "Parser.cpp" break; } - case 300: { + case 302: { { -#line 1243 "Parser.rl" +#line 1250 "Parser.rl" {te = p+1;{ -#line 1243 "Parser.rl" +#line 1250 "Parser.rl" inComment = false; {cs = 746;goto _again;}} }} -#line 8057 "Parser.cpp" +#line 8078 "Parser.cpp" break; } - case 301: { + case 303: { { #line 1 "-" {te = p+1;}} -#line 8065 "Parser.cpp" +#line 8086 "Parser.cpp" break; } - case 302: { + case 304: { { -#line 1255 "Parser.rl" +#line 1262 "Parser.rl" {te = p+1;{ -#line 1255 "Parser.rl" +#line 1262 "Parser.rl" inComment = false; {cs = 746;goto _again;}} }} -#line 8076 "Parser.cpp" +#line 8097 "Parser.cpp" break; } - case 303: { + case 305: { { #line 1 "-" {te = p+1;}} -#line 8084 "Parser.cpp" +#line 8105 "Parser.cpp" break; } - case 304: { + case 306: { { -#line 1491 "Parser.rl" - {act = 288;}} +#line 1498 "Parser.rl" + {act = 290;}} -#line 8092 "Parser.cpp" +#line 8113 "Parser.cpp" break; } - case 305: { + case 307: { { -#line 1508 "Parser.rl" - {act = 290;}} +#line 1515 "Parser.rl" + {act = 292;}} -#line 8100 "Parser.cpp" +#line 8121 "Parser.cpp" break; } - case 306: { + case 308: { { -#line 1737 "Parser.rl" - {act = 330;}} +#line 1744 "Parser.rl" + {act = 332;}} -#line 8108 "Parser.cpp" +#line 8129 "Parser.cpp" break; } - case 307: { + case 309: { { -#line 362 "Parser.rl" +#line 363 "Parser.rl" {te = p+1;{ -#line 362 "Parser.rl" +#line 363 "Parser.rl" if (sequences.empty()) { throw LocatedParseError("Unmatched parentheses"); @@ -8440,98 +8463,98 @@ static const short _regex_actions[] = { } }} -#line 8125 "Parser.cpp" +#line 8146 "Parser.cpp" break; } - case 308: { + case 310: { { -#line 1274 "Parser.rl" +#line 1281 "Parser.rl" {te = p+1;{ -#line 1274 "Parser.rl" +#line 1281 "Parser.rl" currentSeq->addAlternation(); } }} -#line 8138 "Parser.cpp" +#line 8159 "Parser.cpp" break; } - case 309: { + case 311: { { -#line 1279 "Parser.rl" +#line 1286 "Parser.rl" {te = p+1;{ -#line 1279 "Parser.rl" +#line 1286 "Parser.rl" throw LocatedParseError("POSIX named classes are only " "supported inside a class"); } }} -#line 8152 "Parser.cpp" +#line 8173 "Parser.cpp" break; } - case 310: { + case 312: { { -#line 1286 "Parser.rl" +#line 1293 "Parser.rl" {te = p+1;{ -#line 1286 "Parser.rl" +#line 1293 "Parser.rl" throw LocatedParseError("Unsupported POSIX collating " "element"); } }} -#line 8166 "Parser.cpp" +#line 8187 "Parser.cpp" break; } - case 311: { + case 313: { { -#line 1293 "Parser.rl" +#line 1300 "Parser.rl" {te = p+1;{ -#line 1293 "Parser.rl" +#line 1300 "Parser.rl" - {cs = 838;goto _again;}} + {cs = 839;goto _again;}} }} -#line 8178 "Parser.cpp" +#line 8199 "Parser.cpp" break; } - case 312: { + case 314: { { -#line 1297 "Parser.rl" +#line 1304 "Parser.rl" {te = p+1;{ -#line 1297 "Parser.rl" +#line 1304 "Parser.rl" /* noop */ } }} -#line 8189 "Parser.cpp" +#line 8210 "Parser.cpp" break; } - case 313: { + case 315: { { -#line 1299 "Parser.rl" +#line 1306 "Parser.rl" {te = p+1;{ -#line 1299 "Parser.rl" +#line 1306 "Parser.rl" currentSeq->addComponent(generateComponent(CLASS_ANY, false, mode)); } }} -#line 8202 "Parser.cpp" +#line 8223 "Parser.cpp" break; } - case 314: { + case 316: { { -#line 1303 "Parser.rl" +#line 1310 "Parser.rl" {te = p+1;{ -#line 1303 "Parser.rl" +#line 1310 "Parser.rl" if (mode.utf8) { throw LocatedParseError("\\C is unsupported in UTF8"); @@ -8540,15 +8563,15 @@ static const short _regex_actions[] = { } }} -#line 8218 "Parser.cpp" +#line 8239 "Parser.cpp" break; } - case 315: { + case 317: { { -#line 1317 "Parser.rl" +#line 1324 "Parser.rl" {te = p+1;{ -#line 1317 "Parser.rl" +#line 1324 "Parser.rl" if (!currentSeq->addRepeat(0, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_NONGREEDY)) { @@ -8557,15 +8580,15 @@ static const short _regex_actions[] = { } }} -#line 8234 "Parser.cpp" +#line 8255 "Parser.cpp" break; } - case 316: { + case 318: { { -#line 1324 "Parser.rl" +#line 1331 "Parser.rl" {te = p+1;{ -#line 1324 "Parser.rl" +#line 1331 "Parser.rl" if (!currentSeq->addRepeat(0, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_POSSESSIVE)) { @@ -8574,15 +8597,15 @@ static const short _regex_actions[] = { } }} -#line 8250 "Parser.cpp" +#line 8271 "Parser.cpp" break; } - case 317: { + case 319: { { -#line 1338 "Parser.rl" +#line 1345 "Parser.rl" {te = p+1;{ -#line 1338 "Parser.rl" +#line 1345 "Parser.rl" if (!currentSeq->addRepeat(1, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_NONGREEDY)) { @@ -8591,15 +8614,15 @@ static const short _regex_actions[] = { } }} -#line 8266 "Parser.cpp" +#line 8287 "Parser.cpp" break; } - case 318: { + case 320: { { -#line 1345 "Parser.rl" +#line 1352 "Parser.rl" {te = p+1;{ -#line 1345 "Parser.rl" +#line 1352 "Parser.rl" if (!currentSeq->addRepeat(1, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_POSSESSIVE)) { @@ -8608,15 +8631,15 @@ static const short _regex_actions[] = { } }} -#line 8282 "Parser.cpp" +#line 8303 "Parser.cpp" break; } - case 319: { + case 321: { { -#line 1359 "Parser.rl" +#line 1366 "Parser.rl" {te = p+1;{ -#line 1359 "Parser.rl" +#line 1366 "Parser.rl" if (!currentSeq->addRepeat( 0, 1, ComponentRepeat::REPEAT_NONGREEDY)) { @@ -8625,15 +8648,15 @@ static const short _regex_actions[] = { } }} -#line 8298 "Parser.cpp" +#line 8319 "Parser.cpp" break; } - case 320: { + case 322: { { -#line 1366 "Parser.rl" +#line 1373 "Parser.rl" {te = p+1;{ -#line 1366 "Parser.rl" +#line 1373 "Parser.rl" if (!currentSeq->addRepeat( 0, 1, ComponentRepeat::REPEAT_POSSESSIVE)) { @@ -8642,15 +8665,15 @@ static const short _regex_actions[] = { } }} -#line 8314 "Parser.cpp" +#line 8335 "Parser.cpp" break; } - case 321: { + case 323: { { -#line 1383 "Parser.rl" +#line 1390 "Parser.rl" {te = p+1;{ -#line 1383 "Parser.rl" +#line 1390 "Parser.rl" if (repeatN > repeatM || repeatM == 0) { throwInvalidRepeat(); @@ -8662,15 +8685,15 @@ static const short _regex_actions[] = { } }} -#line 8333 "Parser.cpp" +#line 8354 "Parser.cpp" break; } - case 322: { + case 324: { { -#line 1393 "Parser.rl" +#line 1400 "Parser.rl" {te = p+1;{ -#line 1393 "Parser.rl" +#line 1400 "Parser.rl" if (repeatN > repeatM || repeatM == 0) { throwInvalidRepeat(); @@ -8682,31 +8705,31 @@ static const short _regex_actions[] = { } }} -#line 8352 "Parser.cpp" +#line 8373 "Parser.cpp" break; } - case 323: { + case 325: { { -#line 322 "Parser.rl" +#line 323 "Parser.rl" {te = p+1;{ -#line 322 "Parser.rl" +#line 323 "Parser.rl" inComment = true; - {cs = 849;goto _again;}} + {cs = 850;goto _again;}} }} -#line 8365 "Parser.cpp" +#line 8386 "Parser.cpp" break; } - case 324: { + case 326: { { -#line 1410 "Parser.rl" +#line 1417 "Parser.rl" {te = p+1;{ -#line 1410 "Parser.rl" +#line 1417 "Parser.rl" {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -8716,27 +8739,27 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 787;goto _again;}} }} -#line 8384 "Parser.cpp" +#line 8405 "Parser.cpp" break; } - case 325: { + case 327: { { -#line 1414 "Parser.rl" +#line 1421 "Parser.rl" {te = p+1;{ -#line 1414 "Parser.rl" +#line 1421 "Parser.rl" assert(0); {p += 1; goto _out; } } }} -#line 8395 "Parser.cpp" +#line 8416 "Parser.cpp" break; } - case 326: { + case 328: { { -#line 1421 "Parser.rl" +#line 1428 "Parser.rl" {te = p+1;{ -#line 1421 "Parser.rl" +#line 1428 "Parser.rl" auto bound = mode.multiline ? ComponentBoundary::BEGIN_LINE : ComponentBoundary::BEGIN_STRING; @@ -8744,15 +8767,15 @@ static const short _regex_actions[] = { } }} -#line 8410 "Parser.cpp" +#line 8431 "Parser.cpp" break; } - case 327: { + case 329: { { -#line 1428 "Parser.rl" +#line 1435 "Parser.rl" {te = p+1;{ -#line 1428 "Parser.rl" +#line 1435 "Parser.rl" auto bound = mode.multiline ? ComponentBoundary::END_LINE : ComponentBoundary::END_STRING_OPTIONAL_LF; @@ -8760,188 +8783,188 @@ static const short _regex_actions[] = { } }} -#line 8425 "Parser.cpp" +#line 8446 "Parser.cpp" break; } - case 328: { + case 330: { { -#line 1434 "Parser.rl" +#line 1441 "Parser.rl" {te = p+1;{ -#line 1434 "Parser.rl" +#line 1441 "Parser.rl" auto bound = ComponentBoundary::BEGIN_STRING; currentSeq->addComponent(std::make_unique(bound)); } }} -#line 8439 "Parser.cpp" +#line 8460 "Parser.cpp" break; } - case 329: { + case 331: { { -#line 1439 "Parser.rl" +#line 1446 "Parser.rl" {te = p+1;{ -#line 1439 "Parser.rl" +#line 1446 "Parser.rl" auto bound = ComponentBoundary::END_STRING_OPTIONAL_LF; currentSeq->addComponent(std::make_unique(bound)); } }} -#line 8453 "Parser.cpp" +#line 8474 "Parser.cpp" break; } - case 330: { + case 332: { { -#line 1444 "Parser.rl" +#line 1451 "Parser.rl" {te = p+1;{ -#line 1444 "Parser.rl" +#line 1451 "Parser.rl" auto bound = ComponentBoundary::END_STRING; currentSeq->addComponent(std::make_unique(bound)); } }} -#line 8467 "Parser.cpp" +#line 8488 "Parser.cpp" break; } - case 331: { + case 333: { { -#line 1449 "Parser.rl" +#line 1456 "Parser.rl" {te = p+1;{ -#line 1449 "Parser.rl" +#line 1456 "Parser.rl" currentSeq->addComponent( std::make_unique(ts - ptr, false, mode)); } }} -#line 8481 "Parser.cpp" +#line 8502 "Parser.cpp" break; } - case 332: { + case 334: { { -#line 1454 "Parser.rl" +#line 1461 "Parser.rl" {te = p+1;{ -#line 1454 "Parser.rl" +#line 1461 "Parser.rl" currentSeq->addComponent( std::make_unique(ts - ptr, true, mode)); } }} -#line 8495 "Parser.cpp" +#line 8516 "Parser.cpp" break; } - case 333: { + case 335: { { -#line 1464 "Parser.rl" +#line 1471 "Parser.rl" {te = p+1;{ -#line 1464 "Parser.rl" +#line 1471 "Parser.rl" addLiteral(currentSeq, '\x09', mode); } }} -#line 8508 "Parser.cpp" +#line 8529 "Parser.cpp" break; } - case 334: { + case 336: { { -#line 1468 "Parser.rl" +#line 1475 "Parser.rl" {te = p+1;{ -#line 1468 "Parser.rl" +#line 1475 "Parser.rl" addLiteral(currentSeq, '\x0a', mode); } }} -#line 8521 "Parser.cpp" +#line 8542 "Parser.cpp" break; } - case 335: { + case 337: { { -#line 1472 "Parser.rl" +#line 1479 "Parser.rl" {te = p+1;{ -#line 1472 "Parser.rl" +#line 1479 "Parser.rl" addLiteral(currentSeq, '\x0d', mode); } }} -#line 8534 "Parser.cpp" +#line 8555 "Parser.cpp" break; } - case 336: { + case 338: { { -#line 1476 "Parser.rl" +#line 1483 "Parser.rl" {te = p+1;{ -#line 1476 "Parser.rl" +#line 1483 "Parser.rl" addLiteral(currentSeq, '\x0c', mode); } }} -#line 8547 "Parser.cpp" +#line 8568 "Parser.cpp" break; } - case 337: { + case 339: { { -#line 1480 "Parser.rl" +#line 1487 "Parser.rl" {te = p+1;{ -#line 1480 "Parser.rl" +#line 1487 "Parser.rl" addLiteral(currentSeq, '\x07', mode); } }} -#line 8560 "Parser.cpp" +#line 8581 "Parser.cpp" break; } - case 338: { + case 340: { { -#line 1484 "Parser.rl" +#line 1491 "Parser.rl" {te = p+1;{ -#line 1484 "Parser.rl" +#line 1491 "Parser.rl" addLiteral(currentSeq, '\x1b', mode); } }} -#line 8573 "Parser.cpp" +#line 8594 "Parser.cpp" break; } - case 339: { + case 341: { { -#line 1488 "Parser.rl" +#line 1495 "Parser.rl" {te = p+1;{ -#line 1488 "Parser.rl" +#line 1495 "Parser.rl" addLiteral(currentSeq, octAccumulator, mode); } }} -#line 8586 "Parser.cpp" +#line 8607 "Parser.cpp" break; } - case 340: { + case 342: { { -#line 479 "Parser.rl" +#line 480 "Parser.rl" {te = p+1;{ -#line 479 "Parser.rl" +#line 480 "Parser.rl" if (accumulator == 0) { throw LocatedParseError("Numbered reference cannot be zero"); @@ -8950,15 +8973,15 @@ static const short _regex_actions[] = { } }} -#line 8602 "Parser.cpp" +#line 8623 "Parser.cpp" break; } - case 341: { + case 343: { { -#line 486 "Parser.rl" +#line 487 "Parser.rl" {te = p+1;{ -#line 486 "Parser.rl" +#line 487 "Parser.rl" // Accumulator is a negative offset. if (accumulator == 0) { @@ -8972,15 +8995,15 @@ static const short _regex_actions[] = { } }} -#line 8623 "Parser.cpp" +#line 8644 "Parser.cpp" break; } - case 342: { + case 344: { { -#line 479 "Parser.rl" +#line 480 "Parser.rl" {te = p+1;{ -#line 479 "Parser.rl" +#line 480 "Parser.rl" if (accumulator == 0) { throw LocatedParseError("Numbered reference cannot be zero"); @@ -8989,15 +9012,15 @@ static const short _regex_actions[] = { } }} -#line 8639 "Parser.cpp" +#line 8660 "Parser.cpp" break; } - case 343: { + case 345: { { -#line 486 "Parser.rl" +#line 487 "Parser.rl" {te = p+1;{ -#line 486 "Parser.rl" +#line 487 "Parser.rl" // Accumulator is a negative offset. if (accumulator == 0) { @@ -9011,85 +9034,85 @@ static const short _regex_actions[] = { } }} -#line 8660 "Parser.cpp" +#line 8681 "Parser.cpp" break; } - case 344: { + case 346: { { -#line 498 "Parser.rl" +#line 499 "Parser.rl" {te = p+1;{ -#line 498 "Parser.rl" +#line 499 "Parser.rl" currentSeq->addComponent(std::make_unique(label)); } }} -#line 8673 "Parser.cpp" +#line 8694 "Parser.cpp" break; } - case 345: { + case 347: { { -#line 498 "Parser.rl" +#line 499 "Parser.rl" {te = p+1;{ -#line 498 "Parser.rl" +#line 499 "Parser.rl" currentSeq->addComponent(std::make_unique(label)); } }} -#line 8686 "Parser.cpp" +#line 8707 "Parser.cpp" break; } - case 346: { + case 348: { { -#line 498 "Parser.rl" +#line 499 "Parser.rl" {te = p+1;{ -#line 498 "Parser.rl" +#line 499 "Parser.rl" currentSeq->addComponent(std::make_unique(label)); } }} -#line 8699 "Parser.cpp" +#line 8720 "Parser.cpp" break; } - case 347: { + case 349: { { -#line 498 "Parser.rl" +#line 499 "Parser.rl" {te = p+1;{ -#line 498 "Parser.rl" +#line 499 "Parser.rl" currentSeq->addComponent(std::make_unique(label)); } }} -#line 8712 "Parser.cpp" +#line 8733 "Parser.cpp" break; } - case 348: { + case 350: { { -#line 498 "Parser.rl" +#line 499 "Parser.rl" {te = p+1;{ -#line 498 "Parser.rl" +#line 499 "Parser.rl" currentSeq->addComponent(std::make_unique(label)); } }} -#line 8725 "Parser.cpp" +#line 8746 "Parser.cpp" break; } - case 349: { + case 351: { { -#line 1549 "Parser.rl" +#line 1556 "Parser.rl" {te = p+1;{ -#line 1549 "Parser.rl" +#line 1556 "Parser.rl" ostringstream str; str << "Onigiruma subroutine call at index " << ts - ptr << @@ -9098,15 +9121,15 @@ static const short _regex_actions[] = { } }} -#line 8741 "Parser.cpp" +#line 8762 "Parser.cpp" break; } - case 350: { + case 352: { { -#line 1560 "Parser.rl" +#line 1567 "Parser.rl" {te = p+1;{ -#line 1560 "Parser.rl" +#line 1567 "Parser.rl" string oct(ts + 3, te - ts - 4); unsigned long val; @@ -9122,29 +9145,29 @@ static const short _regex_actions[] = { } }} -#line 8764 "Parser.cpp" +#line 8785 "Parser.cpp" break; } - case 351: { + case 353: { { -#line 1578 "Parser.rl" +#line 1585 "Parser.rl" {te = p+1;{ -#line 1578 "Parser.rl" +#line 1585 "Parser.rl" addEscapedHex(currentSeq, accumulator, mode); } }} -#line 8777 "Parser.cpp" +#line 8798 "Parser.cpp" break; } - case 352: { + case 354: { { -#line 1582 "Parser.rl" +#line 1589 "Parser.rl" {te = p+1;{ -#line 1582 "Parser.rl" +#line 1589 "Parser.rl" string hex(ts + 3, te - ts - 4); unsigned long val; @@ -9160,15 +9183,15 @@ static const short _regex_actions[] = { } }} -#line 8800 "Parser.cpp" +#line 8821 "Parser.cpp" break; } - case 353: { + case 355: { { -#line 1600 "Parser.rl" +#line 1607 "Parser.rl" {te = p+1;{ -#line 1600 "Parser.rl" +#line 1607 "Parser.rl" if (te - ts < 3) { assert(te - ts == 2); @@ -9180,15 +9203,15 @@ static const short _regex_actions[] = { } }} -#line 8819 "Parser.cpp" +#line 8840 "Parser.cpp" break; } - case 354: { + case 356: { { -#line 1610 "Parser.rl" +#line 1617 "Parser.rl" {te = p+1;{ -#line 1610 "Parser.rl" +#line 1617 "Parser.rl" ostringstream str; str << "'\\" << *(ts + 1) << "' at index " << ts - ptr @@ -9197,172 +9220,172 @@ static const short _regex_actions[] = { } }} -#line 8835 "Parser.cpp" +#line 8856 "Parser.cpp" break; } - case 355: { + case 357: { { -#line 1618 "Parser.rl" +#line 1625 "Parser.rl" {te = p+1;{ -#line 1618 "Parser.rl" +#line 1625 "Parser.rl" auto cc = generateComponent(CLASS_WORD, false, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8849 "Parser.cpp" +#line 8870 "Parser.cpp" break; } - case 356: { + case 358: { { -#line 1623 "Parser.rl" +#line 1630 "Parser.rl" {te = p+1;{ -#line 1623 "Parser.rl" +#line 1630 "Parser.rl" auto cc = generateComponent(CLASS_WORD, true, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8863 "Parser.cpp" +#line 8884 "Parser.cpp" break; } - case 357: { + case 359: { { -#line 1628 "Parser.rl" +#line 1635 "Parser.rl" {te = p+1;{ -#line 1628 "Parser.rl" +#line 1635 "Parser.rl" auto cc = generateComponent(CLASS_SPACE, false, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8877 "Parser.cpp" +#line 8898 "Parser.cpp" break; } - case 358: { + case 360: { { -#line 1633 "Parser.rl" +#line 1640 "Parser.rl" {te = p+1;{ -#line 1633 "Parser.rl" +#line 1640 "Parser.rl" auto cc = generateComponent(CLASS_SPACE, true, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8891 "Parser.cpp" +#line 8912 "Parser.cpp" break; } - case 359: { + case 361: { { -#line 1638 "Parser.rl" +#line 1645 "Parser.rl" {te = p+1;{ -#line 1638 "Parser.rl" +#line 1645 "Parser.rl" auto cc = generateComponent(CLASS_DIGIT, false, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8905 "Parser.cpp" +#line 8926 "Parser.cpp" break; } - case 360: { + case 362: { { -#line 1643 "Parser.rl" +#line 1650 "Parser.rl" {te = p+1;{ -#line 1643 "Parser.rl" +#line 1650 "Parser.rl" auto cc = generateComponent(CLASS_DIGIT, true, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8919 "Parser.cpp" +#line 8940 "Parser.cpp" break; } - case 361: { + case 363: { { -#line 1648 "Parser.rl" +#line 1655 "Parser.rl" {te = p+1;{ -#line 1648 "Parser.rl" +#line 1655 "Parser.rl" auto cc = generateComponent(CLASS_HORZ, false, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8933 "Parser.cpp" +#line 8954 "Parser.cpp" break; } - case 362: { + case 364: { { -#line 1653 "Parser.rl" +#line 1660 "Parser.rl" {te = p+1;{ -#line 1653 "Parser.rl" +#line 1660 "Parser.rl" auto cc = generateComponent(CLASS_HORZ, true, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8947 "Parser.cpp" +#line 8968 "Parser.cpp" break; } - case 363: { + case 365: { { -#line 1658 "Parser.rl" +#line 1665 "Parser.rl" {te = p+1;{ -#line 1658 "Parser.rl" +#line 1665 "Parser.rl" auto cc = generateComponent(CLASS_VERT, false, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8961 "Parser.cpp" +#line 8982 "Parser.cpp" break; } - case 364: { + case 366: { { -#line 1663 "Parser.rl" +#line 1670 "Parser.rl" {te = p+1;{ -#line 1663 "Parser.rl" +#line 1670 "Parser.rl" auto cc = generateComponent(CLASS_VERT, true, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8975 "Parser.cpp" +#line 8996 "Parser.cpp" break; } - case 365: { + case 367: { { -#line 1668 "Parser.rl" +#line 1675 "Parser.rl" {te = p+1;{ -#line 1668 "Parser.rl" +#line 1675 "Parser.rl" assert(!currentCls && !inCharClass); currentCls = getComponentClass(mode); negated = false; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -9372,22 +9395,22 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 559;goto _again;}} }} -#line 8999 "Parser.cpp" +#line 9020 "Parser.cpp" break; } - case 366: { + case 368: { { -#line 1676 "Parser.rl" +#line 1683 "Parser.rl" {te = p+1;{ -#line 1676 "Parser.rl" +#line 1683 "Parser.rl" assert(!currentCls && !inCharClass); currentCls = getComponentClass(mode); negated = false; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -9397,22 +9420,22 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 818;goto _again;}} }} -#line 9023 "Parser.cpp" +#line 9044 "Parser.cpp" break; } - case 367: { + case 369: { { -#line 1684 "Parser.rl" +#line 1691 "Parser.rl" {te = p+1;{ -#line 1684 "Parser.rl" +#line 1691 "Parser.rl" assert(!currentCls && !inCharClass); currentCls = getComponentClass(mode); negated = true; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -9422,22 +9445,22 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 559;goto _again;}} }} -#line 9047 "Parser.cpp" +#line 9068 "Parser.cpp" break; } - case 368: { + case 370: { { -#line 1692 "Parser.rl" +#line 1699 "Parser.rl" {te = p+1;{ -#line 1692 "Parser.rl" +#line 1699 "Parser.rl" assert(!currentCls && !inCharClass); currentCls = getComponentClass(mode); negated = true; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -9447,15 +9470,15 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 818;goto _again;}} }} -#line 9071 "Parser.cpp" +#line 9092 "Parser.cpp" break; } - case 369: { + case 371: { { -#line 1704 "Parser.rl" +#line 1711 "Parser.rl" {te = p+1;{ -#line 1704 "Parser.rl" +#line 1711 "Parser.rl" ostringstream str; str << "\\R at index " << ts - ptr << " not supported."; @@ -9463,15 +9486,15 @@ static const short _regex_actions[] = { } }} -#line 9086 "Parser.cpp" +#line 9107 "Parser.cpp" break; } - case 370: { + case 372: { { -#line 1711 "Parser.rl" +#line 1718 "Parser.rl" {te = p+1;{ -#line 1711 "Parser.rl" +#line 1718 "Parser.rl" ostringstream str; str << "\\K at index " << ts - ptr << " not supported."; @@ -9479,15 +9502,15 @@ static const short _regex_actions[] = { } }} -#line 9101 "Parser.cpp" +#line 9122 "Parser.cpp" break; } - case 371: { + case 373: { { -#line 1726 "Parser.rl" +#line 1733 "Parser.rl" {te = p+1;{ -#line 1726 "Parser.rl" +#line 1733 "Parser.rl" ostringstream str; str << "\\G at index " << ts - ptr << " not supported."; @@ -9495,72 +9518,72 @@ static const short _regex_actions[] = { } }} -#line 9116 "Parser.cpp" +#line 9137 "Parser.cpp" break; } - case 372: { + case 374: { { -#line 1732 "Parser.rl" +#line 1739 "Parser.rl" {te = p+1;{ -#line 1732 "Parser.rl" +#line 1739 "Parser.rl" currentSeq->addComponent(std::make_unique(ts - ptr, mode)); } }} -#line 9129 "Parser.cpp" +#line 9150 "Parser.cpp" break; } - case 373: { + case 375: { { -#line 1737 "Parser.rl" +#line 1744 "Parser.rl" {te = p+1;{ -#line 1737 "Parser.rl" +#line 1744 "Parser.rl" addLiteral(currentSeq, *(ts + 1), mode); } }} -#line 9142 "Parser.cpp" +#line 9163 "Parser.cpp" break; } - case 374: { + case 376: { { -#line 316 "Parser.rl" +#line 317 "Parser.rl" {te = p+1;{ -#line 316 "Parser.rl" +#line 317 "Parser.rl" inComment = true; - {cs = 848;goto _again;}} + {cs = 849;goto _again;}} }} -#line 9155 "Parser.cpp" +#line 9176 "Parser.cpp" break; } - case 375: { + case 377: { { -#line 433 "Parser.rl" +#line 434 "Parser.rl" {te = p+1;{ -#line 433 "Parser.rl" +#line 434 "Parser.rl" mode = newMode; currentSeq->addComponent(std::make_unique()); } }} -#line 9169 "Parser.cpp" +#line 9190 "Parser.cpp" break; } - case 376: { + case 378: { { -#line 355 "Parser.rl" +#line 356 "Parser.rl" {te = p+1;{ -#line 355 "Parser.rl" +#line 356 "Parser.rl" PUSH_SEQUENCE; mode = newMode; @@ -9569,15 +9592,15 @@ static const short _regex_actions[] = { } }} -#line 9185 "Parser.cpp" +#line 9206 "Parser.cpp" break; } - case 377: { + case 379: { { -#line 369 "Parser.rl" +#line 370 "Parser.rl" {te = p+1;{ -#line 369 "Parser.rl" +#line 370 "Parser.rl" PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, @@ -9586,15 +9609,15 @@ static const short _regex_actions[] = { } }} -#line 9201 "Parser.cpp" +#line 9222 "Parser.cpp" break; } - case 378: { + case 380: { { -#line 375 "Parser.rl" +#line 376 "Parser.rl" {te = p+1;{ -#line 375 "Parser.rl" +#line 376 "Parser.rl" PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, @@ -9603,15 +9626,15 @@ static const short _regex_actions[] = { } }} -#line 9217 "Parser.cpp" +#line 9238 "Parser.cpp" break; } - case 379: { + case 381: { { -#line 381 "Parser.rl" +#line 382 "Parser.rl" {te = p+1;{ -#line 381 "Parser.rl" +#line 382 "Parser.rl" PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, @@ -9620,15 +9643,15 @@ static const short _regex_actions[] = { } }} -#line 9233 "Parser.cpp" +#line 9254 "Parser.cpp" break; } - case 380: { + case 382: { { -#line 387 "Parser.rl" +#line 388 "Parser.rl" {te = p+1;{ -#line 387 "Parser.rl" +#line 388 "Parser.rl" PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, @@ -9637,43 +9660,43 @@ static const short _regex_actions[] = { } }} -#line 9249 "Parser.cpp" +#line 9270 "Parser.cpp" break; } - case 381: { + case 383: { { -#line 393 "Parser.rl" +#line 394 "Parser.rl" {te = p+1;{ -#line 393 "Parser.rl" +#line 394 "Parser.rl" throw LocatedParseError("Embedded code is not supported"); } }} -#line 9262 "Parser.cpp" +#line 9283 "Parser.cpp" break; } - case 382: { + case 384: { { -#line 393 "Parser.rl" +#line 394 "Parser.rl" {te = p+1;{ -#line 393 "Parser.rl" +#line 394 "Parser.rl" throw LocatedParseError("Embedded code is not supported"); } }} -#line 9275 "Parser.cpp" +#line 9296 "Parser.cpp" break; } - case 383: { + case 385: { { -#line 416 "Parser.rl" +#line 417 "Parser.rl" {te = p+1;{ -#line 416 "Parser.rl" +#line 417 "Parser.rl" PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, @@ -9681,15 +9704,15 @@ static const short _regex_actions[] = { } }} -#line 9290 "Parser.cpp" +#line 9311 "Parser.cpp" break; } - case 384: { + case 386: { { -#line 336 "Parser.rl" +#line 337 "Parser.rl" {te = p+1;{ -#line 336 "Parser.rl" +#line 337 "Parser.rl" assert(!label.empty()); // should be guaranteed by machine char c = *label.begin(); @@ -9703,145 +9726,145 @@ static const short _regex_actions[] = { auto seq = std::make_unique(); seq->setCaptureIndex(groupIndex++); seq->setCaptureName(label); - currentSeq = enterSequence(currentSeq, move(seq)); + currentSeq = enterSequence(currentSeq, std::move(seq)); } }} -#line 9315 "Parser.cpp" +#line 9336 "Parser.cpp" break; } - case 385: { + case 387: { { -#line 399 "Parser.rl" +#line 400 "Parser.rl" {te = p+1;{ -#line 399 "Parser.rl" +#line 400 "Parser.rl" throw LocatedParseError("Subpattern reference unsupported"); } }} -#line 9328 "Parser.cpp" +#line 9349 "Parser.cpp" break; } - case 386: { + case 388: { { -#line 399 "Parser.rl" +#line 400 "Parser.rl" {te = p+1;{ -#line 399 "Parser.rl" +#line 400 "Parser.rl" throw LocatedParseError("Subpattern reference unsupported"); } }} -#line 9341 "Parser.cpp" +#line 9362 "Parser.cpp" break; } - case 387: { + case 389: { { -#line 1783 "Parser.rl" +#line 1790 "Parser.rl" {te = p+1;{ -#line 1783 "Parser.rl" +#line 1790 "Parser.rl" auto a = std::make_unique( ComponentAssertion::LOOKAHEAD, ComponentAssertion::POS); ComponentAssertion *a_seq = a.get(); PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, - std::make_unique(move(a))); + std::make_unique(std::move(a))); PUSH_SEQUENCE; currentSeq = a_seq; } }} -#line 9361 "Parser.cpp" +#line 9382 "Parser.cpp" break; } - case 388: { + case 390: { { -#line 1794 "Parser.rl" +#line 1801 "Parser.rl" {te = p+1;{ -#line 1794 "Parser.rl" +#line 1801 "Parser.rl" auto a = std::make_unique( ComponentAssertion::LOOKAHEAD, ComponentAssertion::NEG); ComponentAssertion *a_seq = a.get(); PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, - std::make_unique(move(a))); + std::make_unique(std::move(a))); PUSH_SEQUENCE; currentSeq = a_seq; } }} -#line 9381 "Parser.cpp" +#line 9402 "Parser.cpp" break; } - case 389: { + case 391: { { -#line 1805 "Parser.rl" +#line 1812 "Parser.rl" {te = p+1;{ -#line 1805 "Parser.rl" +#line 1812 "Parser.rl" auto a = std::make_unique( ComponentAssertion::LOOKBEHIND, ComponentAssertion::POS); ComponentAssertion *a_seq = a.get(); PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, - std::make_unique(move(a))); + std::make_unique(std::move(a))); PUSH_SEQUENCE; currentSeq = a_seq; } }} -#line 9401 "Parser.cpp" +#line 9422 "Parser.cpp" break; } - case 390: { + case 392: { { -#line 1816 "Parser.rl" +#line 1823 "Parser.rl" {te = p+1;{ -#line 1816 "Parser.rl" +#line 1823 "Parser.rl" auto a = std::make_unique( ComponentAssertion::LOOKBEHIND, ComponentAssertion::NEG); ComponentAssertion *a_seq = a.get(); PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, - std::make_unique(move(a))); + std::make_unique(std::move(a))); PUSH_SEQUENCE; currentSeq = a_seq; } }} -#line 9421 "Parser.cpp" +#line 9442 "Parser.cpp" break; } - case 391: { + case 393: { { -#line 1828 "Parser.rl" +#line 1835 "Parser.rl" {te = p+1;{ -#line 1828 "Parser.rl" +#line 1835 "Parser.rl" throw LocatedParseError("Pattern recursion not supported"); } }} -#line 9434 "Parser.cpp" +#line 9455 "Parser.cpp" break; } - case 392: { + case 394: { { -#line 402 "Parser.rl" +#line 403 "Parser.rl" {te = p+1;{ -#line 402 "Parser.rl" +#line 403 "Parser.rl" if (accumulator == 0) { throw LocatedParseError("Numbered reference cannot be zero"); @@ -9852,15 +9875,15 @@ static const short _regex_actions[] = { } }} -#line 9452 "Parser.cpp" +#line 9473 "Parser.cpp" break; } - case 393: { + case 395: { { -#line 410 "Parser.rl" +#line 411 "Parser.rl" {te = p+1;{ -#line 410 "Parser.rl" +#line 411 "Parser.rl" PUSH_SEQUENCE; assert(!label.empty()); @@ -9869,15 +9892,15 @@ static const short _regex_actions[] = { } }} -#line 9468 "Parser.cpp" +#line 9489 "Parser.cpp" break; } - case 394: { + case 396: { { -#line 1844 "Parser.rl" +#line 1851 "Parser.rl" {te = p+1;{ -#line 1844 "Parser.rl" +#line 1851 "Parser.rl" ostringstream str; str << "Callout at index " << ts - ptr << " not supported."; @@ -9885,101 +9908,101 @@ static const short _regex_actions[] = { } }} -#line 9483 "Parser.cpp" +#line 9504 "Parser.cpp" break; } - case 395: { + case 397: { { -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" {te = p+1;{ -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" throw LocatedParseError("Unrecognised character after (?"); } }} -#line 9496 "Parser.cpp" +#line 9517 "Parser.cpp" break; } - case 396: { + case 398: { { -#line 1857 "Parser.rl" +#line 1864 "Parser.rl" {te = p+1;{ -#line 1857 "Parser.rl" +#line 1864 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint2c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 9514 "Parser.cpp" +#line 9535 "Parser.cpp" break; } - case 397: { + case 399: { { -#line 1866 "Parser.rl" +#line 1873 "Parser.rl" {te = p+1;{ -#line 1866 "Parser.rl" +#line 1873 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint3c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 9532 "Parser.cpp" +#line 9553 "Parser.cpp" break; } - case 398: { + case 400: { { -#line 1875 "Parser.rl" +#line 1882 "Parser.rl" {te = p+1;{ -#line 1875 "Parser.rl" +#line 1882 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint4c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 9550 "Parser.cpp" +#line 9571 "Parser.cpp" break; } - case 399: { + case 401: { { -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" {te = p+1;{ -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 9564 "Parser.cpp" +#line 9585 "Parser.cpp" break; } - case 400: { + case 402: { { -#line 1893 "Parser.rl" +#line 1900 "Parser.rl" {te = p+1;{ -#line 1893 "Parser.rl" +#line 1900 "Parser.rl" if (mode.ignore_space == false) { addLiteral(currentSeq, *ts, mode); @@ -9987,46 +10010,46 @@ static const short _regex_actions[] = { } }} -#line 9579 "Parser.cpp" +#line 9600 "Parser.cpp" break; } - case 401: { + case 403: { { -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" {te = p+1;{ -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" addLiteral(currentSeq, *ts, mode); } }} -#line 9592 "Parser.cpp" +#line 9613 "Parser.cpp" break; } - case 402: { + case 404: { { -#line 328 "Parser.rl" +#line 329 "Parser.rl" {te = p;p = p - 1;{ -#line 328 "Parser.rl" +#line 329 "Parser.rl" PUSH_SEQUENCE; auto seq = std::make_unique(); seq->setCaptureIndex(groupIndex++); - currentSeq = enterSequence(currentSeq, move(seq)); + currentSeq = enterSequence(currentSeq, std::move(seq)); } }} -#line 9608 "Parser.cpp" +#line 9629 "Parser.cpp" break; } - case 403: { + case 405: { { -#line 421 "Parser.rl" +#line 422 "Parser.rl" {te = p;p = p - 1;{ -#line 421 "Parser.rl" +#line 422 "Parser.rl" assert(!currentCls); assert(!inCharClass); // not reentrant @@ -10034,18 +10057,18 @@ static const short _regex_actions[] = { inCharClass = true; inCharClassEarly = true; currentClsBegin = ts; - {cs = 836;goto _again;}} + {cs = 837;goto _again;}} }} -#line 9626 "Parser.cpp" +#line 9647 "Parser.cpp" break; } - case 404: { + case 406: { { -#line 1310 "Parser.rl" +#line 1317 "Parser.rl" {te = p;p = p - 1;{ -#line 1310 "Parser.rl" +#line 1317 "Parser.rl" if (!currentSeq->addRepeat(0, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_GREEDY)) { @@ -10054,15 +10077,15 @@ static const short _regex_actions[] = { } }} -#line 9642 "Parser.cpp" +#line 9663 "Parser.cpp" break; } - case 405: { + case 407: { { -#line 1331 "Parser.rl" +#line 1338 "Parser.rl" {te = p;p = p - 1;{ -#line 1331 "Parser.rl" +#line 1338 "Parser.rl" if (!currentSeq->addRepeat(1, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_GREEDY)) { @@ -10071,15 +10094,15 @@ static const short _regex_actions[] = { } }} -#line 9658 "Parser.cpp" +#line 9679 "Parser.cpp" break; } - case 406: { + case 408: { { -#line 1352 "Parser.rl" +#line 1359 "Parser.rl" {te = p;p = p - 1;{ -#line 1352 "Parser.rl" +#line 1359 "Parser.rl" if (!currentSeq->addRepeat( 0, 1, ComponentRepeat::REPEAT_GREEDY)) { @@ -10088,15 +10111,15 @@ static const short _regex_actions[] = { } }} -#line 9674 "Parser.cpp" +#line 9695 "Parser.cpp" break; } - case 407: { + case 409: { { -#line 1373 "Parser.rl" +#line 1380 "Parser.rl" {te = p;p = p - 1;{ -#line 1373 "Parser.rl" +#line 1380 "Parser.rl" if (repeatN > repeatM || repeatM == 0) { throwInvalidRepeat(); @@ -10108,29 +10131,29 @@ static const short _regex_actions[] = { } }} -#line 9693 "Parser.cpp" +#line 9714 "Parser.cpp" break; } - case 408: { + case 410: { { -#line 1488 "Parser.rl" +#line 1495 "Parser.rl" {te = p;p = p - 1;{ -#line 1488 "Parser.rl" +#line 1495 "Parser.rl" addLiteral(currentSeq, octAccumulator, mode); } }} -#line 9706 "Parser.cpp" +#line 9727 "Parser.cpp" break; } - case 409: { + case 411: { { -#line 1491 "Parser.rl" +#line 1498 "Parser.rl" {te = p;p = p - 1;{ -#line 1491 "Parser.rl" +#line 1498 "Parser.rl" // If there are enough capturing sub expressions, this may be // a back reference @@ -10143,15 +10166,15 @@ static const short _regex_actions[] = { } }} -#line 9726 "Parser.cpp" +#line 9747 "Parser.cpp" break; } - case 410: { + case 412: { { -#line 479 "Parser.rl" +#line 480 "Parser.rl" {te = p;p = p - 1;{ -#line 479 "Parser.rl" +#line 480 "Parser.rl" if (accumulator == 0) { throw LocatedParseError("Numbered reference cannot be zero"); @@ -10160,15 +10183,15 @@ static const short _regex_actions[] = { } }} -#line 9742 "Parser.cpp" +#line 9763 "Parser.cpp" break; } - case 411: { + case 413: { { -#line 479 "Parser.rl" +#line 480 "Parser.rl" {te = p;p = p - 1;{ -#line 479 "Parser.rl" +#line 480 "Parser.rl" if (accumulator == 0) { throw LocatedParseError("Numbered reference cannot be zero"); @@ -10177,15 +10200,15 @@ static const short _regex_actions[] = { } }} -#line 9758 "Parser.cpp" +#line 9779 "Parser.cpp" break; } - case 412: { + case 414: { { -#line 486 "Parser.rl" +#line 487 "Parser.rl" {te = p;p = p - 1;{ -#line 486 "Parser.rl" +#line 487 "Parser.rl" // Accumulator is a negative offset. if (accumulator == 0) { @@ -10199,71 +10222,71 @@ static const short _regex_actions[] = { } }} -#line 9779 "Parser.cpp" +#line 9800 "Parser.cpp" break; } - case 413: { + case 415: { { -#line 1557 "Parser.rl" +#line 1564 "Parser.rl" {te = p;p = p - 1;{ -#line 1557 "Parser.rl" +#line 1564 "Parser.rl" throw LocatedParseError("Invalid reference after \\g"); } }} -#line 9792 "Parser.cpp" +#line 9813 "Parser.cpp" break; } - case 414: { + case 416: { { -#line 1574 "Parser.rl" +#line 1581 "Parser.rl" {te = p;p = p - 1;{ -#line 1574 "Parser.rl" +#line 1581 "Parser.rl" throw LocatedParseError("Value in \\o{...} sequence is non-octal or missing braces"); } }} -#line 9805 "Parser.cpp" +#line 9826 "Parser.cpp" break; } - case 415: { + case 417: { { -#line 1578 "Parser.rl" +#line 1585 "Parser.rl" {te = p;p = p - 1;{ -#line 1578 "Parser.rl" +#line 1585 "Parser.rl" addEscapedHex(currentSeq, accumulator, mode); } }} -#line 9818 "Parser.cpp" +#line 9839 "Parser.cpp" break; } - case 416: { + case 418: { { -#line 1596 "Parser.rl" +#line 1603 "Parser.rl" {te = p;p = p - 1;{ -#line 1596 "Parser.rl" +#line 1603 "Parser.rl" throw LocatedParseError("Value in \\x{...} sequence is non-hex or missing }"); } }} -#line 9831 "Parser.cpp" +#line 9852 "Parser.cpp" break; } - case 417: { + case 419: { { -#line 1600 "Parser.rl" +#line 1607 "Parser.rl" {te = p;p = p - 1;{ -#line 1600 "Parser.rl" +#line 1607 "Parser.rl" if (te - ts < 3) { assert(te - ts == 2); @@ -10275,39 +10298,39 @@ static const short _regex_actions[] = { } }} -#line 9850 "Parser.cpp" +#line 9871 "Parser.cpp" break; } - case 418: { + case 420: { { -#line 1700 "Parser.rl" +#line 1707 "Parser.rl" {te = p;p = p - 1;{ -#line 1700 "Parser.rl" +#line 1707 "Parser.rl" throw LocatedParseError("Malformed property"); } }} -#line 9861 "Parser.cpp" +#line 9882 "Parser.cpp" break; } - case 419: { + case 421: { { -#line 1701 "Parser.rl" +#line 1708 "Parser.rl" {te = p;p = p - 1;{ -#line 1701 "Parser.rl" +#line 1708 "Parser.rl" throw LocatedParseError("Malformed property"); } }} -#line 9872 "Parser.cpp" +#line 9893 "Parser.cpp" break; } - case 420: { + case 422: { { -#line 1719 "Parser.rl" +#line 1726 "Parser.rl" {te = p;p = p - 1;{ -#line 1719 "Parser.rl" +#line 1726 "Parser.rl" ostringstream str; str << "\\k at index " << ts - ptr << " not supported."; @@ -10315,15 +10338,15 @@ static const short _regex_actions[] = { } }} -#line 9887 "Parser.cpp" +#line 9908 "Parser.cpp" break; } - case 421: { + case 423: { { -#line 1742 "Parser.rl" +#line 1749 "Parser.rl" {te = p;p = p - 1;{ -#line 1742 "Parser.rl" +#line 1749 "Parser.rl" assert(ts + 1 == pe); ostringstream str; @@ -10332,91 +10355,91 @@ static const short _regex_actions[] = { } }} -#line 9903 "Parser.cpp" +#line 9924 "Parser.cpp" break; } - case 422: { + case 424: { { -#line 396 "Parser.rl" +#line 397 "Parser.rl" {te = p;p = p - 1;{ -#line 396 "Parser.rl" +#line 397 "Parser.rl" throw LocatedParseError("Conditional subpattern unsupported"); } }} -#line 9916 "Parser.cpp" +#line 9937 "Parser.cpp" break; } - case 423: { + case 425: { { -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" {te = p;p = p - 1;{ -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" throw LocatedParseError("Unrecognised character after (?"); } }} -#line 9929 "Parser.cpp" +#line 9950 "Parser.cpp" break; } - case 424: { + case 426: { { -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" {te = p;p = p - 1;{ -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 9943 "Parser.cpp" +#line 9964 "Parser.cpp" break; } - case 425: { + case 427: { { -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" {te = p;p = p - 1;{ -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" addLiteral(currentSeq, *ts, mode); } }} -#line 9956 "Parser.cpp" +#line 9977 "Parser.cpp" break; } - case 426: { + case 428: { { -#line 328 "Parser.rl" +#line 329 "Parser.rl" {p = ((te))-1; { -#line 328 "Parser.rl" +#line 329 "Parser.rl" PUSH_SEQUENCE; auto seq = std::make_unique(); seq->setCaptureIndex(groupIndex++); - currentSeq = enterSequence(currentSeq, move(seq)); + currentSeq = enterSequence(currentSeq, std::move(seq)); } }} -#line 9973 "Parser.cpp" +#line 9994 "Parser.cpp" break; } - case 427: { + case 429: { { -#line 421 "Parser.rl" +#line 422 "Parser.rl" {p = ((te))-1; { -#line 421 "Parser.rl" +#line 422 "Parser.rl" assert(!currentCls); assert(!inCharClass); // not reentrant @@ -10424,64 +10447,64 @@ static const short _regex_actions[] = { inCharClass = true; inCharClassEarly = true; currentClsBegin = ts; - {cs = 836;goto _again;}} + {cs = 837;goto _again;}} }} -#line 9992 "Parser.cpp" +#line 10013 "Parser.cpp" break; } - case 428: { + case 430: { { -#line 1557 "Parser.rl" +#line 1564 "Parser.rl" {p = ((te))-1; { -#line 1557 "Parser.rl" +#line 1564 "Parser.rl" throw LocatedParseError("Invalid reference after \\g"); } }} -#line 10006 "Parser.cpp" +#line 10027 "Parser.cpp" break; } - case 429: { + case 431: { { -#line 1574 "Parser.rl" +#line 1581 "Parser.rl" {p = ((te))-1; { -#line 1574 "Parser.rl" +#line 1581 "Parser.rl" throw LocatedParseError("Value in \\o{...} sequence is non-octal or missing braces"); } }} -#line 10020 "Parser.cpp" +#line 10041 "Parser.cpp" break; } - case 430: { + case 432: { { -#line 1596 "Parser.rl" +#line 1603 "Parser.rl" {p = ((te))-1; { -#line 1596 "Parser.rl" +#line 1603 "Parser.rl" throw LocatedParseError("Value in \\x{...} sequence is non-hex or missing }"); } }} -#line 10034 "Parser.cpp" +#line 10055 "Parser.cpp" break; } - case 431: { + case 433: { { -#line 1719 "Parser.rl" +#line 1726 "Parser.rl" {p = ((te))-1; { -#line 1719 "Parser.rl" +#line 1726 "Parser.rl" ostringstream str; str << "\\k at index " << ts - ptr << " not supported."; @@ -10489,79 +10512,79 @@ static const short _regex_actions[] = { } }} -#line 10050 "Parser.cpp" +#line 10071 "Parser.cpp" break; } - case 432: { + case 434: { { -#line 396 "Parser.rl" +#line 397 "Parser.rl" {p = ((te))-1; { -#line 396 "Parser.rl" +#line 397 "Parser.rl" throw LocatedParseError("Conditional subpattern unsupported"); } }} -#line 10064 "Parser.cpp" +#line 10085 "Parser.cpp" break; } - case 433: { + case 435: { { -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" {p = ((te))-1; { -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" throw LocatedParseError("Unrecognised character after (?"); } }} -#line 10078 "Parser.cpp" +#line 10099 "Parser.cpp" break; } - case 434: { + case 436: { { -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" {p = ((te))-1; { -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 10093 "Parser.cpp" +#line 10114 "Parser.cpp" break; } - case 435: { + case 437: { { -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" {p = ((te))-1; { -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" addLiteral(currentSeq, *ts, mode); } }} -#line 10107 "Parser.cpp" +#line 10128 "Parser.cpp" break; } - case 436: { + case 438: { { #line 1 "NONE" {switch( act ) { - case 288: { + case 290: { p = ((te))-1; { -#line 1491 "Parser.rl" +#line 1498 "Parser.rl" // If there are enough capturing sub expressions, this may be // a back reference @@ -10574,10 +10597,10 @@ static const short _regex_actions[] = { } break; } - case 290: { + case 292: { p = ((te))-1; { -#line 1508 "Parser.rl" +#line 1515 "Parser.rl" // if there are enough left parens to this point, back ref if (accumulator < groupIndex) { @@ -10611,10 +10634,10 @@ static const short _regex_actions[] = { } break; } - case 330: { + case 332: { p = ((te))-1; { -#line 1737 "Parser.rl" +#line 1744 "Parser.rl" addLiteral(currentSeq, *(ts + 1), mode); } @@ -10623,7 +10646,7 @@ static const short _regex_actions[] = { }} } -#line 10179 "Parser.cpp" +#line 10200 "Parser.cpp" break; } @@ -10650,7 +10673,7 @@ else { #line 1 "NONE" {ts = 0;}} -#line 10205 "Parser.cpp" +#line 10226 "Parser.cpp" break; } @@ -10667,7 +10690,7 @@ else { _out: {} } -#line 1983 "Parser.rl" +#line 1990 "Parser.rl" if (p != pe && *p != '\0') { @@ -10711,7 +10734,7 @@ assert(currentSeq == rootSeq.get()); // Ensure that all references are valid. checkReferences(*rootSeq, groupIndex, groupNames); -return move(rootSeq); +return rootSeq; } catch (LocatedParseError &error) { if (ts >= ptr && ts <= pe) { error.locate(ts - ptr); diff --git a/contrib/vectorscan-cmake/rageled_files/aarch64/control_verbs.cpp b/contrib/vectorscan-cmake/rageled_files/aarch64/control_verbs.cpp index 6204fc0a31462..57cea09266693 100644 --- a/contrib/vectorscan-cmake/rageled_files/aarch64/control_verbs.cpp +++ b/contrib/vectorscan-cmake/rageled_files/aarch64/control_verbs.cpp @@ -74,7 +74,7 @@ static const signed char _ControlVerbs_actions[] = { 155, 157, 159, 160, 161, 0 }; - static const char _ControlVerbs_trans_keys[] = { + static const unsigned char _ControlVerbs_trans_keys[] = { 41u, 65u, 66u, 67u, 76u, 78u, 85u, 41u, 41u, 78u, 41u, 89u, 41u, 67u, 41u, 82u, 41u, 76u, 41u, 70u, 41u, 41u, 83u, 41u, @@ -276,7 +276,7 @@ static const signed char _ControlVerbs_actions[] = { te = 0; } -#line 105 "control_verbs.rl" +#line 106 "control_verbs.rl" try { @@ -285,7 +285,7 @@ static const signed char _ControlVerbs_actions[] = { { int _klen; unsigned int _trans = 0; - const char * _keys; + const unsigned char * _keys; const signed char * _acts; unsigned int _nacts; _resume: {} @@ -321,9 +321,9 @@ static const signed char _ControlVerbs_actions[] = { _klen = (int)_ControlVerbs_single_lengths[cs]; if ( _klen > 0 ) { - const char *_lower = _keys; - const char *_upper = _keys + _klen - 1; - const char *_mid; + const unsigned char *_lower = _keys; + const unsigned char *_upper = _keys + _klen - 1; + const unsigned char *_mid; while ( 1 ) { if ( _upper < _lower ) { _keys += _klen; @@ -345,9 +345,9 @@ static const signed char _ControlVerbs_actions[] = { _klen = (int)_ControlVerbs_range_lengths[cs]; if ( _klen > 0 ) { - const char *_lower = _keys; - const char *_upper = _keys + (_klen<<1) - 2; - const char *_mid; + const unsigned char *_lower = _keys; + const unsigned char *_upper = _keys + (_klen<<1) - 2; + const unsigned char *_mid; while ( 1 ) { if ( _upper < _lower ) { _trans += (unsigned int)_klen; @@ -389,9 +389,9 @@ static const signed char _ControlVerbs_actions[] = { } case 3: { { -#line 76 "control_verbs.rl" +#line 77 "control_verbs.rl" {te = p+1;{ -#line 76 "control_verbs.rl" +#line 77 "control_verbs.rl" mode.utf8 = true; } @@ -403,9 +403,9 @@ static const signed char _ControlVerbs_actions[] = { } case 4: { { -#line 80 "control_verbs.rl" +#line 81 "control_verbs.rl" {te = p+1;{ -#line 80 "control_verbs.rl" +#line 81 "control_verbs.rl" mode.ucp = true; } @@ -417,9 +417,9 @@ static const signed char _ControlVerbs_actions[] = { } case 5: { { -#line 84 "control_verbs.rl" +#line 85 "control_verbs.rl" {te = p+1;{ -#line 84 "control_verbs.rl" +#line 85 "control_verbs.rl" ostringstream str; str << "Unsupported control verb " << string(ts, te - ts); @@ -433,9 +433,9 @@ static const signed char _ControlVerbs_actions[] = { } case 6: { { -#line 90 "control_verbs.rl" +#line 91 "control_verbs.rl" {te = p+1;{ -#line 90 "control_verbs.rl" +#line 91 "control_verbs.rl" ostringstream str; str << "Unknown control verb " << string(ts, te - ts); @@ -449,9 +449,9 @@ static const signed char _ControlVerbs_actions[] = { } case 7: { { -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {te = p+1;{ -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {p = p - 1; } {p += 1; goto _out; } @@ -464,9 +464,9 @@ static const signed char _ControlVerbs_actions[] = { } case 8: { { -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {te = p;p = p - 1;{ -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {p = p - 1; } {p += 1; goto _out; } @@ -479,10 +479,10 @@ static const signed char _ControlVerbs_actions[] = { } case 9: { { -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {p = ((te))-1; { -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {p = p - 1; } {p += 1; goto _out; } @@ -530,9 +530,11 @@ static const signed char _ControlVerbs_actions[] = { _out: {} } -#line 108 "control_verbs.rl" +#line 109 "control_verbs.rl" } catch (LocatedParseError &error) { + // cppcheck-suppress pointerLessThanZero + // cppcheck-suppress pointerPositive if (ts >= ptr && ts <= pe) { error.locate(ts - ptr + start); } else { diff --git a/contrib/vectorscan-cmake/rageled_files/amd64/Parser.cpp b/contrib/vectorscan-cmake/rageled_files/amd64/Parser.cpp index ffda15155821d..05494b959c23a 100644 --- a/contrib/vectorscan-cmake/rageled_files/amd64/Parser.cpp +++ b/contrib/vectorscan-cmake/rageled_files/amd64/Parser.cpp @@ -164,8 +164,8 @@ namespace ue2 { assert(child); ComponentSequence *seq = child.get(); - parent->addComponent(move(child)); - return seq; + parent->addComponent(std::move(child)); + return seq; // cppcheck-suppress returnDanglingLifetime } static @@ -176,7 +176,7 @@ namespace ue2 { assert(cc); cc->add(c); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } else { currentSeq->addComponent(getLiteralComponentClass(c, mode.caseless)); } @@ -191,7 +191,7 @@ namespace ue2 { assert(cc); cc->add(accum); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } else { if (accum > 255) { throw LocatedParseError(err_msg); @@ -225,7 +225,7 @@ namespace ue2 { static unichar readUtf8CodePoint2c(const char *s) { - auto *ts = (const u8 *)s; + auto *ts = reinterpret_cast(s); assert(ts[0] >= 0xc0 && ts[0] < 0xe0); assert(ts[1] >= 0x80 && ts[1] < 0xc0); unichar val = ts[0] & 0x1f; @@ -238,7 +238,7 @@ namespace ue2 { static unichar readUtf8CodePoint3c(const char *s) { - auto *ts = (const u8 *)s; + auto *ts = reinterpret_cast(s); assert(ts[0] >= 0xe0 && ts[0] < 0xf0); assert(ts[1] >= 0x80 && ts[1] < 0xc0); assert(ts[2] >= 0x80 && ts[2] < 0xc0); @@ -254,7 +254,7 @@ namespace ue2 { static unichar readUtf8CodePoint4c(const char *s) { - auto *ts = (const u8 *)s; + auto *ts = reinterpret_cast(s); assert(ts[0] >= 0xf0 && ts[0] < 0xf8); assert(ts[1] >= 0x80 && ts[1] < 0xc0); assert(ts[2] >= 0x80 && ts[2] < 0xc0); @@ -272,7 +272,7 @@ namespace ue2 { } -#line 1909 "Parser.rl" +#line 1916 "Parser.rl" @@ -352,18 +352,18 @@ static const short _regex_actions[] = { 289, 1, 290, 1, 291, 1, 292, 1, 293, 1, 294, 1, 295, 1, 296, 1, 297, 1, 298, 1, 299, 1, 300, 1, - 301, 1, 302, 1, 303, 1, 307, 1, - 308, 1, 309, 1, 310, 1, 311, 1, + 301, 1, 302, 1, 303, 1, 304, 1, + 305, 1, 309, 1, 310, 1, 311, 1, 312, 1, 313, 1, 314, 1, 315, 1, 316, 1, 317, 1, 318, 1, 319, 1, 320, 1, 321, 1, 322, 1, 323, 1, 324, 1, 325, 1, 326, 1, 327, 1, 328, 1, 329, 1, 330, 1, 331, 1, 332, 1, 333, 1, 334, 1, 335, 1, - 336, 1, 337, 1, 338, 1, 342, 1, - 343, 1, 344, 1, 345, 1, 346, 1, + 336, 1, 337, 1, 338, 1, 339, 1, + 340, 1, 344, 1, 345, 1, 346, 1, 347, 1, 348, 1, 349, 1, 350, 1, - 352, 1, 353, 1, 354, 1, 355, 1, + 351, 1, 352, 1, 354, 1, 355, 1, 356, 1, 357, 1, 358, 1, 359, 1, 360, 1, 361, 1, 362, 1, 363, 1, 364, 1, 365, 1, 366, 1, 367, 1, @@ -384,29 +384,30 @@ static const short _regex_actions[] = { 424, 1, 425, 1, 426, 1, 427, 1, 428, 1, 429, 1, 430, 1, 431, 1, 432, 1, 433, 1, 434, 1, 435, 1, - 436, 2, 3, 0, 2, 4, 5, 2, - 5, 1, 2, 9, 10, 2, 9, 238, - 2, 9, 239, 2, 9, 339, 2, 10, - 1, 2, 10, 340, 2, 10, 341, 2, - 11, 241, 2, 11, 351, 2, 12, 241, - 2, 12, 351, 2, 13, 241, 2, 13, - 351, 2, 14, 375, 2, 14, 376, 2, - 25, 0, 2, 25, 3, 2, 25, 6, - 2, 25, 14, 3, 25, 5, 306, 3, - 25, 10, 305, 3, 25, 14, 15, 4, - 25, 9, 304, 10, 0 + 436, 1, 437, 1, 438, 2, 3, 0, + 2, 4, 5, 2, 5, 1, 2, 9, + 10, 2, 9, 238, 2, 9, 239, 2, + 9, 341, 2, 10, 1, 2, 10, 342, + 2, 10, 343, 2, 11, 241, 2, 11, + 353, 2, 12, 241, 2, 12, 353, 2, + 13, 241, 2, 13, 353, 2, 14, 377, + 2, 14, 378, 2, 25, 0, 2, 25, + 3, 2, 25, 6, 2, 25, 14, 3, + 25, 5, 308, 3, 25, 10, 307, 3, + 25, 14, 15, 4, 25, 9, 306, 10, + 0 }; static const short _regex_key_offsets[] = { - 0, 0, 2, 4, 6, 7, 29, 37, - 45, 52, 60, 61, 69, 77, 85, 92, - 100, 103, 105, 114, 121, 129, 137, 140, - 146, 154, 157, 164, 171, 179, 186, 190, - 197, 200, 203, 205, 208, 211, 213, 216, - 219, 221, 222, 224, 225, 233, 235, 238, - 241, 242, 250, 258, 266, 274, 281, 289, - 296, 304, 311, 319, 321, 324, 331, 335, - 338, 341, 342, 344, 345, 347, 349, 350, + 0, 0, 1, 23, 31, 39, 46, 54, + 55, 63, 71, 79, 86, 94, 97, 99, + 108, 115, 123, 131, 134, 140, 148, 151, + 158, 165, 173, 180, 184, 191, 194, 197, + 199, 202, 205, 207, 210, 213, 215, 216, + 218, 219, 227, 229, 232, 235, 236, 244, + 252, 260, 268, 275, 283, 290, 298, 305, + 313, 315, 318, 325, 329, 332, 335, 337, + 339, 341, 342, 344, 345, 347, 349, 350, 351, 353, 354, 355, 356, 357, 358, 359, 360, 361, 362, 363, 364, 365, 366, 369, 370, 371, 372, 373, 374, 375, 376, 377, @@ -468,312 +469,312 @@ static const short _regex_actions[] = { 849, 850, 851, 853, 854, 855, 856, 857, 858, 859, 860, 861, 862, 863, 864, 865, 866, 867, 868, 869, 870, 871, 872, 873, - 874, 875, 876, 877, 879, 881, 883, 886, - 889, 891, 906, 909, 912, 914, 928, 933, - 938, 942, 946, 949, 952, 956, 960, 963, - 966, 970, 974, 978, 981, 984, 988, 992, - 996, 1000, 1003, 1006, 1010, 1014, 1018, 1022, - 1025, 1028, 1032, 1036, 1040, 1044, 1047, 1050, - 1054, 1058, 1062, 1066, 1069, 1072, 1076, 1080, - 1084, 1088, 1091, 1094, 1099, 1103, 1107, 1111, - 1114, 1117, 1121, 1125, 1129, 1132, 1135, 1139, - 1143, 1147, 1151, 1154, 1157, 1161, 1165, 1169, - 1173, 1176, 1179, 1183, 1187, 1191, 1194, 1197, - 1201, 1205, 1209, 1213, 1217, 1220, 1223, 1228, - 1233, 1237, 1241, 1244, 1247, 1251, 1255, 1258, - 1261, 1265, 1269, 1273, 1276, 1279, 1283, 1287, - 1291, 1295, 1298, 1301, 1305, 1309, 1313, 1317, - 1320, 1323, 1327, 1331, 1335, 1339, 1342, 1345, - 1349, 1353, 1357, 1361, 1364, 1367, 1371, 1375, - 1379, 1383, 1386, 1389, 1394, 1398, 1402, 1406, - 1409, 1412, 1416, 1420, 1424, 1427, 1430, 1434, - 1438, 1442, 1446, 1449, 1452, 1456, 1460, 1464, - 1468, 1471, 1474, 1478, 1482, 1486, 1489, 1492, - 1496, 1500, 1504, 1508, 1512, 1515, 1518, 1521, - 1524, 1526, 1528, 1531, 1538, 1540, 1542, 1544, - 1546, 1548, 1550, 1577, 1579, 1581, 1583, 1585, - 1592, 1599, 1613, 1615, 1621, 1624, 1633, 1634, - 1637, 1640, 1647, 1649, 1651, 1653, 1656, 1701, - 1703, 1705, 1709, 1713, 1715, 1716, 1716, 1722, - 1724, 1726, 1728, 1730, 1733, 1734, 1735, 1742, - 1748, 1754, 1756, 1758, 1760, 1761, 1764, 1787, + 874, 875, 876, 877, 880, 883, 885, 900, + 903, 906, 908, 922, 927, 932, 936, 940, + 943, 946, 950, 954, 957, 960, 964, 968, + 972, 975, 978, 982, 986, 990, 994, 997, + 1000, 1004, 1008, 1012, 1016, 1019, 1022, 1026, + 1030, 1034, 1038, 1041, 1044, 1048, 1052, 1056, + 1060, 1063, 1066, 1070, 1074, 1078, 1082, 1085, + 1088, 1093, 1097, 1101, 1105, 1108, 1111, 1115, + 1119, 1123, 1126, 1129, 1133, 1137, 1141, 1145, + 1148, 1151, 1155, 1159, 1163, 1167, 1170, 1173, + 1177, 1181, 1185, 1188, 1191, 1195, 1199, 1203, + 1207, 1211, 1214, 1217, 1222, 1227, 1231, 1235, + 1238, 1241, 1245, 1249, 1252, 1255, 1259, 1263, + 1267, 1270, 1273, 1277, 1281, 1285, 1289, 1292, + 1295, 1299, 1303, 1307, 1311, 1314, 1317, 1321, + 1325, 1329, 1333, 1336, 1339, 1343, 1347, 1351, + 1355, 1358, 1361, 1365, 1369, 1373, 1377, 1380, + 1383, 1388, 1392, 1396, 1400, 1403, 1406, 1410, + 1414, 1418, 1421, 1424, 1428, 1432, 1436, 1440, + 1443, 1446, 1450, 1454, 1458, 1462, 1465, 1468, + 1472, 1476, 1480, 1483, 1486, 1490, 1494, 1498, + 1502, 1506, 1509, 1512, 1515, 1518, 1520, 1522, + 1525, 1532, 1534, 1536, 1538, 1540, 1542, 1544, + 1546, 1548, 1550, 1577, 1579, 1586, 1593, 1607, + 1609, 1615, 1618, 1627, 1628, 1631, 1634, 1641, + 1643, 1645, 1647, 1650, 1695, 1697, 1699, 1703, + 1707, 1709, 1710, 1710, 1716, 1718, 1720, 1722, + 1724, 1727, 1728, 1729, 1736, 1742, 1748, 1750, + 1752, 1754, 1756, 1758, 1760, 1761, 1764, 1787, 1790, 1795, 1804, 1806, 1807, 1809, 1814, 1817, 1819, 1821, 1822, 1824, 1834, 1840, 1841, 1846, 1850, 1858, 1860, 1869, 1873, 1874, 1875, 1879, - 1880, 1883, 1883, 1890, 1904, 1906, 1908, 1910, - 1913, 1952, 1954, 1956, 1958, 1960, 1961, 1961, - 1962, 1963, 1970, 1976, 1982, 1985, 1987, 1998, - 2000, 2002, 2004, 2005, 2016, 2018, 2020, 2022, - 2023, 2024, 0 + 1880, 1883, 1883, 1890, 1905, 1906, 1909, 1948, + 1950, 1952, 1954, 1956, 1957, 1957, 1958, 1959, + 1966, 1972, 1978, 1980, 1982, 1984, 1987, 1989, + 2000, 2001, 2003, 2005, 2007, 2018, 2019, 2021, + 2023, 2025, 2026, 0 }; - static const char _regex_trans_keys[] = { - -128, -65, -128, -65, -128, -65, 41, 33, - 35, 38, 39, 40, 41, 43, 45, 58, - 60, 61, 62, 63, 67, 80, 105, 109, - 115, 120, 123, 48, 57, 41, 95, 48, - 57, 65, 90, 97, 122, 39, 95, 48, - 57, 65, 90, 97, 122, 95, 48, 57, - 65, 90, 97, 122, 39, 95, 48, 57, - 65, 90, 97, 122, 41, 41, 95, 48, - 57, 65, 90, 97, 122, 41, 95, 48, - 57, 65, 90, 97, 122, 41, 95, 48, - 57, 65, 90, 97, 122, 95, 48, 57, - 65, 90, 97, 122, 62, 95, 48, 57, - 65, 90, 97, 122, 33, 60, 61, 33, - 61, 38, 41, 95, 48, 57, 65, 90, - 97, 122, 95, 48, 57, 65, 90, 97, - 122, 41, 95, 48, 57, 65, 90, 97, - 122, 41, 95, 48, 57, 65, 90, 97, - 122, 41, 48, 57, 41, 58, 105, 109, - 115, 120, 62, 95, 48, 57, 65, 90, - 97, 122, 41, 48, 57, 95, 48, 57, - 65, 90, 97, 122, 95, 48, 57, 65, - 90, 97, 122, 41, 95, 48, 57, 65, - 90, 97, 122, 95, 48, 57, 65, 90, - 97, 122, 105, 109, 115, 120, 41, 45, - 58, 105, 109, 115, 120, 46, 92, 93, - 46, 92, 93, 46, 92, 58, 92, 93, - 58, 92, 93, 58, 92, 61, 92, 93, - 61, 92, 93, 61, 92, 39, 48, 57, - 62, 45, 95, 48, 57, 65, 90, 97, - 122, 48, 57, 125, 48, 57, 125, 48, - 57, 125, 95, 125, 48, 57, 65, 90, - 97, 122, 95, 125, 48, 57, 65, 90, - 97, 122, 95, 125, 48, 57, 65, 90, - 97, 122, 95, 125, 48, 57, 65, 90, - 97, 122, 95, 48, 57, 65, 90, 97, - 122, 39, 95, 48, 57, 65, 90, 97, - 122, 95, 48, 57, 65, 90, 97, 122, - 62, 95, 48, 57, 65, 90, 97, 122, - 95, 48, 57, 65, 90, 97, 122, 95, - 125, 48, 57, 65, 90, 97, 122, 48, - 55, 125, 48, 55, 125, 48, 57, 65, - 70, 97, 102, 44, 125, 48, 57, 125, - 48, 57, 125, 48, 57, 41, 41, 80, - 41, 41, 70, 41, 56, 41, 121, 97, - 109, 98, 105, 99, 101, 110, 105, 97, - 110, 101, 115, 116, 97, 110, 108, 109, - 116, 105, 110, 101, 115, 101, 117, 109, - 97, 107, 110, 103, 97, 108, 105, 112, - 111, 109, 111, 102, 111, 97, 104, 105, - 109, 105, 108, 108, 101, 103, 104, 105, - 110, 101, 115, 101, 105, 100, 110, 114, - 97, 100, 105, 97, 110, 95, 65, 98, - 111, 114, 105, 103, 105, 110, 97, 108, - 105, 97, 110, 97, 101, 109, 114, 111, - 107, 101, 101, 109, 111, 110, 116, 105, - 99, 110, 101, 105, 102, 111, 114, 109, - 112, 114, 114, 105, 111, 116, 105, 108, - 108, 105, 99, 115, 118, 101, 114, 101, - 116, 97, 110, 97, 103, 97, 114, 105, - 121, 112, 116, 105, 97, 110, 95, 72, - 105, 101, 114, 111, 103, 108, 121, 112, - 104, 115, 104, 105, 111, 112, 105, 99, - 111, 114, 103, 105, 97, 110, 97, 103, - 111, 108, 105, 116, 105, 99, 116, 104, - 105, 99, 101, 101, 107, 106, 114, 97, - 114, 97, 116, 105, 109, 117, 107, 104, - 105, 110, 117, 108, 110, 111, 111, 98, - 114, 101, 119, 114, 97, 103, 97, 110, - 97, 112, 101, 114, 105, 97, 108, 95, - 65, 114, 97, 109, 97, 105, 99, 104, - 115, 101, 114, 105, 116, 101, 100, 99, - 114, 105, 112, 116, 105, 111, 110, 97, - 108, 95, 80, 97, 104, 114, 108, 97, - 118, 105, 116, 104, 105, 97, 110, 118, - 97, 110, 101, 115, 101, 105, 110, 116, - 121, 116, 104, 105, 110, 97, 100, 97, - 97, 107, 97, 110, 97, 97, 104, 95, - 76, 105, 97, 109, 114, 111, 115, 104, - 116, 104, 105, 101, 114, 111, 116, 105, - 110, 112, 99, 104, 97, 109, 110, 115, - 98, 117, 101, 97, 114, 95, 66, 117, - 99, 100, 105, 97, 110, 105, 97, 110, - 108, 110, 97, 121, 97, 108, 97, 109, - 100, 97, 105, 99, 116, 101, 105, 95, - 77, 97, 121, 101, 107, 110, 103, 111, - 108, 105, 97, 110, 97, 110, 109, 97, - 114, 119, 95, 84, 97, 105, 95, 76, - 117, 101, 111, 104, 97, 109, 95, 100, - 67, 104, 105, 107, 105, 95, 73, 80, - 83, 84, 116, 97, 108, 105, 99, 101, - 114, 115, 105, 97, 110, 111, 117, 116, - 104, 95, 65, 114, 97, 98, 105, 97, - 110, 117, 114, 107, 105, 99, 105, 121, - 97, 109, 97, 110, 121, 97, 97, 111, - 103, 115, 95, 80, 97, 101, 110, 105, - 99, 105, 97, 110, 106, 97, 110, 103, - 110, 105, 99, 109, 117, 97, 114, 105, - 116, 97, 110, 114, 97, 115, 104, 116, - 114, 97, 97, 118, 105, 97, 110, 110, - 104, 97, 108, 97, 110, 100, 97, 110, - 101, 115, 101, 108, 114, 111, 116, 105, - 95, 78, 97, 103, 114, 105, 105, 97, - 99, 103, 105, 109, 97, 98, 108, 111, - 103, 97, 110, 119, 97, 95, 76, 84, - 86, 101, 104, 97, 109, 105, 101, 116, - 105, 108, 108, 117, 103, 117, 97, 97, - 105, 110, 97, 98, 102, 101, 116, 97, - 110, 105, 110, 97, 103, 104, 97, 114, - 105, 116, 105, 99, 105, 110, 115, 112, - 100, 123, 94, 125, 94, -128, -65, -128, - -65, -128, -65, 46, 92, 93, 46, 92, - 93, 46, 92, 58, 92, 93, 94, 97, - 98, 99, 100, 103, 108, 112, 115, 117, - 119, 120, 58, 92, 93, 58, 92, 93, - 58, 92, 58, 92, 93, 97, 98, 99, - 100, 103, 108, 112, 115, 117, 119, 120, - 58, 92, 93, 108, 115, 58, 92, 93, - 110, 112, 58, 92, 93, 117, 58, 92, - 93, 109, 58, 92, 93, 58, 92, 93, - 58, 92, 93, 104, 58, 92, 93, 97, - 58, 92, 93, 58, 92, 93, 58, 92, - 93, 99, 58, 92, 93, 105, 58, 92, - 93, 105, 58, 92, 93, 58, 92, 93, - 58, 92, 93, 108, 58, 92, 93, 97, - 58, 92, 93, 110, 58, 92, 93, 107, - 58, 92, 93, 58, 92, 93, 58, 92, - 93, 110, 58, 92, 93, 116, 58, 92, - 93, 114, 58, 92, 93, 108, 58, 92, - 93, 58, 92, 93, 58, 92, 93, 105, - 58, 92, 93, 103, 58, 92, 93, 105, - 58, 92, 93, 116, 58, 92, 93, 58, - 92, 93, 58, 92, 93, 114, 58, 92, - 93, 97, 58, 92, 93, 112, 58, 92, - 93, 104, 58, 92, 93, 58, 92, 93, - 58, 92, 93, 111, 58, 92, 93, 119, - 58, 92, 93, 101, 58, 92, 93, 114, - 58, 92, 93, 58, 92, 93, 58, 92, - 93, 114, 117, 58, 92, 93, 105, 58, - 92, 93, 110, 58, 92, 93, 116, 58, - 92, 93, 58, 92, 93, 58, 92, 93, - 110, 58, 92, 93, 99, 58, 92, 93, - 116, 58, 92, 93, 58, 92, 93, 58, - 92, 93, 112, 58, 92, 93, 97, 58, - 92, 93, 99, 58, 92, 93, 101, 58, - 92, 93, 58, 92, 93, 58, 92, 93, - 112, 58, 92, 93, 112, 58, 92, 93, - 101, 58, 92, 93, 114, 58, 92, 93, - 58, 92, 93, 58, 92, 93, 111, 58, - 92, 93, 114, 58, 92, 93, 100, 58, - 92, 93, 58, 92, 93, 58, 92, 93, - 100, 58, 92, 93, 105, 58, 92, 93, - 103, 58, 92, 93, 105, 58, 92, 93, - 116, 58, 92, 93, 58, 92, 93, 58, - 92, 93, 108, 115, 58, 92, 93, 110, - 112, 58, 92, 93, 117, 58, 92, 93, - 109, 58, 92, 93, 58, 92, 93, 58, - 92, 93, 104, 58, 92, 93, 97, 58, - 92, 93, 58, 92, 93, 58, 92, 93, - 99, 58, 92, 93, 105, 58, 92, 93, - 105, 58, 92, 93, 58, 92, 93, 58, - 92, 93, 108, 58, 92, 93, 97, 58, - 92, 93, 110, 58, 92, 93, 107, 58, - 92, 93, 58, 92, 93, 58, 92, 93, - 110, 58, 92, 93, 116, 58, 92, 93, - 114, 58, 92, 93, 108, 58, 92, 93, - 58, 92, 93, 58, 92, 93, 105, 58, - 92, 93, 103, 58, 92, 93, 105, 58, - 92, 93, 116, 58, 92, 93, 58, 92, - 93, 58, 92, 93, 114, 58, 92, 93, - 97, 58, 92, 93, 112, 58, 92, 93, - 104, 58, 92, 93, 58, 92, 93, 58, - 92, 93, 111, 58, 92, 93, 119, 58, - 92, 93, 101, 58, 92, 93, 114, 58, - 92, 93, 58, 92, 93, 58, 92, 93, - 114, 117, 58, 92, 93, 105, 58, 92, - 93, 110, 58, 92, 93, 116, 58, 92, - 93, 58, 92, 93, 58, 92, 93, 110, - 58, 92, 93, 99, 58, 92, 93, 116, - 58, 92, 93, 58, 92, 93, 58, 92, - 93, 112, 58, 92, 93, 97, 58, 92, - 93, 99, 58, 92, 93, 101, 58, 92, - 93, 58, 92, 93, 58, 92, 93, 112, - 58, 92, 93, 112, 58, 92, 93, 101, - 58, 92, 93, 114, 58, 92, 93, 58, - 92, 93, 58, 92, 93, 111, 58, 92, - 93, 114, 58, 92, 93, 100, 58, 92, - 93, 58, 92, 93, 58, 92, 93, 100, - 58, 92, 93, 105, 58, 92, 93, 103, - 58, 92, 93, 105, 58, 92, 93, 116, - 58, 92, 93, 58, 92, 93, 61, 92, - 93, 61, 92, 93, 61, 92, 48, 55, - 125, 48, 55, 125, 48, 57, 65, 70, - 97, 102, -128, -65, -128, -65, -128, -65, - -128, -65, -128, -65, -128, -65, 0, 32, - 35, 36, 40, 41, 42, 43, 46, 63, - 91, 92, 94, 123, 124, -128, -65, -64, - -33, -32, -17, -16, -9, -8, -1, 9, - 13, -128, -65, -128, -65, -128, -65, 42, - 63, 95, 48, 57, 65, 90, 97, 122, - 95, 48, 57, 65, 90, 97, 122, 39, - 48, 60, 63, 82, 95, 49, 55, 56, - 57, 65, 90, 97, 122, 48, 57, 105, - 109, 115, 120, 48, 57, 41, 48, 57, - 33, 61, 95, 48, 57, 65, 90, 97, - 122, 123, 41, 48, 57, 60, 61, 62, - 41, 45, 58, 105, 109, 115, 120, 43, - 63, 43, 63, 43, 63, 46, 58, 61, - 48, 65, 66, 67, 68, 69, 71, 72, - 75, 76, 78, 80, 81, 82, 83, 85, - 86, 87, 88, 90, 97, 98, 99, 100, - 101, 102, 103, 104, 107, 108, 110, 111, - 112, 114, 115, 116, 117, 118, 119, 120, - 122, 49, 55, 56, 57, 48, 55, 48, - 55, 48, 55, 56, 57, 48, 55, 56, - 57, 48, 57, 123, 39, 45, 60, 123, - 48, 57, 48, 57, 48, 57, 48, 57, - 48, 57, 39, 60, 123, 123, 123, 123, - 48, 57, 65, 70, 97, 102, 48, 57, - 65, 70, 97, 102, 48, 57, 65, 70, - 97, 102, 48, 57, 43, 63, 41, 85, - 41, 41, 67, 84, 65, 66, 67, 68, - 69, 71, 72, 73, 74, 75, 76, 77, - 78, 79, 80, 82, 83, 84, 85, 86, - 88, 89, 90, 110, 114, 118, 97, 101, - 111, 114, 117, 97, 99, 102, 104, 110, - 111, 115, 117, 121, 109, 112, 101, 103, - 116, 101, 108, 111, 114, 117, 97, 101, - 105, 103, 117, 109, 110, 97, 97, 104, - 38, 97, 101, 105, 108, 109, 111, 116, - 117, 121, 97, 99, 101, 110, 111, 121, - 101, 100, 101, 107, 108, 111, 103, 108, - 114, 115, 99, 100, 101, 102, 104, 105, - 111, 115, 101, 117, 97, 99, 104, 105, - 107, 109, 111, 117, 121, 97, 101, 104, - 105, 103, 97, 97, 112, 115, 119, 105, - 108, 112, 115, 67, 76, 77, 78, 80, - 83, 90, 45, 91, 92, 93, -128, -65, - -64, -33, -32, -17, -16, -9, -8, -1, - -128, -65, -128, -65, -128, -65, 46, 58, - 61, 48, 68, 69, 72, 76, 78, 80, - 81, 83, 85, 86, 87, 97, 98, 99, - 100, 101, 102, 103, 104, 108, 110, 111, - 112, 114, 115, 116, 117, 118, 119, 120, - 49, 55, 56, 57, 65, 90, 105, 122, - 48, 55, 48, 55, 48, 55, 48, 55, - 123, 123, 123, 123, 48, 57, 65, 70, - 97, 102, 48, 57, 65, 70, 97, 102, - 48, 57, 65, 70, 97, 102, 92, 93, - 94, 69, 81, 92, -128, -65, -64, -33, - -32, -17, -16, -9, -8, -1, -128, -65, - -128, -65, -128, -65, 69, 92, -128, -65, - -64, -33, -32, -17, -16, -9, -8, -1, - -128, -65, -128, -65, -128, -65, 69, 41, - 10, 0 + static const unsigned char _regex_trans_keys[] = { + 41u, 33u, 35u, 38u, 39u, 40u, 41u, 43u, + 45u, 58u, 60u, 61u, 62u, 63u, 67u, 80u, + 105u, 109u, 115u, 120u, 123u, 48u, 57u, 41u, + 95u, 48u, 57u, 65u, 90u, 97u, 122u, 39u, + 95u, 48u, 57u, 65u, 90u, 97u, 122u, 95u, + 48u, 57u, 65u, 90u, 97u, 122u, 39u, 95u, + 48u, 57u, 65u, 90u, 97u, 122u, 41u, 41u, + 95u, 48u, 57u, 65u, 90u, 97u, 122u, 41u, + 95u, 48u, 57u, 65u, 90u, 97u, 122u, 41u, + 95u, 48u, 57u, 65u, 90u, 97u, 122u, 95u, + 48u, 57u, 65u, 90u, 97u, 122u, 62u, 95u, + 48u, 57u, 65u, 90u, 97u, 122u, 33u, 60u, + 61u, 33u, 61u, 38u, 41u, 95u, 48u, 57u, + 65u, 90u, 97u, 122u, 95u, 48u, 57u, 65u, + 90u, 97u, 122u, 41u, 95u, 48u, 57u, 65u, + 90u, 97u, 122u, 41u, 95u, 48u, 57u, 65u, + 90u, 97u, 122u, 41u, 48u, 57u, 41u, 58u, + 105u, 109u, 115u, 120u, 62u, 95u, 48u, 57u, + 65u, 90u, 97u, 122u, 41u, 48u, 57u, 95u, + 48u, 57u, 65u, 90u, 97u, 122u, 95u, 48u, + 57u, 65u, 90u, 97u, 122u, 41u, 95u, 48u, + 57u, 65u, 90u, 97u, 122u, 95u, 48u, 57u, + 65u, 90u, 97u, 122u, 105u, 109u, 115u, 120u, + 41u, 45u, 58u, 105u, 109u, 115u, 120u, 46u, + 92u, 93u, 46u, 92u, 93u, 46u, 92u, 58u, + 92u, 93u, 58u, 92u, 93u, 58u, 92u, 61u, + 92u, 93u, 61u, 92u, 93u, 61u, 92u, 39u, + 48u, 57u, 62u, 45u, 95u, 48u, 57u, 65u, + 90u, 97u, 122u, 48u, 57u, 125u, 48u, 57u, + 125u, 48u, 57u, 125u, 95u, 125u, 48u, 57u, + 65u, 90u, 97u, 122u, 95u, 125u, 48u, 57u, + 65u, 90u, 97u, 122u, 95u, 125u, 48u, 57u, + 65u, 90u, 97u, 122u, 95u, 125u, 48u, 57u, + 65u, 90u, 97u, 122u, 95u, 48u, 57u, 65u, + 90u, 97u, 122u, 39u, 95u, 48u, 57u, 65u, + 90u, 97u, 122u, 95u, 48u, 57u, 65u, 90u, + 97u, 122u, 62u, 95u, 48u, 57u, 65u, 90u, + 97u, 122u, 95u, 48u, 57u, 65u, 90u, 97u, + 122u, 95u, 125u, 48u, 57u, 65u, 90u, 97u, + 122u, 48u, 55u, 125u, 48u, 55u, 125u, 48u, + 57u, 65u, 70u, 97u, 102u, 44u, 125u, 48u, + 57u, 125u, 48u, 57u, 125u, 48u, 57u, 128u, + 191u, 128u, 191u, 128u, 191u, 41u, 41u, 80u, + 41u, 41u, 70u, 41u, 56u, 41u, 121u, 97u, + 109u, 98u, 105u, 99u, 101u, 110u, 105u, 97u, + 110u, 101u, 115u, 116u, 97u, 110u, 108u, 109u, + 116u, 105u, 110u, 101u, 115u, 101u, 117u, 109u, + 97u, 107u, 110u, 103u, 97u, 108u, 105u, 112u, + 111u, 109u, 111u, 102u, 111u, 97u, 104u, 105u, + 109u, 105u, 108u, 108u, 101u, 103u, 104u, 105u, + 110u, 101u, 115u, 101u, 105u, 100u, 110u, 114u, + 97u, 100u, 105u, 97u, 110u, 95u, 65u, 98u, + 111u, 114u, 105u, 103u, 105u, 110u, 97u, 108u, + 105u, 97u, 110u, 97u, 101u, 109u, 114u, 111u, + 107u, 101u, 101u, 109u, 111u, 110u, 116u, 105u, + 99u, 110u, 101u, 105u, 102u, 111u, 114u, 109u, + 112u, 114u, 114u, 105u, 111u, 116u, 105u, 108u, + 108u, 105u, 99u, 115u, 118u, 101u, 114u, 101u, + 116u, 97u, 110u, 97u, 103u, 97u, 114u, 105u, + 121u, 112u, 116u, 105u, 97u, 110u, 95u, 72u, + 105u, 101u, 114u, 111u, 103u, 108u, 121u, 112u, + 104u, 115u, 104u, 105u, 111u, 112u, 105u, 99u, + 111u, 114u, 103u, 105u, 97u, 110u, 97u, 103u, + 111u, 108u, 105u, 116u, 105u, 99u, 116u, 104u, + 105u, 99u, 101u, 101u, 107u, 106u, 114u, 97u, + 114u, 97u, 116u, 105u, 109u, 117u, 107u, 104u, + 105u, 110u, 117u, 108u, 110u, 111u, 111u, 98u, + 114u, 101u, 119u, 114u, 97u, 103u, 97u, 110u, + 97u, 112u, 101u, 114u, 105u, 97u, 108u, 95u, + 65u, 114u, 97u, 109u, 97u, 105u, 99u, 104u, + 115u, 101u, 114u, 105u, 116u, 101u, 100u, 99u, + 114u, 105u, 112u, 116u, 105u, 111u, 110u, 97u, + 108u, 95u, 80u, 97u, 104u, 114u, 108u, 97u, + 118u, 105u, 116u, 104u, 105u, 97u, 110u, 118u, + 97u, 110u, 101u, 115u, 101u, 105u, 110u, 116u, + 121u, 116u, 104u, 105u, 110u, 97u, 100u, 97u, + 97u, 107u, 97u, 110u, 97u, 97u, 104u, 95u, + 76u, 105u, 97u, 109u, 114u, 111u, 115u, 104u, + 116u, 104u, 105u, 101u, 114u, 111u, 116u, 105u, + 110u, 112u, 99u, 104u, 97u, 109u, 110u, 115u, + 98u, 117u, 101u, 97u, 114u, 95u, 66u, 117u, + 99u, 100u, 105u, 97u, 110u, 105u, 97u, 110u, + 108u, 110u, 97u, 121u, 97u, 108u, 97u, 109u, + 100u, 97u, 105u, 99u, 116u, 101u, 105u, 95u, + 77u, 97u, 121u, 101u, 107u, 110u, 103u, 111u, + 108u, 105u, 97u, 110u, 97u, 110u, 109u, 97u, + 114u, 119u, 95u, 84u, 97u, 105u, 95u, 76u, + 117u, 101u, 111u, 104u, 97u, 109u, 95u, 100u, + 67u, 104u, 105u, 107u, 105u, 95u, 73u, 80u, + 83u, 84u, 116u, 97u, 108u, 105u, 99u, 101u, + 114u, 115u, 105u, 97u, 110u, 111u, 117u, 116u, + 104u, 95u, 65u, 114u, 97u, 98u, 105u, 97u, + 110u, 117u, 114u, 107u, 105u, 99u, 105u, 121u, + 97u, 109u, 97u, 110u, 121u, 97u, 97u, 111u, + 103u, 115u, 95u, 80u, 97u, 101u, 110u, 105u, + 99u, 105u, 97u, 110u, 106u, 97u, 110u, 103u, + 110u, 105u, 99u, 109u, 117u, 97u, 114u, 105u, + 116u, 97u, 110u, 114u, 97u, 115u, 104u, 116u, + 114u, 97u, 97u, 118u, 105u, 97u, 110u, 110u, + 104u, 97u, 108u, 97u, 110u, 100u, 97u, 110u, + 101u, 115u, 101u, 108u, 114u, 111u, 116u, 105u, + 95u, 78u, 97u, 103u, 114u, 105u, 105u, 97u, + 99u, 103u, 105u, 109u, 97u, 98u, 108u, 111u, + 103u, 97u, 110u, 119u, 97u, 95u, 76u, 84u, + 86u, 101u, 104u, 97u, 109u, 105u, 101u, 116u, + 105u, 108u, 108u, 117u, 103u, 117u, 97u, 97u, + 105u, 110u, 97u, 98u, 102u, 101u, 116u, 97u, + 110u, 105u, 110u, 97u, 103u, 104u, 97u, 114u, + 105u, 116u, 105u, 99u, 105u, 110u, 115u, 112u, + 100u, 123u, 94u, 125u, 94u, 46u, 92u, 93u, + 46u, 92u, 93u, 46u, 92u, 58u, 92u, 93u, + 94u, 97u, 98u, 99u, 100u, 103u, 108u, 112u, + 115u, 117u, 119u, 120u, 58u, 92u, 93u, 58u, + 92u, 93u, 58u, 92u, 58u, 92u, 93u, 97u, + 98u, 99u, 100u, 103u, 108u, 112u, 115u, 117u, + 119u, 120u, 58u, 92u, 93u, 108u, 115u, 58u, + 92u, 93u, 110u, 112u, 58u, 92u, 93u, 117u, + 58u, 92u, 93u, 109u, 58u, 92u, 93u, 58u, + 92u, 93u, 58u, 92u, 93u, 104u, 58u, 92u, + 93u, 97u, 58u, 92u, 93u, 58u, 92u, 93u, + 58u, 92u, 93u, 99u, 58u, 92u, 93u, 105u, + 58u, 92u, 93u, 105u, 58u, 92u, 93u, 58u, + 92u, 93u, 58u, 92u, 93u, 108u, 58u, 92u, + 93u, 97u, 58u, 92u, 93u, 110u, 58u, 92u, + 93u, 107u, 58u, 92u, 93u, 58u, 92u, 93u, + 58u, 92u, 93u, 110u, 58u, 92u, 93u, 116u, + 58u, 92u, 93u, 114u, 58u, 92u, 93u, 108u, + 58u, 92u, 93u, 58u, 92u, 93u, 58u, 92u, + 93u, 105u, 58u, 92u, 93u, 103u, 58u, 92u, + 93u, 105u, 58u, 92u, 93u, 116u, 58u, 92u, + 93u, 58u, 92u, 93u, 58u, 92u, 93u, 114u, + 58u, 92u, 93u, 97u, 58u, 92u, 93u, 112u, + 58u, 92u, 93u, 104u, 58u, 92u, 93u, 58u, + 92u, 93u, 58u, 92u, 93u, 111u, 58u, 92u, + 93u, 119u, 58u, 92u, 93u, 101u, 58u, 92u, + 93u, 114u, 58u, 92u, 93u, 58u, 92u, 93u, + 58u, 92u, 93u, 114u, 117u, 58u, 92u, 93u, + 105u, 58u, 92u, 93u, 110u, 58u, 92u, 93u, + 116u, 58u, 92u, 93u, 58u, 92u, 93u, 58u, + 92u, 93u, 110u, 58u, 92u, 93u, 99u, 58u, + 92u, 93u, 116u, 58u, 92u, 93u, 58u, 92u, + 93u, 58u, 92u, 93u, 112u, 58u, 92u, 93u, + 97u, 58u, 92u, 93u, 99u, 58u, 92u, 93u, + 101u, 58u, 92u, 93u, 58u, 92u, 93u, 58u, + 92u, 93u, 112u, 58u, 92u, 93u, 112u, 58u, + 92u, 93u, 101u, 58u, 92u, 93u, 114u, 58u, + 92u, 93u, 58u, 92u, 93u, 58u, 92u, 93u, + 111u, 58u, 92u, 93u, 114u, 58u, 92u, 93u, + 100u, 58u, 92u, 93u, 58u, 92u, 93u, 58u, + 92u, 93u, 100u, 58u, 92u, 93u, 105u, 58u, + 92u, 93u, 103u, 58u, 92u, 93u, 105u, 58u, + 92u, 93u, 116u, 58u, 92u, 93u, 58u, 92u, + 93u, 58u, 92u, 93u, 108u, 115u, 58u, 92u, + 93u, 110u, 112u, 58u, 92u, 93u, 117u, 58u, + 92u, 93u, 109u, 58u, 92u, 93u, 58u, 92u, + 93u, 58u, 92u, 93u, 104u, 58u, 92u, 93u, + 97u, 58u, 92u, 93u, 58u, 92u, 93u, 58u, + 92u, 93u, 99u, 58u, 92u, 93u, 105u, 58u, + 92u, 93u, 105u, 58u, 92u, 93u, 58u, 92u, + 93u, 58u, 92u, 93u, 108u, 58u, 92u, 93u, + 97u, 58u, 92u, 93u, 110u, 58u, 92u, 93u, + 107u, 58u, 92u, 93u, 58u, 92u, 93u, 58u, + 92u, 93u, 110u, 58u, 92u, 93u, 116u, 58u, + 92u, 93u, 114u, 58u, 92u, 93u, 108u, 58u, + 92u, 93u, 58u, 92u, 93u, 58u, 92u, 93u, + 105u, 58u, 92u, 93u, 103u, 58u, 92u, 93u, + 105u, 58u, 92u, 93u, 116u, 58u, 92u, 93u, + 58u, 92u, 93u, 58u, 92u, 93u, 114u, 58u, + 92u, 93u, 97u, 58u, 92u, 93u, 112u, 58u, + 92u, 93u, 104u, 58u, 92u, 93u, 58u, 92u, + 93u, 58u, 92u, 93u, 111u, 58u, 92u, 93u, + 119u, 58u, 92u, 93u, 101u, 58u, 92u, 93u, + 114u, 58u, 92u, 93u, 58u, 92u, 93u, 58u, + 92u, 93u, 114u, 117u, 58u, 92u, 93u, 105u, + 58u, 92u, 93u, 110u, 58u, 92u, 93u, 116u, + 58u, 92u, 93u, 58u, 92u, 93u, 58u, 92u, + 93u, 110u, 58u, 92u, 93u, 99u, 58u, 92u, + 93u, 116u, 58u, 92u, 93u, 58u, 92u, 93u, + 58u, 92u, 93u, 112u, 58u, 92u, 93u, 97u, + 58u, 92u, 93u, 99u, 58u, 92u, 93u, 101u, + 58u, 92u, 93u, 58u, 92u, 93u, 58u, 92u, + 93u, 112u, 58u, 92u, 93u, 112u, 58u, 92u, + 93u, 101u, 58u, 92u, 93u, 114u, 58u, 92u, + 93u, 58u, 92u, 93u, 58u, 92u, 93u, 111u, + 58u, 92u, 93u, 114u, 58u, 92u, 93u, 100u, + 58u, 92u, 93u, 58u, 92u, 93u, 58u, 92u, + 93u, 100u, 58u, 92u, 93u, 105u, 58u, 92u, + 93u, 103u, 58u, 92u, 93u, 105u, 58u, 92u, + 93u, 116u, 58u, 92u, 93u, 58u, 92u, 93u, + 61u, 92u, 93u, 61u, 92u, 93u, 61u, 92u, + 48u, 55u, 125u, 48u, 55u, 125u, 48u, 57u, + 65u, 70u, 97u, 102u, 128u, 191u, 128u, 191u, + 128u, 191u, 128u, 191u, 128u, 191u, 128u, 191u, + 128u, 191u, 128u, 191u, 128u, 191u, 0u, 32u, + 35u, 36u, 40u, 41u, 42u, 43u, 46u, 63u, + 91u, 92u, 94u, 123u, 124u, 9u, 13u, 128u, + 191u, 192u, 223u, 224u, 239u, 240u, 247u, 248u, + 255u, 42u, 63u, 95u, 48u, 57u, 65u, 90u, + 97u, 122u, 95u, 48u, 57u, 65u, 90u, 97u, + 122u, 39u, 48u, 60u, 63u, 82u, 95u, 49u, + 55u, 56u, 57u, 65u, 90u, 97u, 122u, 48u, + 57u, 105u, 109u, 115u, 120u, 48u, 57u, 41u, + 48u, 57u, 33u, 61u, 95u, 48u, 57u, 65u, + 90u, 97u, 122u, 123u, 41u, 48u, 57u, 60u, + 61u, 62u, 41u, 45u, 58u, 105u, 109u, 115u, + 120u, 43u, 63u, 43u, 63u, 43u, 63u, 46u, + 58u, 61u, 48u, 65u, 66u, 67u, 68u, 69u, + 71u, 72u, 75u, 76u, 78u, 80u, 81u, 82u, + 83u, 85u, 86u, 87u, 88u, 90u, 97u, 98u, + 99u, 100u, 101u, 102u, 103u, 104u, 107u, 108u, + 110u, 111u, 112u, 114u, 115u, 116u, 117u, 118u, + 119u, 120u, 122u, 49u, 55u, 56u, 57u, 48u, + 55u, 48u, 55u, 48u, 55u, 56u, 57u, 48u, + 55u, 56u, 57u, 48u, 57u, 123u, 39u, 45u, + 60u, 123u, 48u, 57u, 48u, 57u, 48u, 57u, + 48u, 57u, 48u, 57u, 39u, 60u, 123u, 123u, + 123u, 123u, 48u, 57u, 65u, 70u, 97u, 102u, + 48u, 57u, 65u, 70u, 97u, 102u, 48u, 57u, + 65u, 70u, 97u, 102u, 48u, 57u, 43u, 63u, + 128u, 191u, 128u, 191u, 128u, 191u, 41u, 85u, + 41u, 41u, 67u, 84u, 65u, 66u, 67u, 68u, + 69u, 71u, 72u, 73u, 74u, 75u, 76u, 77u, + 78u, 79u, 80u, 82u, 83u, 84u, 85u, 86u, + 88u, 89u, 90u, 110u, 114u, 118u, 97u, 101u, + 111u, 114u, 117u, 97u, 99u, 102u, 104u, 110u, + 111u, 115u, 117u, 121u, 109u, 112u, 101u, 103u, + 116u, 101u, 108u, 111u, 114u, 117u, 97u, 101u, + 105u, 103u, 117u, 109u, 110u, 97u, 97u, 104u, + 38u, 97u, 101u, 105u, 108u, 109u, 111u, 116u, + 117u, 121u, 97u, 99u, 101u, 110u, 111u, 121u, + 101u, 100u, 101u, 107u, 108u, 111u, 103u, 108u, + 114u, 115u, 99u, 100u, 101u, 102u, 104u, 105u, + 111u, 115u, 101u, 117u, 97u, 99u, 104u, 105u, + 107u, 109u, 111u, 117u, 121u, 97u, 101u, 104u, + 105u, 103u, 97u, 97u, 112u, 115u, 119u, 105u, + 108u, 112u, 115u, 67u, 76u, 77u, 78u, 80u, + 83u, 90u, 38u, 45u, 91u, 92u, 93u, 128u, + 191u, 192u, 223u, 224u, 239u, 240u, 247u, 248u, + 255u, 38u, 46u, 58u, 61u, 48u, 68u, 69u, + 72u, 76u, 78u, 80u, 81u, 83u, 85u, 86u, + 87u, 97u, 98u, 99u, 100u, 101u, 102u, 103u, + 104u, 108u, 110u, 111u, 112u, 114u, 115u, 116u, + 117u, 118u, 119u, 120u, 49u, 55u, 56u, 57u, + 65u, 90u, 105u, 122u, 48u, 55u, 48u, 55u, + 48u, 55u, 48u, 55u, 123u, 123u, 123u, 123u, + 48u, 57u, 65u, 70u, 97u, 102u, 48u, 57u, + 65u, 70u, 97u, 102u, 48u, 57u, 65u, 70u, + 97u, 102u, 128u, 191u, 128u, 191u, 128u, 191u, + 92u, 93u, 94u, 69u, 81u, 92u, 128u, 191u, + 192u, 223u, 224u, 239u, 240u, 247u, 248u, 255u, + 69u, 128u, 191u, 128u, 191u, 128u, 191u, 92u, + 128u, 191u, 192u, 223u, 224u, 239u, 240u, 247u, + 248u, 255u, 69u, 128u, 191u, 128u, 191u, 128u, + 191u, 41u, 10u, 0u }; static const signed char _regex_single_lengths[] = { - 0, 0, 0, 0, 1, 20, 2, 2, - 1, 2, 1, 2, 2, 2, 1, 2, - 3, 2, 3, 1, 2, 2, 1, 6, - 2, 1, 1, 1, 2, 1, 4, 7, - 3, 3, 2, 3, 3, 2, 3, 3, - 2, 1, 0, 1, 2, 0, 1, 1, - 1, 2, 2, 2, 2, 1, 2, 1, - 2, 1, 2, 0, 1, 1, 2, 1, - 1, 1, 2, 1, 2, 2, 1, 1, + 0, 1, 20, 2, 2, 1, 2, 1, + 2, 2, 2, 1, 2, 3, 2, 3, + 1, 2, 2, 1, 6, 2, 1, 1, + 1, 2, 1, 4, 7, 3, 3, 2, + 3, 3, 2, 3, 3, 2, 1, 0, + 1, 2, 0, 1, 1, 1, 2, 2, + 2, 2, 1, 2, 1, 2, 1, 2, + 0, 1, 1, 2, 1, 1, 0, 0, + 0, 1, 2, 1, 2, 2, 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -835,54 +836,54 @@ static const short _regex_actions[] = { 1, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 0, 0, 0, 3, 3, - 2, 15, 3, 3, 2, 14, 5, 5, - 4, 4, 3, 3, 4, 4, 3, 3, - 4, 4, 4, 3, 3, 4, 4, 4, + 1, 1, 1, 3, 3, 2, 15, 3, + 3, 2, 14, 5, 5, 4, 4, 3, + 3, 4, 4, 3, 3, 4, 4, 4, + 3, 3, 4, 4, 4, 4, 3, 3, + 4, 4, 4, 4, 3, 3, 4, 4, + 4, 4, 3, 3, 4, 4, 4, 4, + 3, 3, 4, 4, 4, 4, 3, 3, + 5, 4, 4, 4, 3, 3, 4, 4, 4, 3, 3, 4, 4, 4, 4, 3, 3, 4, 4, 4, 4, 3, 3, 4, - 4, 4, 4, 3, 3, 4, 4, 4, - 4, 3, 3, 5, 4, 4, 4, 3, - 3, 4, 4, 4, 3, 3, 4, 4, 4, 4, 3, 3, 4, 4, 4, 4, - 3, 3, 4, 4, 4, 3, 3, 4, - 4, 4, 4, 4, 3, 3, 5, 5, - 4, 4, 3, 3, 4, 4, 3, 3, - 4, 4, 4, 3, 3, 4, 4, 4, + 4, 3, 3, 5, 5, 4, 4, 3, + 3, 4, 4, 3, 3, 4, 4, 4, + 3, 3, 4, 4, 4, 4, 3, 3, + 4, 4, 4, 4, 3, 3, 4, 4, + 4, 4, 3, 3, 4, 4, 4, 4, + 3, 3, 4, 4, 4, 4, 3, 3, + 5, 4, 4, 4, 3, 3, 4, 4, 4, 3, 3, 4, 4, 4, 4, 3, 3, 4, 4, 4, 4, 3, 3, 4, - 4, 4, 4, 3, 3, 4, 4, 4, - 4, 3, 3, 5, 4, 4, 4, 3, - 3, 4, 4, 4, 3, 3, 4, 4, 4, 4, 3, 3, 4, 4, 4, 4, - 3, 3, 4, 4, 4, 3, 3, 4, - 4, 4, 4, 4, 3, 3, 3, 3, - 2, 0, 1, 1, 0, 0, 0, 0, - 0, 0, 15, 0, 0, 0, 2, 1, - 1, 6, 0, 4, 1, 3, 1, 1, - 3, 7, 2, 2, 2, 3, 41, 0, - 0, 0, 0, 0, 1, 0, 4, 0, - 0, 0, 0, 3, 1, 1, 1, 0, - 0, 0, 2, 2, 1, 3, 23, 3, + 4, 3, 3, 3, 3, 2, 0, 1, + 1, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 15, 2, 1, 1, 6, 0, + 4, 1, 3, 1, 1, 3, 7, 2, + 2, 2, 3, 41, 0, 0, 0, 0, + 0, 1, 0, 4, 0, 0, 0, 0, + 3, 1, 1, 1, 0, 0, 0, 2, + 0, 0, 0, 2, 1, 3, 23, 3, 5, 9, 2, 1, 2, 5, 3, 2, 2, 1, 2, 10, 6, 1, 5, 4, 8, 2, 9, 4, 1, 1, 4, 1, - 3, 0, 7, 4, 0, 0, 0, 3, - 31, 0, 0, 0, 0, 1, 0, 1, - 1, 1, 0, 0, 3, 2, 1, 0, - 0, 0, 1, 1, 0, 0, 0, 1, - 1, 1, 0 + 3, 0, 7, 5, 1, 3, 31, 0, + 0, 0, 0, 1, 0, 1, 1, 1, + 0, 0, 0, 0, 0, 3, 2, 1, + 1, 0, 0, 0, 1, 1, 0, 0, + 0, 1, 1, 0 }; static const signed char _regex_range_lengths[] = { - 0, 1, 1, 1, 0, 1, 3, 3, - 3, 3, 0, 3, 3, 3, 3, 3, - 0, 0, 3, 3, 3, 3, 1, 0, - 3, 1, 3, 3, 3, 3, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1, 0, 3, 1, 1, 1, - 0, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3, 1, 1, 3, 1, 1, + 0, 0, 1, 3, 3, 3, 3, 0, + 3, 3, 3, 3, 3, 0, 0, 3, + 3, 3, 3, 1, 0, 3, 1, 3, + 3, 3, 3, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1, + 0, 3, 1, 1, 1, 0, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, + 1, 1, 3, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -945,7 +946,6 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -967,33 +967,34 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 1, 1, 3, 1, 1, 1, 1, - 1, 1, 6, 1, 1, 1, 0, 3, - 3, 4, 1, 1, 1, 3, 0, 1, - 0, 0, 0, 0, 0, 0, 2, 1, - 1, 2, 2, 1, 0, 0, 1, 1, - 1, 1, 1, 0, 0, 0, 3, 3, - 3, 1, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 1, + 3, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 6, 0, 3, 3, 4, 1, + 1, 1, 3, 0, 1, 0, 0, 0, + 0, 0, 0, 2, 1, 1, 2, 2, + 1, 0, 0, 1, 1, 1, 1, 1, + 0, 0, 0, 3, 3, 3, 1, 0, + 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 5, 1, 1, 1, 0, - 4, 1, 1, 1, 1, 0, 0, 0, - 0, 3, 3, 3, 0, 0, 5, 1, - 1, 1, 0, 5, 1, 1, 1, 0, - 0, 0, 0 + 0, 0, 0, 5, 0, 0, 4, 1, + 1, 1, 1, 0, 0, 0, 0, 3, + 3, 3, 1, 1, 1, 0, 0, 5, + 0, 1, 1, 1, 5, 0, 1, 1, + 1, 0, 0, 0 }; static const short _regex_index_offsets[] = { - 0, 0, 2, 4, 6, 8, 30, 36, - 42, 47, 53, 55, 61, 67, 73, 78, - 84, 88, 91, 98, 103, 109, 115, 118, - 125, 131, 134, 139, 144, 150, 155, 160, - 168, 172, 176, 179, 183, 187, 190, 194, - 198, 201, 203, 205, 207, 213, 215, 218, - 221, 223, 229, 235, 241, 247, 252, 258, - 263, 269, 274, 280, 282, 285, 290, 294, - 297, 300, 302, 305, 307, 310, 313, 315, + 0, 0, 2, 24, 30, 36, 41, 47, + 49, 55, 61, 67, 72, 78, 82, 85, + 92, 97, 103, 109, 112, 119, 125, 128, + 133, 138, 144, 149, 154, 162, 166, 170, + 173, 177, 181, 184, 188, 192, 195, 197, + 199, 201, 207, 209, 212, 215, 217, 223, + 229, 235, 241, 246, 252, 257, 263, 268, + 274, 276, 279, 284, 288, 291, 294, 296, + 298, 300, 302, 305, 307, 310, 313, 315, 317, 320, 322, 324, 326, 328, 330, 332, 334, 336, 338, 340, 342, 344, 346, 350, 352, 354, 356, 358, 360, 362, 364, 366, @@ -1055,47 +1056,46 @@ static const short _regex_actions[] = { 1279, 1281, 1283, 1286, 1288, 1290, 1292, 1294, 1296, 1298, 1300, 1302, 1304, 1306, 1308, 1310, 1312, 1314, 1316, 1318, 1320, 1322, 1324, 1326, - 1328, 1330, 1332, 1334, 1336, 1338, 1340, 1344, - 1348, 1351, 1367, 1371, 1375, 1378, 1393, 1399, - 1405, 1410, 1415, 1419, 1423, 1428, 1433, 1437, - 1441, 1446, 1451, 1456, 1460, 1464, 1469, 1474, - 1479, 1484, 1488, 1492, 1497, 1502, 1507, 1512, - 1516, 1520, 1525, 1530, 1535, 1540, 1544, 1548, - 1553, 1558, 1563, 1568, 1572, 1576, 1581, 1586, - 1591, 1596, 1600, 1604, 1610, 1615, 1620, 1625, - 1629, 1633, 1638, 1643, 1648, 1652, 1656, 1661, - 1666, 1671, 1676, 1680, 1684, 1689, 1694, 1699, - 1704, 1708, 1712, 1717, 1722, 1727, 1731, 1735, - 1740, 1745, 1750, 1755, 1760, 1764, 1768, 1774, - 1780, 1785, 1790, 1794, 1798, 1803, 1808, 1812, - 1816, 1821, 1826, 1831, 1835, 1839, 1844, 1849, - 1854, 1859, 1863, 1867, 1872, 1877, 1882, 1887, - 1891, 1895, 1900, 1905, 1910, 1915, 1919, 1923, - 1928, 1933, 1938, 1943, 1947, 1951, 1956, 1961, - 1966, 1971, 1975, 1979, 1985, 1990, 1995, 2000, - 2004, 2008, 2013, 2018, 2023, 2027, 2031, 2036, - 2041, 2046, 2051, 2055, 2059, 2064, 2069, 2074, - 2079, 2083, 2087, 2092, 2097, 2102, 2106, 2110, - 2115, 2120, 2125, 2130, 2135, 2139, 2143, 2147, - 2151, 2154, 2156, 2159, 2164, 2166, 2168, 2170, - 2172, 2174, 2176, 2198, 2200, 2202, 2204, 2207, - 2212, 2217, 2228, 2230, 2236, 2239, 2246, 2248, - 2251, 2255, 2263, 2266, 2269, 2272, 2276, 2320, - 2322, 2324, 2327, 2330, 2332, 2334, 2335, 2341, - 2343, 2345, 2347, 2349, 2353, 2355, 2357, 2362, - 2366, 2370, 2372, 2375, 2378, 2380, 2384, 2408, + 1328, 1330, 1332, 1334, 1338, 1342, 1345, 1361, + 1365, 1369, 1372, 1387, 1393, 1399, 1404, 1409, + 1413, 1417, 1422, 1427, 1431, 1435, 1440, 1445, + 1450, 1454, 1458, 1463, 1468, 1473, 1478, 1482, + 1486, 1491, 1496, 1501, 1506, 1510, 1514, 1519, + 1524, 1529, 1534, 1538, 1542, 1547, 1552, 1557, + 1562, 1566, 1570, 1575, 1580, 1585, 1590, 1594, + 1598, 1604, 1609, 1614, 1619, 1623, 1627, 1632, + 1637, 1642, 1646, 1650, 1655, 1660, 1665, 1670, + 1674, 1678, 1683, 1688, 1693, 1698, 1702, 1706, + 1711, 1716, 1721, 1725, 1729, 1734, 1739, 1744, + 1749, 1754, 1758, 1762, 1768, 1774, 1779, 1784, + 1788, 1792, 1797, 1802, 1806, 1810, 1815, 1820, + 1825, 1829, 1833, 1838, 1843, 1848, 1853, 1857, + 1861, 1866, 1871, 1876, 1881, 1885, 1889, 1894, + 1899, 1904, 1909, 1913, 1917, 1922, 1927, 1932, + 1937, 1941, 1945, 1950, 1955, 1960, 1965, 1969, + 1973, 1979, 1984, 1989, 1994, 1998, 2002, 2007, + 2012, 2017, 2021, 2025, 2030, 2035, 2040, 2045, + 2049, 2053, 2058, 2063, 2068, 2073, 2077, 2081, + 2086, 2091, 2096, 2100, 2104, 2109, 2114, 2119, + 2124, 2129, 2133, 2137, 2141, 2145, 2148, 2150, + 2153, 2158, 2160, 2162, 2164, 2166, 2168, 2170, + 2172, 2174, 2176, 2198, 2201, 2206, 2211, 2222, + 2224, 2230, 2233, 2240, 2242, 2245, 2249, 2257, + 2260, 2263, 2266, 2270, 2314, 2316, 2318, 2321, + 2324, 2326, 2328, 2329, 2335, 2337, 2339, 2341, + 2343, 2347, 2349, 2351, 2356, 2360, 2364, 2366, + 2369, 2371, 2373, 2375, 2378, 2380, 2384, 2408, 2412, 2418, 2428, 2431, 2433, 2436, 2442, 2446, 2449, 2452, 2454, 2457, 2468, 2475, 2477, 2483, 2488, 2497, 2500, 2510, 2515, 2517, 2519, 2524, - 2526, 2530, 2531, 2539, 2549, 2551, 2553, 2555, - 2559, 2595, 2597, 2599, 2601, 2603, 2605, 2606, - 2608, 2610, 2615, 2619, 2623, 2627, 2630, 2637, - 2639, 2641, 2643, 2645, 2652, 2654, 2656, 2658, - 2660, 2662, 0 + 2526, 2530, 2531, 2539, 2550, 2552, 2556, 2592, + 2594, 2596, 2598, 2600, 2602, 2603, 2605, 2607, + 2612, 2616, 2620, 2622, 2624, 2626, 2630, 2633, + 2640, 2642, 2644, 2646, 2648, 2655, 2657, 2659, + 2661, 2663, 2665, 0 }; static const signed char _regex_trans_cond_spaces[] = { - 0, -1, 0, -1, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, @@ -1132,6 +1132,8 @@ static const short _regex_actions[] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 0, -1, + 0, -1, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, @@ -1261,8 +1263,6 @@ static const short _regex_actions[] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, 0, -1, - 0, -1, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, @@ -1365,13 +1365,12 @@ static const short _regex_actions[] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, 0, -1, 0, -1, + -1, -1, -1, -1, -1, -1, 0, -1, + 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, 0, -1, -1, -1, 1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 0, - 0, 0, 0, 0, -1, -1, 0, -1, - 0, -1, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, @@ -1393,6 +1392,7 @@ static const short _regex_actions[] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, 0, -1, 0, -1, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, @@ -1412,24 +1412,24 @@ static const short _regex_actions[] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, 0, - 0, 0, 0, 0, -1, 0, -1, 0, - -1, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + 0, 0, 0, 0, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 2, 2, -1, -1, -1, -1, -1, 0, - 0, 0, 0, 0, -1, 0, -1, 0, - -1, 0, -1, -1, -1, -1, 0, 0, - 0, 0, 0, -1, 0, -1, 0, -1, - 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, 0, -1, 0, -1, + 0, -1, -1, 2, 2, -1, -1, -1, + -1, -1, 0, 0, 0, 0, 0, -1, + -1, -1, 0, -1, 0, -1, 0, -1, + -1, 0, 0, 0, 0, 0, -1, -1, + -1, 0, -1, 0, -1, 0, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, @@ -1534,48 +1534,49 @@ static const short _regex_actions[] = { -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, 0 + -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, 0 }; static const short _regex_trans_offsets[] = { - 0, 2, 3, 5, 6, 8, 9, 10, - 11, 12, 13, 14, 15, 16, 17, 18, - 19, 20, 21, 22, 23, 24, 25, 26, - 27, 28, 29, 30, 31, 32, 33, 34, - 35, 36, 37, 38, 39, 40, 41, 42, - 43, 44, 45, 46, 47, 48, 49, 50, - 51, 52, 53, 54, 55, 56, 57, 58, - 59, 60, 61, 62, 63, 64, 65, 66, - 67, 68, 69, 70, 71, 72, 73, 74, - 75, 76, 77, 78, 79, 80, 81, 82, - 83, 84, 85, 86, 87, 88, 89, 90, - 91, 92, 93, 94, 95, 96, 97, 98, - 99, 100, 101, 102, 103, 104, 105, 106, - 107, 108, 109, 110, 111, 112, 113, 114, - 115, 116, 117, 118, 119, 120, 121, 122, - 123, 124, 125, 126, 127, 128, 129, 130, - 131, 132, 133, 134, 135, 136, 137, 138, - 139, 140, 141, 142, 143, 144, 145, 146, - 147, 148, 149, 150, 151, 152, 153, 154, - 155, 156, 157, 158, 159, 160, 161, 162, - 163, 164, 165, 166, 167, 168, 169, 170, - 171, 172, 173, 174, 175, 176, 177, 178, - 179, 180, 181, 182, 183, 184, 185, 186, - 187, 188, 189, 190, 191, 192, 193, 194, - 195, 196, 197, 198, 199, 200, 201, 202, - 203, 204, 205, 206, 207, 208, 209, 210, - 211, 212, 213, 214, 215, 216, 217, 218, - 219, 220, 221, 222, 223, 224, 225, 226, - 227, 228, 229, 230, 231, 232, 233, 234, - 235, 236, 237, 238, 239, 240, 241, 242, - 243, 244, 245, 246, 247, 248, 249, 250, - 251, 252, 253, 254, 255, 256, 257, 258, - 259, 260, 261, 262, 263, 264, 265, 266, - 267, 268, 269, 270, 271, 272, 273, 274, - 275, 276, 277, 278, 279, 280, 281, 282, - 283, 284, 285, 286, 287, 288, 289, 290, - 291, 292, 293, 294, 295, 296, 297, 298, - 299, 300, 301, 302, 303, 304, 305, 306, + 0, 1, 2, 3, 4, 5, 6, 7, + 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, + 24, 25, 26, 27, 28, 29, 30, 31, + 32, 33, 34, 35, 36, 37, 38, 39, + 40, 41, 42, 43, 44, 45, 46, 47, + 48, 49, 50, 51, 52, 53, 54, 55, + 56, 57, 58, 59, 60, 61, 62, 63, + 64, 65, 66, 67, 68, 69, 70, 71, + 72, 73, 74, 75, 76, 77, 78, 79, + 80, 81, 82, 83, 84, 85, 86, 87, + 88, 89, 90, 91, 92, 93, 94, 95, + 96, 97, 98, 99, 100, 101, 102, 103, + 104, 105, 106, 107, 108, 109, 110, 111, + 112, 113, 114, 115, 116, 117, 118, 119, + 120, 121, 122, 123, 124, 125, 126, 127, + 128, 129, 130, 131, 132, 133, 134, 135, + 136, 137, 138, 139, 140, 141, 142, 143, + 144, 145, 146, 147, 148, 149, 150, 151, + 152, 153, 154, 155, 156, 157, 158, 159, + 160, 161, 162, 163, 164, 165, 166, 167, + 168, 169, 170, 171, 172, 173, 174, 175, + 176, 177, 178, 179, 180, 181, 182, 183, + 184, 185, 186, 187, 188, 189, 190, 191, + 192, 193, 194, 195, 196, 197, 198, 199, + 200, 201, 202, 203, 204, 205, 206, 207, + 208, 209, 210, 211, 212, 213, 214, 215, + 216, 217, 218, 219, 220, 221, 222, 223, + 224, 225, 226, 227, 228, 229, 230, 231, + 232, 233, 234, 235, 236, 237, 238, 239, + 240, 241, 242, 243, 244, 245, 246, 247, + 248, 249, 250, 251, 252, 253, 254, 255, + 256, 257, 258, 259, 260, 261, 262, 263, + 264, 265, 266, 267, 268, 269, 270, 271, + 272, 273, 274, 275, 276, 277, 278, 279, + 280, 281, 282, 283, 284, 285, 286, 287, + 288, 289, 290, 291, 292, 293, 294, 296, + 297, 299, 300, 302, 303, 304, 305, 306, 307, 308, 309, 310, 311, 312, 313, 314, 315, 316, 317, 318, 319, 320, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, @@ -1704,137 +1705,137 @@ static const short _regex_actions[] = { 1307, 1308, 1309, 1310, 1311, 1312, 1313, 1314, 1315, 1316, 1317, 1318, 1319, 1320, 1321, 1322, 1323, 1324, 1325, 1326, 1327, 1328, 1329, 1330, - 1331, 1332, 1333, 1334, 1335, 1336, 1337, 1339, - 1340, 1342, 1343, 1345, 1346, 1347, 1348, 1349, - 1350, 1351, 1352, 1353, 1354, 1355, 1356, 1357, - 1358, 1359, 1360, 1361, 1362, 1363, 1364, 1365, - 1366, 1367, 1368, 1369, 1370, 1371, 1372, 1373, - 1374, 1375, 1376, 1377, 1378, 1379, 1380, 1381, - 1382, 1383, 1384, 1385, 1386, 1387, 1388, 1389, - 1390, 1391, 1392, 1393, 1394, 1395, 1396, 1397, - 1398, 1399, 1400, 1401, 1402, 1403, 1404, 1405, - 1406, 1407, 1408, 1409, 1410, 1411, 1412, 1413, - 1414, 1415, 1416, 1417, 1418, 1419, 1420, 1421, - 1422, 1423, 1424, 1425, 1426, 1427, 1428, 1429, - 1430, 1431, 1432, 1433, 1434, 1435, 1436, 1437, - 1438, 1439, 1440, 1441, 1442, 1443, 1444, 1445, - 1446, 1447, 1448, 1449, 1450, 1451, 1452, 1453, - 1454, 1455, 1456, 1457, 1458, 1459, 1460, 1461, - 1462, 1463, 1464, 1465, 1466, 1467, 1468, 1469, - 1470, 1471, 1472, 1473, 1474, 1475, 1476, 1477, - 1478, 1479, 1480, 1481, 1482, 1483, 1484, 1485, - 1486, 1487, 1488, 1489, 1490, 1491, 1492, 1493, - 1494, 1495, 1496, 1497, 1498, 1499, 1500, 1501, - 1502, 1503, 1504, 1505, 1506, 1507, 1508, 1509, - 1510, 1511, 1512, 1513, 1514, 1515, 1516, 1517, - 1518, 1519, 1520, 1521, 1522, 1523, 1524, 1525, - 1526, 1527, 1528, 1529, 1530, 1531, 1532, 1533, - 1534, 1535, 1536, 1537, 1538, 1539, 1540, 1541, - 1542, 1543, 1544, 1545, 1546, 1547, 1548, 1549, - 1550, 1551, 1552, 1553, 1554, 1555, 1556, 1557, - 1558, 1559, 1560, 1561, 1562, 1563, 1564, 1565, - 1566, 1567, 1568, 1569, 1570, 1571, 1572, 1573, - 1574, 1575, 1576, 1577, 1578, 1579, 1580, 1581, - 1582, 1583, 1584, 1585, 1586, 1587, 1588, 1589, - 1590, 1591, 1592, 1593, 1594, 1595, 1596, 1597, - 1598, 1599, 1600, 1601, 1602, 1603, 1604, 1605, - 1606, 1607, 1608, 1609, 1610, 1611, 1612, 1613, - 1614, 1615, 1616, 1617, 1618, 1619, 1620, 1621, - 1622, 1623, 1624, 1625, 1626, 1627, 1628, 1629, - 1630, 1631, 1632, 1633, 1634, 1635, 1636, 1637, - 1638, 1639, 1640, 1641, 1642, 1643, 1644, 1645, - 1646, 1647, 1648, 1649, 1650, 1651, 1652, 1653, - 1654, 1655, 1656, 1657, 1658, 1659, 1660, 1661, - 1662, 1663, 1664, 1665, 1666, 1667, 1668, 1669, - 1670, 1671, 1672, 1673, 1674, 1675, 1676, 1677, - 1678, 1679, 1680, 1681, 1682, 1683, 1684, 1685, - 1686, 1687, 1688, 1689, 1690, 1691, 1692, 1693, - 1694, 1695, 1696, 1697, 1698, 1699, 1700, 1701, - 1702, 1703, 1704, 1705, 1706, 1707, 1708, 1709, - 1710, 1711, 1712, 1713, 1714, 1715, 1716, 1717, - 1718, 1719, 1720, 1721, 1722, 1723, 1724, 1725, - 1726, 1727, 1728, 1729, 1730, 1731, 1732, 1733, - 1734, 1735, 1736, 1737, 1738, 1739, 1740, 1741, - 1742, 1743, 1744, 1745, 1746, 1747, 1748, 1749, - 1750, 1751, 1752, 1753, 1754, 1755, 1756, 1757, - 1758, 1759, 1760, 1761, 1762, 1763, 1764, 1765, - 1766, 1767, 1768, 1769, 1770, 1771, 1772, 1773, - 1774, 1775, 1776, 1777, 1778, 1779, 1780, 1781, - 1782, 1783, 1784, 1785, 1786, 1787, 1788, 1789, - 1790, 1791, 1792, 1793, 1794, 1795, 1796, 1797, - 1798, 1799, 1800, 1801, 1802, 1803, 1804, 1805, - 1806, 1807, 1808, 1809, 1810, 1811, 1812, 1813, - 1814, 1815, 1816, 1817, 1818, 1819, 1820, 1821, - 1822, 1823, 1824, 1825, 1826, 1827, 1828, 1829, - 1830, 1831, 1832, 1833, 1834, 1835, 1836, 1837, - 1838, 1839, 1840, 1841, 1842, 1843, 1844, 1845, - 1846, 1847, 1848, 1849, 1850, 1851, 1852, 1853, - 1854, 1855, 1856, 1857, 1858, 1859, 1860, 1861, - 1862, 1863, 1864, 1865, 1866, 1867, 1868, 1869, - 1870, 1871, 1872, 1873, 1874, 1875, 1876, 1877, - 1878, 1879, 1880, 1881, 1882, 1883, 1884, 1885, - 1886, 1887, 1888, 1889, 1890, 1891, 1892, 1893, - 1894, 1895, 1896, 1897, 1898, 1899, 1900, 1901, - 1902, 1903, 1904, 1905, 1906, 1907, 1908, 1909, - 1910, 1911, 1912, 1913, 1914, 1915, 1916, 1917, - 1918, 1919, 1920, 1921, 1922, 1923, 1924, 1925, - 1926, 1927, 1928, 1929, 1930, 1931, 1932, 1933, - 1934, 1935, 1936, 1937, 1938, 1939, 1940, 1941, - 1942, 1943, 1944, 1945, 1946, 1947, 1948, 1949, - 1950, 1951, 1952, 1953, 1954, 1955, 1956, 1957, - 1958, 1959, 1960, 1961, 1962, 1963, 1964, 1965, - 1966, 1967, 1968, 1969, 1970, 1971, 1972, 1973, - 1974, 1975, 1976, 1977, 1978, 1979, 1980, 1981, - 1982, 1983, 1984, 1985, 1986, 1987, 1988, 1989, - 1990, 1991, 1992, 1993, 1994, 1995, 1996, 1997, - 1998, 1999, 2000, 2001, 2002, 2003, 2004, 2005, - 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, - 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, - 2022, 2023, 2024, 2025, 2026, 2027, 2028, 2029, - 2030, 2031, 2032, 2033, 2034, 2035, 2036, 2037, - 2038, 2039, 2040, 2041, 2042, 2043, 2044, 2045, - 2046, 2047, 2048, 2049, 2050, 2051, 2052, 2053, - 2054, 2055, 2056, 2057, 2058, 2059, 2060, 2061, - 2062, 2063, 2064, 2065, 2066, 2067, 2068, 2069, - 2070, 2071, 2072, 2073, 2074, 2075, 2076, 2077, - 2078, 2079, 2080, 2081, 2082, 2083, 2084, 2085, - 2086, 2087, 2088, 2089, 2090, 2091, 2092, 2093, - 2094, 2095, 2096, 2097, 2098, 2099, 2100, 2101, - 2102, 2103, 2104, 2105, 2106, 2107, 2108, 2109, - 2110, 2111, 2112, 2113, 2114, 2115, 2116, 2117, - 2118, 2119, 2120, 2121, 2122, 2123, 2124, 2125, - 2126, 2127, 2128, 2129, 2130, 2131, 2132, 2133, - 2134, 2135, 2136, 2137, 2138, 2139, 2140, 2141, - 2142, 2143, 2144, 2145, 2146, 2147, 2148, 2149, - 2150, 2151, 2152, 2153, 2154, 2155, 2156, 2157, - 2158, 2159, 2160, 2161, 2162, 2163, 2164, 2165, - 2166, 2167, 2168, 2169, 2170, 2172, 2173, 2175, + 1331, 1332, 1333, 1334, 1335, 1336, 1337, 1338, + 1339, 1340, 1341, 1342, 1343, 1344, 1345, 1346, + 1347, 1348, 1349, 1350, 1351, 1352, 1353, 1354, + 1355, 1356, 1357, 1358, 1359, 1360, 1361, 1362, + 1363, 1364, 1365, 1366, 1367, 1368, 1369, 1370, + 1371, 1372, 1373, 1374, 1375, 1376, 1377, 1378, + 1379, 1380, 1381, 1382, 1383, 1384, 1385, 1386, + 1387, 1388, 1389, 1390, 1391, 1392, 1393, 1394, + 1395, 1396, 1397, 1398, 1399, 1400, 1401, 1402, + 1403, 1404, 1405, 1406, 1407, 1408, 1409, 1410, + 1411, 1412, 1413, 1414, 1415, 1416, 1417, 1418, + 1419, 1420, 1421, 1422, 1423, 1424, 1425, 1426, + 1427, 1428, 1429, 1430, 1431, 1432, 1433, 1434, + 1435, 1436, 1437, 1438, 1439, 1440, 1441, 1442, + 1443, 1444, 1445, 1446, 1447, 1448, 1449, 1450, + 1451, 1452, 1453, 1454, 1455, 1456, 1457, 1458, + 1459, 1460, 1461, 1462, 1463, 1464, 1465, 1466, + 1467, 1468, 1469, 1470, 1471, 1472, 1473, 1474, + 1475, 1476, 1477, 1478, 1479, 1480, 1481, 1482, + 1483, 1484, 1485, 1486, 1487, 1488, 1489, 1490, + 1491, 1492, 1493, 1494, 1495, 1496, 1497, 1498, + 1499, 1500, 1501, 1502, 1503, 1504, 1505, 1506, + 1507, 1508, 1509, 1510, 1511, 1512, 1513, 1514, + 1515, 1516, 1517, 1518, 1519, 1520, 1521, 1522, + 1523, 1524, 1525, 1526, 1527, 1528, 1529, 1530, + 1531, 1532, 1533, 1534, 1535, 1536, 1537, 1538, + 1539, 1540, 1541, 1542, 1543, 1544, 1545, 1546, + 1547, 1548, 1549, 1550, 1551, 1552, 1553, 1554, + 1555, 1556, 1557, 1558, 1559, 1560, 1561, 1562, + 1563, 1564, 1565, 1566, 1567, 1568, 1569, 1570, + 1571, 1572, 1573, 1574, 1575, 1576, 1577, 1578, + 1579, 1580, 1581, 1582, 1583, 1584, 1585, 1586, + 1587, 1588, 1589, 1590, 1591, 1592, 1593, 1594, + 1595, 1596, 1597, 1598, 1599, 1600, 1601, 1602, + 1603, 1604, 1605, 1606, 1607, 1608, 1609, 1610, + 1611, 1612, 1613, 1614, 1615, 1616, 1617, 1618, + 1619, 1620, 1621, 1622, 1623, 1624, 1625, 1626, + 1627, 1628, 1629, 1630, 1631, 1632, 1633, 1634, + 1635, 1636, 1637, 1638, 1639, 1640, 1641, 1642, + 1643, 1644, 1645, 1646, 1647, 1648, 1649, 1650, + 1651, 1652, 1653, 1654, 1655, 1656, 1657, 1658, + 1659, 1660, 1661, 1662, 1663, 1664, 1665, 1666, + 1667, 1668, 1669, 1670, 1671, 1672, 1673, 1674, + 1675, 1676, 1677, 1678, 1679, 1680, 1681, 1682, + 1683, 1684, 1685, 1686, 1687, 1688, 1689, 1690, + 1691, 1692, 1693, 1694, 1695, 1696, 1697, 1698, + 1699, 1700, 1701, 1702, 1703, 1704, 1705, 1706, + 1707, 1708, 1709, 1710, 1711, 1712, 1713, 1714, + 1715, 1716, 1717, 1718, 1719, 1720, 1721, 1722, + 1723, 1724, 1725, 1726, 1727, 1728, 1729, 1730, + 1731, 1732, 1733, 1734, 1735, 1736, 1737, 1738, + 1739, 1740, 1741, 1742, 1743, 1744, 1745, 1746, + 1747, 1748, 1749, 1750, 1751, 1752, 1753, 1754, + 1755, 1756, 1757, 1758, 1759, 1760, 1761, 1762, + 1763, 1764, 1765, 1766, 1767, 1768, 1769, 1770, + 1771, 1772, 1773, 1774, 1775, 1776, 1777, 1778, + 1779, 1780, 1781, 1782, 1783, 1784, 1785, 1786, + 1787, 1788, 1789, 1790, 1791, 1792, 1793, 1794, + 1795, 1796, 1797, 1798, 1799, 1800, 1801, 1802, + 1803, 1804, 1805, 1806, 1807, 1808, 1809, 1810, + 1811, 1812, 1813, 1814, 1815, 1816, 1817, 1818, + 1819, 1820, 1821, 1822, 1823, 1824, 1825, 1826, + 1827, 1828, 1829, 1830, 1831, 1832, 1833, 1834, + 1835, 1836, 1837, 1838, 1839, 1840, 1841, 1842, + 1843, 1844, 1845, 1846, 1847, 1848, 1849, 1850, + 1851, 1852, 1853, 1854, 1855, 1856, 1857, 1858, + 1859, 1860, 1861, 1862, 1863, 1864, 1865, 1866, + 1867, 1868, 1869, 1870, 1871, 1872, 1873, 1874, + 1875, 1876, 1877, 1878, 1879, 1880, 1881, 1882, + 1883, 1884, 1885, 1886, 1887, 1888, 1889, 1890, + 1891, 1892, 1893, 1894, 1895, 1896, 1897, 1898, + 1899, 1900, 1901, 1902, 1903, 1904, 1905, 1906, + 1907, 1908, 1909, 1910, 1911, 1912, 1913, 1914, + 1915, 1916, 1917, 1918, 1919, 1920, 1921, 1922, + 1923, 1924, 1925, 1926, 1927, 1928, 1929, 1930, + 1931, 1932, 1933, 1934, 1935, 1936, 1937, 1938, + 1939, 1940, 1941, 1942, 1943, 1944, 1945, 1946, + 1947, 1948, 1949, 1950, 1951, 1952, 1953, 1954, + 1955, 1956, 1957, 1958, 1959, 1960, 1961, 1962, + 1963, 1964, 1965, 1966, 1967, 1968, 1969, 1970, + 1971, 1972, 1973, 1974, 1975, 1976, 1977, 1978, + 1979, 1980, 1981, 1982, 1983, 1984, 1985, 1986, + 1987, 1988, 1989, 1990, 1991, 1992, 1993, 1994, + 1995, 1996, 1997, 1998, 1999, 2000, 2001, 2002, + 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, + 2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, + 2019, 2020, 2021, 2022, 2023, 2024, 2025, 2026, + 2027, 2028, 2029, 2030, 2031, 2032, 2033, 2034, + 2035, 2036, 2037, 2038, 2039, 2040, 2041, 2042, + 2043, 2044, 2045, 2046, 2047, 2048, 2049, 2050, + 2051, 2052, 2053, 2054, 2055, 2056, 2057, 2058, + 2059, 2060, 2061, 2062, 2063, 2064, 2065, 2066, + 2067, 2068, 2069, 2070, 2071, 2072, 2073, 2074, + 2075, 2076, 2077, 2078, 2079, 2080, 2081, 2082, + 2083, 2084, 2085, 2086, 2087, 2088, 2089, 2090, + 2091, 2092, 2093, 2094, 2095, 2096, 2097, 2098, + 2099, 2100, 2101, 2102, 2103, 2104, 2105, 2106, + 2107, 2108, 2109, 2110, 2111, 2112, 2113, 2114, + 2115, 2116, 2117, 2118, 2119, 2120, 2121, 2122, + 2123, 2124, 2125, 2126, 2127, 2128, 2129, 2130, + 2131, 2132, 2133, 2134, 2135, 2136, 2137, 2138, + 2139, 2140, 2141, 2142, 2143, 2144, 2145, 2146, + 2147, 2148, 2149, 2150, 2151, 2152, 2153, 2154, + 2155, 2156, 2157, 2158, 2159, 2160, 2161, 2163, + 2164, 2166, 2167, 2169, 2170, 2172, 2173, 2175, 2176, 2178, 2179, 2181, 2182, 2184, 2185, 2187, 2188, 2189, 2190, 2192, 2193, 2194, 2195, 2196, 2197, 2198, 2199, 2200, 2201, 2202, 2203, 2204, - 2206, 2208, 2210, 2212, 2214, 2215, 2216, 2218, - 2219, 2221, 2222, 2224, 2225, 2226, 2227, 2228, - 2229, 2230, 2231, 2232, 2233, 2234, 2235, 2236, - 2237, 2238, 2239, 2240, 2241, 2242, 2243, 2244, - 2245, 2246, 2247, 2248, 2249, 2250, 2251, 2252, - 2253, 2254, 2255, 2256, 2257, 2258, 2259, 2260, - 2261, 2262, 2263, 2264, 2265, 2266, 2267, 2268, - 2269, 2270, 2271, 2272, 2273, 2274, 2275, 2276, - 2277, 2278, 2279, 2280, 2281, 2282, 2283, 2284, - 2285, 2286, 2287, 2288, 2289, 2290, 2291, 2292, - 2293, 2294, 2295, 2296, 2297, 2298, 2299, 2300, - 2301, 2302, 2303, 2304, 2305, 2306, 2307, 2308, - 2309, 2310, 2311, 2312, 2313, 2314, 2315, 2316, - 2317, 2318, 2319, 2320, 2321, 2322, 2323, 2324, - 2325, 2326, 2327, 2328, 2329, 2330, 2331, 2332, - 2333, 2334, 2335, 2336, 2337, 2338, 2339, 2340, - 2341, 2342, 2343, 2344, 2345, 2346, 2347, 2348, - 2349, 2350, 2351, 2352, 2353, 2354, 2355, 2356, - 2357, 2358, 2359, 2360, 2361, 2362, 2363, 2364, - 2365, 2366, 2367, 2368, 2369, 2370, 2371, 2372, - 2373, 2374, 2375, 2376, 2377, 2378, 2379, 2380, - 2381, 2382, 2383, 2384, 2385, 2386, 2387, 2388, - 2389, 2390, 2391, 2392, 2393, 2394, 2395, 2396, + 2205, 2207, 2209, 2211, 2213, 2215, 2216, 2217, + 2218, 2219, 2220, 2221, 2222, 2223, 2224, 2225, + 2226, 2227, 2228, 2229, 2230, 2231, 2232, 2233, + 2234, 2235, 2236, 2237, 2238, 2239, 2240, 2241, + 2242, 2243, 2244, 2245, 2246, 2247, 2248, 2249, + 2250, 2251, 2252, 2253, 2254, 2255, 2256, 2257, + 2258, 2259, 2260, 2261, 2262, 2263, 2264, 2265, + 2266, 2267, 2268, 2269, 2270, 2271, 2272, 2273, + 2274, 2275, 2276, 2277, 2278, 2279, 2280, 2281, + 2282, 2283, 2284, 2285, 2286, 2287, 2288, 2289, + 2290, 2291, 2292, 2293, 2294, 2295, 2296, 2297, + 2298, 2299, 2300, 2301, 2302, 2303, 2304, 2305, + 2306, 2307, 2308, 2309, 2310, 2311, 2312, 2313, + 2314, 2315, 2316, 2317, 2318, 2319, 2320, 2321, + 2322, 2323, 2324, 2325, 2326, 2327, 2328, 2329, + 2330, 2331, 2332, 2333, 2334, 2335, 2336, 2337, + 2338, 2339, 2340, 2341, 2342, 2343, 2344, 2345, + 2346, 2347, 2348, 2349, 2350, 2351, 2352, 2353, + 2354, 2355, 2356, 2357, 2358, 2359, 2360, 2361, + 2362, 2363, 2364, 2365, 2366, 2367, 2368, 2369, + 2370, 2371, 2372, 2373, 2374, 2375, 2376, 2377, + 2378, 2379, 2380, 2381, 2382, 2383, 2384, 2385, + 2386, 2387, 2389, 2390, 2392, 2393, 2395, 2396, 2397, 2398, 2399, 2400, 2401, 2402, 2403, 2404, 2405, 2406, 2407, 2408, 2409, 2410, 2411, 2412, 2413, 2414, 2415, 2416, 2417, 2418, 2419, 2420, @@ -1856,21 +1857,21 @@ static const short _regex_actions[] = { 2541, 2542, 2543, 2544, 2545, 2546, 2547, 2548, 2549, 2550, 2551, 2552, 2553, 2554, 2555, 2556, 2557, 2558, 2559, 2560, 2561, 2562, 2563, 2564, - 2566, 2568, 2570, 2572, 2574, 2575, 2577, 2578, - 2580, 2581, 2583, 2584, 2585, 2586, 2587, 2588, - 2589, 2590, 2591, 2592, 2593, 2594, 2595, 2596, - 2597, 2598, 2599, 2600, 2601, 2602, 2603, 2604, - 2605, 2606, 2607, 2608, 2609, 2610, 2611, 2612, - 2613, 2614, 2615, 2616, 2617, 2618, 2619, 2620, - 2621, 2622, 2623, 2624, 2625, 2626, 2627, 2628, - 2629, 2630, 2631, 2632, 2633, 2634, 2635, 2636, - 2637, 2638, 2639, 2640, 2641, 2642, 2643, 2644, - 2645, 2646, 2647, 2648, 2649, 2650, 2651, 2652, - 2653, 2655, 2657, 2658, 2659, 2660, 2661, 2662, - 2664, 2666, 2668, 2670, 2672, 2673, 2675, 2676, - 2678, 2679, 2681, 2682, 2683, 2684, 2685, 2687, - 2689, 2691, 2693, 2695, 2696, 2698, 2699, 2701, - 2702, 2704, 2705, 2706, 2707, 2708, 2709, 2710, + 2565, 2567, 2569, 2571, 2573, 2575, 2576, 2577, + 2578, 2579, 2580, 2581, 2582, 2583, 2584, 2585, + 2586, 2587, 2588, 2589, 2590, 2591, 2592, 2593, + 2594, 2595, 2596, 2597, 2598, 2599, 2600, 2601, + 2602, 2603, 2604, 2605, 2606, 2607, 2608, 2609, + 2610, 2611, 2612, 2613, 2614, 2615, 2616, 2617, + 2618, 2619, 2620, 2621, 2622, 2623, 2624, 2625, + 2626, 2627, 2628, 2629, 2630, 2631, 2632, 2633, + 2634, 2635, 2636, 2637, 2638, 2639, 2640, 2641, + 2642, 2643, 2644, 2645, 2646, 2648, 2649, 2651, + 2652, 2654, 2655, 2656, 2658, 2660, 2661, 2662, + 2663, 2664, 2665, 2667, 2669, 2671, 2673, 2675, + 2676, 2677, 2678, 2680, 2681, 2683, 2684, 2686, + 2687, 2688, 2690, 2692, 2694, 2696, 2698, 2699, + 2700, 2701, 2703, 2704, 2706, 2707, 2709, 2710, 2711, 2712, 2713, 2714, 2715, 2716, 2717, 2718, 2719, 2720, 2721, 2722, 2723, 2724, 2725, 2726, 2727, 2728, 2729, 2730, 2731, 2732, 2733, 2734, @@ -1977,11 +1978,10 @@ static const short _regex_actions[] = { 3535, 3536, 3537, 3538, 3539, 3540, 3541, 3542, 3543, 3544, 3545, 3546, 3547, 3548, 3549, 3550, 3551, 3552, 3553, 3554, 3555, 3556, 3557, 3558, - 3559, 3560, 0 + 3559, 3560, 3561, 3562, 3563, 3564, 0 }; static const signed char _regex_trans_lengths[] = { - 2, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -2018,6 +2018,8 @@ static const short _regex_actions[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 2, 1, + 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -2147,8 +2149,6 @@ static const short _regex_actions[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 2, 1, - 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -2251,13 +2251,12 @@ static const short _regex_actions[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 2, 1, 2, 1, + 1, 1, 1, 1, 1, 1, 2, 1, + 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 1, 1, 2, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 2, - 2, 2, 2, 2, 1, 1, 2, 1, - 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -2279,6 +2278,7 @@ static const short _regex_actions[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 1, 2, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -2298,22 +2298,23 @@ static const short _regex_actions[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 2, - 2, 2, 2, 2, 1, 2, 1, 2, - 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 2, 2, 2, 2, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 2, 2, 1, 1, 1, 1, 1, 2, - 2, 2, 2, 2, 1, 2, 1, 2, - 1, 2, 1, 1, 1, 1, 2, 2, - 2, 2, 2, 1, 2, 1, 2, 1, - 2, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, + 1, 1, 1, 1, 2, 1, 2, 1, + 2, 1, 1, 2, 2, 1, 1, 1, + 1, 1, 2, 2, 2, 2, 2, 1, + 1, 1, 2, 1, 2, 1, 2, 1, + 1, 2, 2, 2, 2, 2, 1, 1, + 1, 2, 1, 2, 1, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, @@ -2420,11 +2421,10 @@ static const short _regex_actions[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 0 + 1, 1, 1, 1, 1, 1, 0 }; static const signed char _regex_cond_keys[] = { - 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2461,6 +2461,8 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 1, + 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2591,8 +2593,6 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 1, 0, 0, 1, 0, 0, - 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2695,14 +2695,13 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 1, 0, 0, 1, 0, + 0, 0, 1, 0, 0, 1, 0, 0, + 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 1, - 0, 1, 0, 1, 0, 1, 0, 0, - 0, 1, 0, 0, 1, 0, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2724,6 +2723,8 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 1, 0, 0, 1, + 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2744,9 +2745,8 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 1, 0, 1, - 0, 1, 0, 1, 0, 1, 0, 0, - 1, 0, 0, 1, 0, 0, 1, 0, + 0, 0, 0, 0, 0, 0, 1, 0, + 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2755,14 +2755,15 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 1, 0, - 1, 0, 0, 0, 0, 0, 0, 1, - 0, 1, 0, 1, 0, 1, 0, 1, + 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, - 1, 0, 0, 0, 0, 0, 1, 0, - 1, 0, 1, 0, 1, 0, 1, 0, - 0, 1, 0, 0, 1, 0, 0, 1, - 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 0, 1, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 0, 1, 0, + 1, 0, 1, 0, 0, 0, 0, 1, + 0, 0, 1, 0, 0, 1, 0, 0, + 0, 1, 0, 1, 0, 1, 0, 1, + 0, 1, 0, 0, 0, 0, 1, 0, + 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, @@ -2869,48 +2870,48 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0 + 0, 0, 0, 0, 0, 0 }; static const short _regex_cond_targs[] = { - 746, 746, 746, 746, 3, 746, 746, 746, - 746, 746, 746, 746, 746, 751, 752, 753, - 746, 754, 755, 746, 757, 746, 746, 758, - 759, 760, 761, 761, 761, 761, 746, 756, - 746, 746, 6, 6, 6, 6, 746, 746, - 7, 7, 7, 7, 746, 9, 9, 9, - 9, 746, 10, 9, 9, 9, 9, 746, - 746, 746, 746, 11, 11, 11, 11, 746, - 746, 11, 12, 11, 11, 746, 746, 11, - 12, 11, 11, 746, 15, 15, 15, 15, - 746, 10, 15, 15, 15, 15, 746, 746, - 17, 746, 746, 746, 746, 746, 19, 746, - 11, 21, 11, 11, 746, 20, 20, 20, - 20, 746, 746, 20, 20, 20, 20, 746, - 746, 11, 21, 11, 11, 746, 746, 22, - 746, 746, 746, 23, 23, 23, 23, 746, - 746, 24, 24, 24, 24, 746, 746, 25, - 746, 24, 24, 24, 24, 746, 28, 28, - 28, 28, 746, 746, 28, 28, 28, 28, - 746, 6, 6, 6, 6, 746, 23, 23, - 23, 23, 746, 746, 30, 746, 31, 31, - 31, 31, 746, 33, 34, 746, 32, 33, - 34, 746, 32, 33, 34, 32, 36, 37, - 746, 35, 36, 37, 746, 35, 36, 37, - 35, 39, 40, 746, 38, 39, 40, 746, - 38, 39, 40, 38, 746, 41, 775, 746, - 746, 43, 45, 52, 49, 52, 52, 746, - 46, 746, 746, 47, 746, 746, 48, 746, - 746, 746, 52, 746, 50, 52, 52, 746, - 52, 746, 51, 52, 52, 746, 52, 746, - 52, 52, 52, 746, 52, 746, 52, 52, - 52, 746, 54, 54, 54, 54, 746, 746, - 54, 54, 54, 54, 746, 56, 56, 56, - 56, 746, 746, 56, 56, 56, 56, 746, - 58, 58, 58, 58, 746, 58, 746, 58, - 58, 58, 746, 60, 746, 746, 60, 746, - 746, 61, 61, 61, 746, 63, 786, 62, - 746, 786, 64, 746, 786, 64, 746, 787, + 746, 746, 746, 746, 748, 749, 750, 746, + 751, 752, 746, 754, 746, 746, 755, 756, + 757, 758, 758, 758, 758, 746, 753, 746, + 746, 3, 3, 3, 3, 746, 746, 4, + 4, 4, 4, 746, 6, 6, 6, 6, + 746, 7, 6, 6, 6, 6, 746, 746, + 746, 746, 8, 8, 8, 8, 746, 746, + 8, 9, 8, 8, 746, 746, 8, 9, + 8, 8, 746, 12, 12, 12, 12, 746, + 7, 12, 12, 12, 12, 746, 746, 14, + 746, 746, 746, 746, 746, 16, 746, 8, + 18, 8, 8, 746, 17, 17, 17, 17, + 746, 746, 17, 17, 17, 17, 746, 746, + 8, 18, 8, 8, 746, 746, 19, 746, + 746, 746, 20, 20, 20, 20, 746, 746, + 21, 21, 21, 21, 746, 746, 22, 746, + 21, 21, 21, 21, 746, 25, 25, 25, + 25, 746, 746, 25, 25, 25, 25, 746, + 3, 3, 3, 3, 746, 20, 20, 20, + 20, 746, 746, 27, 746, 28, 28, 28, + 28, 746, 30, 31, 746, 29, 30, 31, + 746, 29, 30, 31, 29, 33, 34, 746, + 32, 33, 34, 746, 32, 33, 34, 32, + 36, 37, 746, 35, 36, 37, 746, 35, + 36, 37, 35, 746, 38, 772, 746, 746, + 40, 42, 49, 46, 49, 49, 746, 43, + 746, 746, 44, 746, 746, 45, 746, 746, + 746, 49, 746, 47, 49, 49, 746, 49, + 746, 48, 49, 49, 746, 49, 746, 49, + 49, 49, 746, 49, 746, 49, 49, 49, + 746, 51, 51, 51, 51, 746, 746, 51, + 51, 51, 51, 746, 53, 53, 53, 53, + 746, 746, 53, 53, 53, 53, 746, 55, + 55, 55, 55, 746, 55, 746, 55, 55, + 55, 746, 57, 746, 746, 57, 746, 746, + 58, 58, 58, 746, 60, 783, 59, 746, + 783, 61, 746, 783, 61, 746, 746, 746, + 746, 746, 64, 746, 746, 746, 746, 787, 65, 787, 67, 65, 787, 65, 787, 69, 65, 787, 70, 65, 787, 65, 790, 790, 73, 76, 790, 74, 790, 75, 790, 790, @@ -3040,139 +3041,139 @@ static const short _regex_actions[] = { 790, 552, 790, 553, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 560, 0, 562, 561, 817, 0, 0, - 561, 819, 819, 819, 819, 565, 819, 819, - 819, 819, 567, 568, 819, 566, 567, 568, - 819, 566, 567, 568, 566, 571, 572, 819, - 573, 654, 669, 675, 681, 687, 693, 699, - 710, 716, 722, 727, 570, 571, 572, 819, - 570, 571, 572, 819, 570, 571, 572, 570, - 571, 572, 819, 574, 589, 595, 601, 607, - 613, 619, 630, 636, 642, 647, 570, 571, - 572, 819, 575, 584, 570, 571, 572, 819, - 576, 580, 570, 571, 572, 819, 577, 570, - 571, 572, 819, 578, 570, 579, 572, 819, - 570, 571, 572, 819, 570, 571, 572, 819, - 581, 570, 571, 572, 819, 582, 570, 583, - 572, 819, 570, 571, 572, 819, 570, 571, - 572, 819, 585, 570, 571, 572, 819, 586, - 570, 571, 572, 819, 587, 570, 588, 572, - 819, 570, 571, 572, 819, 570, 571, 572, - 819, 590, 570, 571, 572, 819, 591, 570, - 571, 572, 819, 592, 570, 571, 572, 819, - 593, 570, 594, 572, 819, 570, 571, 572, - 819, 570, 571, 572, 819, 596, 570, 571, - 572, 819, 597, 570, 571, 572, 819, 598, - 570, 571, 572, 819, 599, 570, 600, 572, - 819, 570, 571, 572, 819, 570, 571, 572, - 819, 602, 570, 571, 572, 819, 603, 570, - 571, 572, 819, 604, 570, 571, 572, 819, - 605, 570, 606, 572, 819, 570, 571, 572, - 819, 570, 571, 572, 819, 608, 570, 571, - 572, 819, 609, 570, 571, 572, 819, 610, - 570, 571, 572, 819, 611, 570, 612, 572, - 819, 570, 571, 572, 819, 570, 571, 572, - 819, 614, 570, 571, 572, 819, 615, 570, - 571, 572, 819, 616, 570, 571, 572, 819, - 617, 570, 618, 572, 819, 570, 571, 572, - 819, 570, 571, 572, 819, 620, 625, 570, - 571, 572, 819, 621, 570, 571, 572, 819, - 622, 570, 571, 572, 819, 623, 570, 624, - 572, 819, 570, 571, 572, 819, 570, 571, - 572, 819, 626, 570, 571, 572, 819, 627, - 570, 571, 572, 819, 628, 570, 629, 572, - 819, 570, 571, 572, 819, 570, 571, 572, - 819, 631, 570, 571, 572, 819, 632, 570, - 571, 572, 819, 633, 570, 571, 572, 819, - 634, 570, 635, 572, 819, 570, 571, 572, - 819, 570, 571, 572, 819, 637, 570, 571, - 572, 819, 638, 570, 571, 572, 819, 639, - 570, 571, 572, 819, 640, 570, 641, 572, - 819, 570, 571, 572, 819, 570, 571, 572, - 819, 643, 570, 571, 572, 819, 644, 570, - 571, 572, 819, 645, 570, 646, 572, 819, - 570, 571, 572, 819, 570, 571, 572, 819, - 648, 570, 571, 572, 819, 649, 570, 571, - 572, 819, 650, 570, 571, 572, 819, 651, - 570, 571, 572, 819, 652, 570, 653, 572, - 819, 570, 571, 572, 819, 570, 571, 572, - 819, 655, 664, 570, 571, 572, 819, 656, - 660, 570, 571, 572, 819, 657, 570, 571, - 572, 819, 658, 570, 659, 572, 819, 570, - 571, 572, 819, 570, 571, 572, 819, 661, - 570, 571, 572, 819, 662, 570, 663, 572, - 819, 570, 571, 572, 819, 570, 571, 572, - 819, 665, 570, 571, 572, 819, 666, 570, - 571, 572, 819, 667, 570, 668, 572, 819, - 570, 571, 572, 819, 570, 571, 572, 819, - 670, 570, 571, 572, 819, 671, 570, 571, - 572, 819, 672, 570, 571, 572, 819, 673, - 570, 674, 572, 819, 570, 571, 572, 819, - 570, 571, 572, 819, 676, 570, 571, 572, - 819, 677, 570, 571, 572, 819, 678, 570, - 571, 572, 819, 679, 570, 680, 572, 819, - 570, 571, 572, 819, 570, 571, 572, 819, - 682, 570, 571, 572, 819, 683, 570, 571, - 572, 819, 684, 570, 571, 572, 819, 685, - 570, 686, 572, 819, 570, 571, 572, 819, - 570, 571, 572, 819, 688, 570, 571, 572, - 819, 689, 570, 571, 572, 819, 690, 570, - 571, 572, 819, 691, 570, 692, 572, 819, - 570, 571, 572, 819, 570, 571, 572, 819, - 694, 570, 571, 572, 819, 695, 570, 571, - 572, 819, 696, 570, 571, 572, 819, 697, - 570, 698, 572, 819, 570, 571, 572, 819, - 570, 571, 572, 819, 700, 705, 570, 571, - 572, 819, 701, 570, 571, 572, 819, 702, - 570, 571, 572, 819, 703, 570, 704, 572, - 819, 570, 571, 572, 819, 570, 571, 572, - 819, 706, 570, 571, 572, 819, 707, 570, - 571, 572, 819, 708, 570, 709, 572, 819, - 570, 571, 572, 819, 570, 571, 572, 819, - 711, 570, 571, 572, 819, 712, 570, 571, - 572, 819, 713, 570, 571, 572, 819, 714, - 570, 715, 572, 819, 570, 571, 572, 819, - 570, 571, 572, 819, 717, 570, 571, 572, - 819, 718, 570, 571, 572, 819, 719, 570, - 571, 572, 819, 720, 570, 721, 572, 819, - 570, 571, 572, 819, 570, 571, 572, 819, - 723, 570, 571, 572, 819, 724, 570, 571, - 572, 819, 725, 570, 726, 572, 819, 570, - 571, 572, 819, 570, 571, 572, 819, 728, - 570, 571, 572, 819, 729, 570, 571, 572, - 819, 730, 570, 571, 572, 819, 731, 570, - 571, 572, 819, 732, 570, 733, 572, 819, - 570, 571, 572, 819, 570, 735, 736, 819, - 734, 735, 736, 819, 734, 735, 736, 734, - 738, 819, 819, 738, 819, 819, 739, 739, - 739, 819, 838, 838, 838, 838, 742, 838, - 838, 838, 838, 843, 843, 843, 843, 745, - 843, 843, 843, 843, 746, 746, 746, 746, - 746, 750, 746, 762, 763, 746, 764, 765, - 766, 746, 785, 746, 746, 746, 746, 747, - 746, 748, 746, 749, 746, 746, 746, 746, - 746, 746, 746, 746, 1, 746, 746, 2, - 746, 4, 5, 746, 6, 6, 6, 6, - 746, 7, 7, 7, 7, 746, 8, 11, - 14, 16, 18, 11, 12, 13, 11, 11, - 746, 22, 746, 23, 23, 23, 23, 22, - 746, 746, 22, 746, 746, 746, 24, 24, - 24, 24, 746, 746, 746, 746, 25, 746, - 26, 27, 29, 746, 746, 30, 746, 31, - 31, 31, 31, 746, 746, 746, 746, 746, - 746, 746, 746, 746, 746, 32, 35, 38, - 746, 767, 746, 746, 746, 746, 746, 746, - 746, 746, 746, 746, 772, 746, 746, 746, - 746, 746, 746, 746, 746, 746, 746, 773, - 746, 746, 746, 774, 746, 779, 746, 746, - 780, 781, 746, 746, 746, 746, 746, 746, - 782, 746, 769, 771, 746, 768, 746, 746, - 746, 770, 771, 746, 771, 771, 746, 771, - 746, 746, 746, 746, 41, 42, 43, 44, - 777, 746, 776, 746, 746, 746, 778, 746, - 746, 746, 53, 55, 57, 746, 59, 746, - 746, 746, 784, 783, 783, 783, 746, 746, - 746, 746, 746, 61, 61, 61, 746, 62, - 746, 746, 746, 746, 787, 789, 788, 787, + 561, 564, 565, 819, 563, 564, 565, 819, + 563, 564, 565, 563, 568, 569, 819, 570, + 651, 666, 672, 678, 684, 690, 696, 707, + 713, 719, 724, 567, 568, 569, 819, 567, + 568, 569, 819, 567, 568, 569, 567, 568, + 569, 819, 571, 586, 592, 598, 604, 610, + 616, 627, 633, 639, 644, 567, 568, 569, + 819, 572, 581, 567, 568, 569, 819, 573, + 577, 567, 568, 569, 819, 574, 567, 568, + 569, 819, 575, 567, 576, 569, 819, 567, + 568, 569, 819, 567, 568, 569, 819, 578, + 567, 568, 569, 819, 579, 567, 580, 569, + 819, 567, 568, 569, 819, 567, 568, 569, + 819, 582, 567, 568, 569, 819, 583, 567, + 568, 569, 819, 584, 567, 585, 569, 819, + 567, 568, 569, 819, 567, 568, 569, 819, + 587, 567, 568, 569, 819, 588, 567, 568, + 569, 819, 589, 567, 568, 569, 819, 590, + 567, 591, 569, 819, 567, 568, 569, 819, + 567, 568, 569, 819, 593, 567, 568, 569, + 819, 594, 567, 568, 569, 819, 595, 567, + 568, 569, 819, 596, 567, 597, 569, 819, + 567, 568, 569, 819, 567, 568, 569, 819, + 599, 567, 568, 569, 819, 600, 567, 568, + 569, 819, 601, 567, 568, 569, 819, 602, + 567, 603, 569, 819, 567, 568, 569, 819, + 567, 568, 569, 819, 605, 567, 568, 569, + 819, 606, 567, 568, 569, 819, 607, 567, + 568, 569, 819, 608, 567, 609, 569, 819, + 567, 568, 569, 819, 567, 568, 569, 819, + 611, 567, 568, 569, 819, 612, 567, 568, + 569, 819, 613, 567, 568, 569, 819, 614, + 567, 615, 569, 819, 567, 568, 569, 819, + 567, 568, 569, 819, 617, 622, 567, 568, + 569, 819, 618, 567, 568, 569, 819, 619, + 567, 568, 569, 819, 620, 567, 621, 569, + 819, 567, 568, 569, 819, 567, 568, 569, + 819, 623, 567, 568, 569, 819, 624, 567, + 568, 569, 819, 625, 567, 626, 569, 819, + 567, 568, 569, 819, 567, 568, 569, 819, + 628, 567, 568, 569, 819, 629, 567, 568, + 569, 819, 630, 567, 568, 569, 819, 631, + 567, 632, 569, 819, 567, 568, 569, 819, + 567, 568, 569, 819, 634, 567, 568, 569, + 819, 635, 567, 568, 569, 819, 636, 567, + 568, 569, 819, 637, 567, 638, 569, 819, + 567, 568, 569, 819, 567, 568, 569, 819, + 640, 567, 568, 569, 819, 641, 567, 568, + 569, 819, 642, 567, 643, 569, 819, 567, + 568, 569, 819, 567, 568, 569, 819, 645, + 567, 568, 569, 819, 646, 567, 568, 569, + 819, 647, 567, 568, 569, 819, 648, 567, + 568, 569, 819, 649, 567, 650, 569, 819, + 567, 568, 569, 819, 567, 568, 569, 819, + 652, 661, 567, 568, 569, 819, 653, 657, + 567, 568, 569, 819, 654, 567, 568, 569, + 819, 655, 567, 656, 569, 819, 567, 568, + 569, 819, 567, 568, 569, 819, 658, 567, + 568, 569, 819, 659, 567, 660, 569, 819, + 567, 568, 569, 819, 567, 568, 569, 819, + 662, 567, 568, 569, 819, 663, 567, 568, + 569, 819, 664, 567, 665, 569, 819, 567, + 568, 569, 819, 567, 568, 569, 819, 667, + 567, 568, 569, 819, 668, 567, 568, 569, + 819, 669, 567, 568, 569, 819, 670, 567, + 671, 569, 819, 567, 568, 569, 819, 567, + 568, 569, 819, 673, 567, 568, 569, 819, + 674, 567, 568, 569, 819, 675, 567, 568, + 569, 819, 676, 567, 677, 569, 819, 567, + 568, 569, 819, 567, 568, 569, 819, 679, + 567, 568, 569, 819, 680, 567, 568, 569, + 819, 681, 567, 568, 569, 819, 682, 567, + 683, 569, 819, 567, 568, 569, 819, 567, + 568, 569, 819, 685, 567, 568, 569, 819, + 686, 567, 568, 569, 819, 687, 567, 568, + 569, 819, 688, 567, 689, 569, 819, 567, + 568, 569, 819, 567, 568, 569, 819, 691, + 567, 568, 569, 819, 692, 567, 568, 569, + 819, 693, 567, 568, 569, 819, 694, 567, + 695, 569, 819, 567, 568, 569, 819, 567, + 568, 569, 819, 697, 702, 567, 568, 569, + 819, 698, 567, 568, 569, 819, 699, 567, + 568, 569, 819, 700, 567, 701, 569, 819, + 567, 568, 569, 819, 567, 568, 569, 819, + 703, 567, 568, 569, 819, 704, 567, 568, + 569, 819, 705, 567, 706, 569, 819, 567, + 568, 569, 819, 567, 568, 569, 819, 708, + 567, 568, 569, 819, 709, 567, 568, 569, + 819, 710, 567, 568, 569, 819, 711, 567, + 712, 569, 819, 567, 568, 569, 819, 567, + 568, 569, 819, 714, 567, 568, 569, 819, + 715, 567, 568, 569, 819, 716, 567, 568, + 569, 819, 717, 567, 718, 569, 819, 567, + 568, 569, 819, 567, 568, 569, 819, 720, + 567, 568, 569, 819, 721, 567, 568, 569, + 819, 722, 567, 723, 569, 819, 567, 568, + 569, 819, 567, 568, 569, 819, 725, 567, + 568, 569, 819, 726, 567, 568, 569, 819, + 727, 567, 568, 569, 819, 728, 567, 568, + 569, 819, 729, 567, 730, 569, 819, 567, + 568, 569, 819, 567, 732, 733, 819, 731, + 732, 733, 819, 731, 732, 733, 731, 735, + 819, 819, 735, 819, 819, 736, 736, 736, + 819, 819, 819, 819, 819, 739, 819, 819, + 819, 819, 839, 839, 839, 839, 742, 839, + 839, 839, 839, 844, 844, 844, 844, 745, + 844, 844, 844, 844, 746, 746, 746, 746, + 746, 747, 746, 759, 760, 746, 761, 762, + 763, 746, 782, 746, 746, 746, 746, 746, + 784, 746, 785, 746, 786, 746, 746, 746, + 1, 2, 746, 3, 3, 3, 3, 746, + 4, 4, 4, 4, 746, 5, 8, 11, + 13, 15, 8, 9, 10, 8, 8, 746, + 19, 746, 20, 20, 20, 20, 19, 746, + 746, 19, 746, 746, 746, 21, 21, 21, + 21, 746, 746, 746, 746, 22, 746, 23, + 24, 26, 746, 746, 27, 746, 28, 28, + 28, 28, 746, 746, 746, 746, 746, 746, + 746, 746, 746, 746, 29, 32, 35, 746, + 764, 746, 746, 746, 746, 746, 746, 746, + 746, 746, 746, 769, 746, 746, 746, 746, + 746, 746, 746, 746, 746, 746, 770, 746, + 746, 746, 771, 746, 776, 746, 746, 777, + 778, 746, 746, 746, 746, 746, 746, 779, + 746, 766, 768, 746, 765, 746, 746, 746, + 767, 768, 746, 768, 768, 746, 768, 746, + 746, 746, 746, 38, 39, 40, 41, 774, + 746, 773, 746, 746, 746, 775, 746, 746, + 746, 50, 52, 54, 746, 56, 746, 746, + 746, 781, 780, 780, 780, 746, 746, 746, + 746, 746, 58, 58, 58, 746, 59, 746, + 746, 746, 746, 746, 746, 746, 746, 62, + 746, 746, 63, 746, 787, 789, 788, 787, 65, 787, 66, 68, 65, 791, 792, 793, 795, 796, 797, 798, 800, 801, 802, 803, 804, 806, 807, 808, 809, 810, 811, 812, @@ -3193,25 +3194,26 @@ static const short _regex_actions[] = { 548, 790, 554, 790, 555, 556, 557, 558, 790, 790, 790, 790, 790, 790, 790, 0, 818, 818, 818, 818, 818, 818, 818, 818, - 819, 823, 824, 819, 819, 819, 819, 820, - 819, 821, 819, 822, 819, 819, 819, 819, - 819, 819, 819, 563, 819, 819, 564, 819, - 566, 569, 734, 819, 825, 819, 819, 819, - 819, 819, 829, 819, 819, 819, 819, 819, - 819, 819, 830, 819, 819, 819, 819, 819, - 819, 819, 831, 832, 819, 819, 819, 819, - 819, 819, 833, 827, 819, 819, 819, 819, - 826, 819, 819, 819, 828, 819, 819, 819, - 819, 819, 819, 737, 819, 819, 819, 835, - 834, 834, 834, 819, 819, 819, 819, 819, - 739, 739, 739, 819, 837, 836, 836, 836, - 836, 836, 836, 836, 836, 842, 838, 838, - 838, 839, 838, 840, 838, 841, 838, 838, - 838, 838, 838, 838, 838, 740, 838, 838, - 741, 838, 838, 838, 847, 843, 843, 843, - 844, 843, 845, 843, 846, 843, 843, 843, - 843, 843, 843, 843, 743, 843, 843, 744, - 843, 843, 843, 848, 848, 849, 849, 0, + 820, 819, 821, 822, 819, 819, 819, 819, + 834, 819, 835, 819, 836, 819, 819, 819, + 819, 819, 563, 566, 731, 819, 823, 819, + 819, 819, 819, 819, 827, 819, 819, 819, + 819, 819, 819, 819, 828, 819, 819, 819, + 819, 819, 819, 819, 829, 830, 819, 819, + 819, 819, 819, 819, 831, 825, 819, 819, + 819, 819, 824, 819, 819, 819, 826, 819, + 819, 819, 819, 819, 819, 734, 819, 819, + 819, 833, 832, 832, 832, 819, 819, 819, + 819, 819, 736, 736, 736, 819, 819, 819, + 819, 819, 737, 819, 819, 738, 819, 838, + 837, 837, 837, 837, 837, 837, 837, 837, + 840, 839, 839, 839, 841, 839, 842, 839, + 843, 839, 839, 839, 839, 839, 839, 839, + 839, 839, 740, 839, 839, 741, 839, 845, + 844, 844, 844, 846, 844, 847, 844, 848, + 844, 844, 844, 844, 844, 844, 844, 844, + 844, 743, 844, 844, 744, 844, 849, 849, + 850, 850, 0, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, @@ -3219,8 +3221,8 @@ static const short _regex_actions[] = { 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, - 746, 746, 746, 746, 746, 746, 746, 746, - 787, 787, 787, 787, 787, 787, 790, 790, + 746, 746, 746, 787, 787, 787, 787, 787, + 787, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, @@ -3281,8 +3283,7 @@ static const short _regex_actions[] = { 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, - 790, 790, 790, 790, 790, 790, 559, 560, - 561, 562, 819, 819, 819, 819, 819, 819, + 790, 559, 560, 561, 562, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, @@ -3304,62 +3305,62 @@ static const short _regex_actions[] = { 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, - 819, 819, 819, 838, 838, 838, 843, 843, - 843, 746, 746, 746, 746, 746, 746, 746, + 819, 819, 819, 819, 819, 819, 839, 839, + 839, 844, 844, 844, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, 746, - 746, 746, 787, 787, 787, 790, 790, 790, + 746, 746, 746, 746, 746, 787, 787, 787, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, 790, - 817, 818, 819, 819, 819, 819, 819, 819, + 790, 790, 790, 817, 818, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, 819, - 819, 819, 819, 836, 836, 838, 838, 838, - 838, 838, 843, 843, 843, 843, 843, 848, - 849, 0 + 819, 819, 819, 819, 819, 819, 819, 837, + 837, 839, 839, 839, 839, 839, 844, 844, + 844, 844, 844, 849, 850, 0 }; static const short _regex_cond_actions[] = { - 843, 769, 843, 843, 0, 843, 843, 771, - 843, 827, 631, 731, 723, 45, 903, 903, - 897, 45, 912, 900, 903, 729, 741, 0, - 45, 45, 923, 923, 923, 923, 737, 45, - 765, 747, 0, 0, 0, 0, 841, 743, - 3, 3, 3, 3, 841, 3, 3, 3, - 3, 839, 0, 3, 3, 3, 3, 839, - 761, 839, 761, 3, 3, 3, 3, 839, - 759, 3, 870, 3, 3, 839, 761, 3, - 870, 3, 3, 839, 3, 3, 3, 3, - 839, 0, 3, 3, 3, 3, 839, 751, - 0, 749, 839, 755, 753, 839, 0, 757, - 3, 3, 3, 3, 839, 0, 0, 0, - 0, 839, 757, 0, 0, 0, 0, 839, - 757, 3, 3, 3, 3, 839, 745, 0, - 841, 725, 727, 27, 27, 27, 27, 841, - 743, 3, 3, 3, 3, 841, 763, 0, - 841, 3, 3, 3, 3, 841, 3, 3, - 3, 3, 841, 673, 3, 3, 3, 3, - 841, 0, 0, 0, 0, 841, 27, 27, - 27, 27, 841, 725, 0, 727, 25, 25, - 25, 25, 841, 0, 0, 829, 0, 0, - 0, 603, 0, 0, 0, 0, 0, 0, - 829, 0, 0, 0, 601, 0, 0, 0, - 0, 0, 0, 829, 0, 0, 0, 603, - 0, 0, 0, 0, 675, 0, 17, 831, - 675, 0, 7, 3, 870, 3, 3, 831, - 17, 831, 663, 17, 831, 663, 17, 831, - 663, 831, 3, 661, 870, 3, 3, 831, - 3, 661, 870, 3, 3, 831, 3, 661, - 3, 3, 3, 831, 3, 665, 3, 3, - 3, 831, 3, 3, 3, 3, 837, 671, - 3, 3, 3, 3, 837, 3, 3, 3, - 3, 837, 669, 3, 3, 3, 3, 837, - 3, 3, 3, 3, 837, 3, 667, 3, - 3, 3, 837, 0, 833, 677, 0, 833, - 679, 0, 0, 0, 835, 0, 29, 11, - 845, 31, 13, 845, 0, 13, 845, 53, + 831, 635, 735, 727, 45, 907, 907, 901, + 45, 916, 904, 907, 733, 745, 0, 45, + 45, 927, 927, 927, 927, 741, 45, 769, + 751, 0, 0, 0, 0, 845, 747, 3, + 3, 3, 3, 845, 3, 3, 3, 3, + 843, 0, 3, 3, 3, 3, 843, 765, + 843, 765, 3, 3, 3, 3, 843, 763, + 3, 874, 3, 3, 843, 765, 3, 874, + 3, 3, 843, 3, 3, 3, 3, 843, + 0, 3, 3, 3, 3, 843, 755, 0, + 753, 843, 759, 757, 843, 0, 761, 3, + 3, 3, 3, 843, 0, 0, 0, 0, + 843, 761, 0, 0, 0, 0, 843, 761, + 3, 3, 3, 3, 843, 749, 0, 845, + 729, 731, 27, 27, 27, 27, 845, 747, + 3, 3, 3, 3, 845, 767, 0, 845, + 3, 3, 3, 3, 845, 3, 3, 3, + 3, 845, 677, 3, 3, 3, 3, 845, + 0, 0, 0, 0, 845, 27, 27, 27, + 27, 845, 729, 0, 731, 25, 25, 25, + 25, 845, 0, 0, 833, 0, 0, 0, + 607, 0, 0, 0, 0, 0, 0, 833, + 0, 0, 0, 605, 0, 0, 0, 0, + 0, 0, 833, 0, 0, 0, 607, 0, + 0, 0, 0, 679, 0, 17, 835, 679, + 0, 7, 3, 874, 3, 3, 835, 17, + 835, 667, 17, 835, 667, 17, 835, 667, + 835, 3, 665, 874, 3, 3, 835, 3, + 665, 874, 3, 3, 835, 3, 665, 3, + 3, 3, 835, 3, 669, 3, 3, 3, + 835, 3, 3, 3, 3, 841, 675, 3, + 3, 3, 3, 841, 3, 3, 3, 3, + 841, 673, 3, 3, 3, 3, 841, 3, + 3, 3, 3, 841, 3, 671, 3, 3, + 3, 841, 0, 837, 681, 0, 837, 683, + 0, 0, 0, 839, 0, 29, 11, 849, + 31, 13, 849, 0, 13, 849, 847, 773, + 847, 847, 0, 847, 847, 775, 847, 53, 0, 53, 0, 0, 51, 0, 53, 0, 0, 49, 0, 0, 47, 0, 315, 359, 0, 0, 359, 0, 359, 0, 359, 127, @@ -3489,139 +3490,139 @@ static const short _regex_actions[] = { 359, 0, 359, 0, 359, 309, 359, 311, 359, 119, 359, 121, 359, 123, 359, 125, 359, 0, 39, 33, 35, 37, 39, 39, - 35, 537, 503, 537, 537, 0, 537, 537, - 505, 537, 0, 0, 539, 0, 0, 0, - 377, 0, 0, 0, 0, 0, 0, 539, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 539, - 0, 0, 0, 435, 0, 0, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 0, 539, - 0, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 381, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 385, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 389, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 393, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 397, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 401, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 405, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 409, 0, 0, 0, 539, 0, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 413, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 417, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 421, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 425, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 429, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 433, 0, 0, 0, - 539, 0, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 379, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 383, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 387, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 391, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 395, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 399, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 403, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 407, - 0, 0, 0, 539, 0, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 411, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 415, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 419, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 423, 0, 0, 0, 539, - 0, 0, 0, 0, 539, 0, 0, 0, - 0, 539, 0, 0, 0, 0, 539, 0, - 0, 0, 427, 0, 0, 0, 539, 0, - 0, 0, 0, 539, 0, 0, 0, 0, - 539, 0, 0, 0, 0, 539, 0, 0, - 0, 0, 539, 0, 0, 0, 0, 539, - 0, 0, 0, 431, 0, 0, 0, 539, - 0, 0, 0, 377, 0, 0, 0, 0, - 0, 533, 471, 0, 533, 475, 0, 0, - 0, 535, 569, 557, 569, 569, 0, 569, - 569, 559, 569, 587, 575, 587, 587, 0, - 587, 587, 577, 587, 633, 775, 777, 629, - 637, 45, 597, 0, 0, 609, 0, 45, - 0, 635, 909, 599, 777, 773, 777, 0, - 777, 45, 777, 45, 777, 773, 775, 777, - 823, 767, 823, 823, 0, 823, 823, 0, - 823, 0, 0, 779, 0, 0, 0, 0, - 821, 3, 3, 3, 3, 821, 1, 3, - 1, 0, 3, 3, 855, 855, 3, 3, - 819, 0, 821, 27, 27, 27, 27, 0, - 821, 745, 0, 821, 735, 733, 3, 3, - 3, 3, 821, 739, 821, 763, 0, 821, - 1, 1, 0, 821, 725, 0, 727, 25, - 25, 25, 25, 821, 615, 613, 783, 619, - 617, 785, 623, 621, 787, 0, 0, 0, - 781, 5, 639, 647, 611, 695, 607, 717, - 699, 715, 683, 683, 0, 605, 713, 691, - 683, 703, 687, 719, 641, 657, 645, 0, - 693, 659, 655, 906, 697, 45, 683, 651, - 45, 0, 653, 689, 649, 683, 701, 685, - 7, 643, 852, 915, 721, 15, 791, 867, - 791, 858, 919, 795, 927, 919, 793, 919, - 847, 709, 711, 681, 0, 7, 0, 849, - 17, 801, 17, 799, 876, 799, 17, 797, - 873, 797, 1, 1, 1, 815, 0, 803, - 705, 707, 45, 19, 23, 21, 805, 882, - 894, 888, 805, 0, 0, 0, 807, 11, - 825, 627, 625, 789, 55, 45, 45, 53, + 35, 0, 0, 543, 0, 0, 0, 377, + 0, 0, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 543, 0, + 0, 0, 435, 0, 0, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 381, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 385, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 389, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 393, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 397, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 401, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 405, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 409, + 0, 0, 0, 543, 0, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 413, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 417, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 421, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 425, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 429, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 433, 0, 0, 0, 543, + 0, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 379, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 383, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 387, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 391, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 395, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 399, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 403, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 407, 0, + 0, 0, 543, 0, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 411, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 415, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 419, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 423, 0, 0, 0, 543, 0, + 0, 0, 0, 543, 0, 0, 0, 0, + 543, 0, 0, 0, 0, 543, 0, 0, + 0, 427, 0, 0, 0, 543, 0, 0, + 0, 0, 543, 0, 0, 0, 0, 543, + 0, 0, 0, 0, 543, 0, 0, 0, + 0, 543, 0, 0, 0, 0, 543, 0, + 0, 0, 431, 0, 0, 0, 543, 0, + 0, 0, 377, 0, 0, 0, 0, 0, + 537, 471, 0, 537, 475, 0, 0, 0, + 539, 541, 503, 541, 541, 0, 541, 541, + 505, 541, 573, 561, 573, 573, 0, 573, + 573, 563, 573, 591, 579, 591, 591, 0, + 591, 591, 581, 591, 637, 779, 781, 633, + 641, 45, 601, 0, 0, 613, 0, 45, + 0, 639, 913, 603, 779, 781, 777, 781, + 0, 781, 45, 781, 45, 781, 777, 781, + 0, 0, 783, 0, 0, 0, 0, 825, + 3, 3, 3, 3, 825, 1, 3, 1, + 0, 3, 3, 859, 859, 3, 3, 823, + 0, 825, 27, 27, 27, 27, 0, 825, + 749, 0, 825, 739, 737, 3, 3, 3, + 3, 825, 743, 825, 767, 0, 825, 1, + 1, 0, 825, 729, 0, 731, 25, 25, + 25, 25, 825, 619, 617, 787, 623, 621, + 789, 627, 625, 791, 0, 0, 0, 785, + 5, 643, 651, 615, 699, 611, 721, 703, + 719, 687, 687, 0, 609, 717, 695, 687, + 707, 691, 723, 645, 661, 649, 0, 697, + 663, 659, 910, 701, 45, 687, 655, 45, + 0, 657, 693, 653, 687, 705, 689, 7, + 647, 856, 919, 725, 15, 795, 871, 795, + 862, 923, 799, 931, 923, 797, 923, 851, + 713, 715, 685, 0, 7, 0, 853, 17, + 805, 17, 803, 880, 803, 17, 801, 877, + 801, 1, 1, 1, 819, 0, 807, 709, + 711, 45, 19, 23, 21, 809, 886, 898, + 892, 809, 0, 0, 0, 811, 11, 829, + 631, 629, 793, 827, 771, 827, 827, 0, + 827, 827, 0, 827, 55, 45, 45, 53, 0, 53, 0, 0, 0, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, 45, @@ -3642,47 +3643,47 @@ static const short _regex_actions[] = { 0, 339, 0, 339, 0, 0, 0, 0, 339, 313, 339, 113, 115, 117, 335, 39, 361, 363, 365, 367, 369, 371, 373, 375, - 491, 45, 0, 511, 509, 507, 509, 0, - 509, 45, 509, 45, 509, 507, 509, 529, - 501, 529, 529, 0, 529, 529, 0, 529, - 0, 0, 0, 531, 5, 489, 439, 457, - 493, 493, 0, 437, 485, 493, 461, 481, - 451, 441, 0, 487, 453, 449, 495, 455, - 493, 445, 45, 0, 447, 483, 443, 493, - 459, 479, 7, 9, 473, 497, 497, 499, - 15, 517, 861, 517, 15, 519, 864, 519, - 467, 469, 477, 0, 521, 463, 465, 45, - 19, 23, 21, 523, 879, 891, 885, 523, - 0, 0, 0, 525, 0, 549, 543, 549, - 541, 549, 547, 545, 551, 0, 563, 561, - 563, 0, 563, 45, 563, 45, 563, 561, - 563, 565, 555, 565, 565, 0, 565, 565, - 0, 565, 553, 567, 0, 581, 579, 581, - 0, 581, 45, 581, 45, 581, 579, 581, - 583, 573, 583, 583, 0, 583, 583, 0, - 583, 571, 585, 589, 591, 593, 595, 0, - 843, 843, 843, 827, 827, 841, 841, 839, - 839, 839, 839, 839, 839, 839, 839, 839, - 839, 839, 839, 839, 839, 841, 841, 841, - 841, 841, 841, 841, 841, 841, 841, 829, - 829, 829, 829, 829, 829, 829, 829, 829, - 831, 831, 831, 831, 831, 831, 831, 831, - 831, 831, 831, 831, 837, 837, 837, 837, - 837, 837, 833, 833, 835, 845, 845, 845, - 59, 59, 59, 59, 59, 59, 359, 359, - 359, 359, 359, 359, 359, 359, 359, 359, + 0, 491, 45, 0, 513, 511, 507, 511, + 0, 511, 45, 511, 45, 511, 507, 511, + 509, 535, 0, 0, 0, 533, 5, 489, + 439, 457, 493, 493, 0, 437, 485, 493, + 461, 481, 451, 441, 0, 487, 453, 449, + 495, 455, 493, 445, 45, 0, 447, 483, + 443, 493, 459, 479, 7, 9, 473, 497, + 497, 499, 15, 519, 865, 519, 15, 521, + 868, 521, 467, 469, 477, 0, 523, 463, + 465, 45, 19, 23, 21, 525, 883, 895, + 889, 525, 0, 0, 0, 527, 531, 501, + 531, 531, 0, 531, 531, 0, 531, 0, + 553, 547, 553, 545, 553, 551, 549, 555, + 0, 567, 565, 567, 0, 567, 45, 567, + 45, 567, 565, 567, 557, 571, 569, 559, + 569, 569, 0, 569, 569, 0, 569, 0, + 585, 583, 585, 0, 585, 45, 585, 45, + 585, 583, 585, 575, 589, 587, 577, 587, + 587, 0, 587, 587, 0, 587, 593, 595, + 597, 599, 0, 831, 831, 845, 845, 843, + 843, 843, 843, 843, 843, 843, 843, 843, + 843, 843, 843, 843, 843, 845, 845, 845, + 845, 845, 845, 845, 845, 845, 845, 833, + 833, 833, 833, 833, 833, 833, 833, 833, + 835, 835, 835, 835, 835, 835, 835, 835, + 835, 835, 835, 835, 841, 841, 841, 841, + 841, 841, 837, 837, 839, 849, 849, 849, + 847, 847, 847, 59, 59, 59, 59, 59, + 59, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, - 359, 341, 341, 341, 341, 341, 341, 341, + 359, 359, 359, 359, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, - 341, 341, 341, 341, 343, 343, 343, 343, - 343, 343, 341, 341, 341, 341, 341, 341, + 341, 341, 341, 341, 341, 341, 341, 343, + 343, 343, 343, 343, 343, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, 341, - 341, 341, 341, 359, 359, 359, 359, 359, + 341, 341, 341, 341, 341, 341, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, @@ -3691,7 +3692,8 @@ static const short _regex_actions[] = { 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, - 357, 357, 357, 357, 357, 359, 359, 359, + 359, 359, 359, 357, 357, 357, 357, 357, + 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, @@ -3702,72 +3704,71 @@ static const short _regex_actions[] = { 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, - 359, 359, 359, 359, 359, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, 345, - 345, 345, 345, 345, 347, 347, 347, 347, - 347, 347, 347, 347, 347, 347, 347, 349, - 349, 349, 349, 349, 349, 349, 349, 349, + 345, 345, 345, 345, 345, 345, 345, 347, 347, 347, 347, 347, 347, 347, 347, 347, - 347, 347, 347, 347, 351, 351, 351, 351, - 351, 351, 351, 351, 351, 351, 359, 359, + 347, 347, 349, 349, 349, 349, 349, 349, + 349, 349, 349, 347, 347, 347, 347, 347, + 347, 347, 347, 347, 347, 347, 347, 351, + 351, 351, 351, 351, 351, 351, 351, 351, + 351, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, - 359, 359, 359, 359, 359, 353, 353, 353, 353, 353, 353, 353, 353, 353, 353, 353, - 353, 353, 359, 359, 359, 359, 359, 359, - 359, 355, 355, 355, 355, 355, 355, 355, + 353, 353, 353, 353, 353, 359, 359, 359, + 359, 359, 359, 359, 355, 355, 355, 355, + 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, 355, - 355, 355, 355, 355, 355, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, 359, - 359, 359, 359, 359, 359, 359, 39, 39, - 39, 39, 537, 537, 537, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 539, 539, 539, 539, 539, 539, 539, 539, - 533, 533, 535, 569, 569, 569, 587, 587, - 587, 0, 823, 823, 823, 779, 821, 821, - 819, 821, 821, 821, 821, 821, 821, 821, - 821, 783, 785, 787, 781, 817, 791, 791, - 795, 793, 847, 811, 809, 801, 799, 799, - 797, 797, 815, 803, 813, 805, 805, 807, - 825, 789, 0, 57, 57, 0, 339, 339, - 319, 321, 339, 339, 339, 339, 337, 339, - 339, 339, 323, 325, 327, 329, 339, 331, - 339, 333, 339, 339, 339, 339, 339, 335, - 0, 0, 0, 529, 529, 529, 531, 531, - 517, 517, 519, 519, 513, 527, 521, 515, - 523, 523, 525, 0, 551, 0, 565, 565, - 565, 567, 0, 583, 583, 583, 585, 0, - 0, 0 + 359, 359, 359, 359, 359, 359, 359, 359, + 359, 39, 39, 39, 39, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 543, 543, 543, 543, 543, 543, 543, 543, + 537, 537, 539, 541, 541, 541, 573, 573, + 573, 591, 591, 591, 0, 783, 825, 825, + 823, 825, 825, 825, 825, 825, 825, 825, + 825, 787, 789, 791, 785, 821, 795, 795, + 799, 797, 851, 815, 813, 805, 803, 803, + 801, 801, 819, 807, 817, 809, 809, 811, + 829, 793, 827, 827, 827, 0, 57, 57, + 0, 339, 339, 319, 321, 339, 339, 339, + 339, 337, 339, 339, 339, 323, 325, 327, + 329, 339, 331, 339, 333, 339, 339, 339, + 339, 339, 335, 0, 0, 0, 535, 533, + 535, 519, 519, 521, 521, 515, 529, 523, + 517, 525, 525, 527, 531, 531, 531, 0, + 555, 0, 571, 569, 569, 569, 0, 589, + 587, 587, 587, 0, 0, 0 }; static const signed char _regex_to_state_actions[] = { @@ -3875,9 +3876,9 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 41, 0, 41, 0, - 0, 0, 0, 41, 0, 0, 0, 0, - 41, 41, 0 + 0, 0, 0, 0, 0, 41, 0, 41, + 0, 0, 0, 0, 41, 0, 0, 0, + 0, 41, 41, 0 }; static const signed char _regex_from_state_actions[] = { @@ -3985,119 +3986,119 @@ static const short _regex_actions[] = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 43, 0, 43, 0, - 0, 0, 0, 43, 0, 0, 0, 0, - 43, 43, 0 + 0, 0, 0, 0, 0, 43, 0, 43, + 0, 0, 0, 0, 43, 0, 0, 0, + 0, 43, 43, 0 }; static const short _regex_eof_trans[] = { - 2665, 2666, 2667, 2668, 2669, 2670, 2671, 2672, - 2673, 2674, 2675, 2676, 2677, 2678, 2679, 2680, - 2681, 2682, 2683, 2684, 2685, 2686, 2687, 2688, - 2689, 2690, 2691, 2692, 2693, 2694, 2695, 2696, - 2697, 2698, 2699, 2700, 2701, 2702, 2703, 2704, - 2705, 2706, 2707, 2708, 2709, 2710, 2711, 2712, - 2713, 2714, 2715, 2716, 2717, 2718, 2719, 2720, - 2721, 2722, 2723, 2724, 2725, 2726, 2727, 2728, - 2729, 2730, 2731, 2732, 2733, 2734, 2735, 2736, - 2737, 2738, 2739, 2740, 2741, 2742, 2743, 2744, - 2745, 2746, 2747, 2748, 2749, 2750, 2751, 2752, - 2753, 2754, 2755, 2756, 2757, 2758, 2759, 2760, - 2761, 2762, 2763, 2764, 2765, 2766, 2767, 2768, - 2769, 2770, 2771, 2772, 2773, 2774, 2775, 2776, - 2777, 2778, 2779, 2780, 2781, 2782, 2783, 2784, - 2785, 2786, 2787, 2788, 2789, 2790, 2791, 2792, - 2793, 2794, 2795, 2796, 2797, 2798, 2799, 2800, - 2801, 2802, 2803, 2804, 2805, 2806, 2807, 2808, - 2809, 2810, 2811, 2812, 2813, 2814, 2815, 2816, - 2817, 2818, 2819, 2820, 2821, 2822, 2823, 2824, - 2825, 2826, 2827, 2828, 2829, 2830, 2831, 2832, - 2833, 2834, 2835, 2836, 2837, 2838, 2839, 2840, - 2841, 2842, 2843, 2844, 2845, 2846, 2847, 2848, - 2849, 2850, 2851, 2852, 2853, 2854, 2855, 2856, - 2857, 2858, 2859, 2860, 2861, 2862, 2863, 2864, - 2865, 2866, 2867, 2868, 2869, 2870, 2871, 2872, - 2873, 2874, 2875, 2876, 2877, 2878, 2879, 2880, - 2881, 2882, 2883, 2884, 2885, 2886, 2887, 2888, - 2889, 2890, 2891, 2892, 2893, 2894, 2895, 2896, - 2897, 2898, 2899, 2900, 2901, 2902, 2903, 2904, - 2905, 2906, 2907, 2908, 2909, 2910, 2911, 2912, - 2913, 2914, 2915, 2916, 2917, 2918, 2919, 2920, - 2921, 2922, 2923, 2924, 2925, 2926, 2927, 2928, - 2929, 2930, 2931, 2932, 2933, 2934, 2935, 2936, - 2937, 2938, 2939, 2940, 2941, 2942, 2943, 2944, - 2945, 2946, 2947, 2948, 2949, 2950, 2951, 2952, - 2953, 2954, 2955, 2956, 2957, 2958, 2959, 2960, - 2961, 2962, 2963, 2964, 2965, 2966, 2967, 2968, - 2969, 2970, 2971, 2972, 2973, 2974, 2975, 2976, - 2977, 2978, 2979, 2980, 2981, 2982, 2983, 2984, - 2985, 2986, 2987, 2988, 2989, 2990, 2991, 2992, - 2993, 2994, 2995, 2996, 2997, 2998, 2999, 3000, - 3001, 3002, 3003, 3004, 3005, 3006, 3007, 3008, - 3009, 3010, 3011, 3012, 3013, 3014, 3015, 3016, - 3017, 3018, 3019, 3020, 3021, 3022, 3023, 3024, - 3025, 3026, 3027, 3028, 3029, 3030, 3031, 3032, - 3033, 3034, 3035, 3036, 3037, 3038, 3039, 3040, - 3041, 3042, 3043, 3044, 3045, 3046, 3047, 3048, - 3049, 3050, 3051, 3052, 3053, 3054, 3055, 3056, - 3057, 3058, 3059, 3060, 3061, 3062, 3063, 3064, - 3065, 3066, 3067, 3068, 3069, 3070, 3071, 3072, - 3073, 3074, 3075, 3076, 3077, 3078, 3079, 3080, - 3081, 3082, 3083, 3084, 3085, 3086, 3087, 3088, - 3089, 3090, 3091, 3092, 3093, 3094, 3095, 3096, - 3097, 3098, 3099, 3100, 3101, 3102, 3103, 3104, - 3105, 3106, 3107, 3108, 3109, 3110, 3111, 3112, - 3113, 3114, 3115, 3116, 3117, 3118, 3119, 3120, - 3121, 3122, 3123, 3124, 3125, 3126, 3127, 3128, - 3129, 3130, 3131, 3132, 3133, 3134, 3135, 3136, - 3137, 3138, 3139, 3140, 3141, 3142, 3143, 3144, - 3145, 3146, 3147, 3148, 3149, 3150, 3151, 3152, - 3153, 3154, 3155, 3156, 3157, 3158, 3159, 3160, - 3161, 3162, 3163, 3164, 3165, 3166, 3167, 3168, - 3169, 3170, 3171, 3172, 3173, 3174, 3175, 3176, - 3177, 3178, 3179, 3180, 3181, 3182, 3183, 3184, - 3185, 3186, 3187, 3188, 3189, 3190, 3191, 3192, - 3193, 3194, 3195, 3196, 3197, 3198, 3199, 3200, - 3201, 3202, 3203, 3204, 3205, 3206, 3207, 3208, - 3209, 3210, 3211, 3212, 3213, 3214, 3215, 3216, - 3217, 3218, 3219, 3220, 3221, 3222, 3223, 3224, - 3225, 3226, 3227, 3228, 3229, 3230, 3231, 3232, - 3233, 3234, 3235, 3236, 3237, 3238, 3239, 3240, - 3241, 3242, 3243, 3244, 3245, 3246, 3247, 3248, - 3249, 3250, 3251, 3252, 3253, 3254, 3255, 3256, - 3257, 3258, 3259, 3260, 3261, 3262, 3263, 3264, - 3265, 3266, 3267, 3268, 3269, 3270, 3271, 3272, - 3273, 3274, 3275, 3276, 3277, 3278, 3279, 3280, - 3281, 3282, 3283, 3284, 3285, 3286, 3287, 3288, - 3289, 3290, 3291, 3292, 3293, 3294, 3295, 3296, - 3297, 3298, 3299, 3300, 3301, 3302, 3303, 3304, - 3305, 3306, 3307, 3308, 3309, 3310, 3311, 3312, - 3313, 3314, 3315, 3316, 3317, 3318, 3319, 3320, - 3321, 3322, 3323, 3324, 3325, 3326, 3327, 3328, - 3329, 3330, 3331, 3332, 3333, 3334, 3335, 3336, - 3337, 3338, 3339, 3340, 3341, 3342, 3343, 3344, - 3345, 3346, 3347, 3348, 3349, 3350, 3351, 3352, - 3353, 3354, 3355, 3356, 3357, 3358, 3359, 3360, - 3361, 3362, 3363, 3364, 3365, 3366, 3367, 3368, - 3369, 3370, 3371, 3372, 3373, 3374, 3375, 3376, - 3377, 3378, 3379, 3380, 3381, 3382, 3383, 3384, - 3385, 3386, 3387, 3388, 3389, 3390, 3391, 3392, - 3393, 3394, 3395, 3396, 3397, 3398, 3399, 3400, - 3401, 3402, 3403, 3404, 3405, 3406, 3407, 3408, - 3409, 3410, 3411, 3412, 3413, 3414, 3415, 3416, - 3417, 3418, 3419, 3420, 3421, 3422, 3423, 3424, - 3425, 3426, 3427, 3428, 3429, 3430, 3431, 3432, - 3433, 3434, 3435, 3436, 3437, 3438, 3439, 3440, - 3441, 3442, 3443, 3444, 3445, 3446, 3447, 3448, - 3449, 3450, 3451, 3452, 3453, 3454, 3455, 3456, - 3457, 3458, 3459, 3460, 3461, 3462, 3463, 3464, - 3465, 3466, 3467, 3468, 3469, 3470, 3471, 3472, - 3473, 3474, 3475, 3476, 3477, 3478, 3479, 3480, - 3481, 3482, 3483, 3484, 3485, 3486, 3487, 3488, - 3489, 3490, 3491, 3492, 3493, 3494, 3495, 3496, - 3497, 3498, 3499, 3500, 3501, 3502, 3503, 3504, - 3505, 3506, 3507, 3508, 3509, 3510, 3511, 3512, - 3513, 3514, 0 + 2668, 2669, 2670, 2671, 2672, 2673, 2674, 2675, + 2676, 2677, 2678, 2679, 2680, 2681, 2682, 2683, + 2684, 2685, 2686, 2687, 2688, 2689, 2690, 2691, + 2692, 2693, 2694, 2695, 2696, 2697, 2698, 2699, + 2700, 2701, 2702, 2703, 2704, 2705, 2706, 2707, + 2708, 2709, 2710, 2711, 2712, 2713, 2714, 2715, + 2716, 2717, 2718, 2719, 2720, 2721, 2722, 2723, + 2724, 2725, 2726, 2727, 2728, 2729, 2730, 2731, + 2732, 2733, 2734, 2735, 2736, 2737, 2738, 2739, + 2740, 2741, 2742, 2743, 2744, 2745, 2746, 2747, + 2748, 2749, 2750, 2751, 2752, 2753, 2754, 2755, + 2756, 2757, 2758, 2759, 2760, 2761, 2762, 2763, + 2764, 2765, 2766, 2767, 2768, 2769, 2770, 2771, + 2772, 2773, 2774, 2775, 2776, 2777, 2778, 2779, + 2780, 2781, 2782, 2783, 2784, 2785, 2786, 2787, + 2788, 2789, 2790, 2791, 2792, 2793, 2794, 2795, + 2796, 2797, 2798, 2799, 2800, 2801, 2802, 2803, + 2804, 2805, 2806, 2807, 2808, 2809, 2810, 2811, + 2812, 2813, 2814, 2815, 2816, 2817, 2818, 2819, + 2820, 2821, 2822, 2823, 2824, 2825, 2826, 2827, + 2828, 2829, 2830, 2831, 2832, 2833, 2834, 2835, + 2836, 2837, 2838, 2839, 2840, 2841, 2842, 2843, + 2844, 2845, 2846, 2847, 2848, 2849, 2850, 2851, + 2852, 2853, 2854, 2855, 2856, 2857, 2858, 2859, + 2860, 2861, 2862, 2863, 2864, 2865, 2866, 2867, + 2868, 2869, 2870, 2871, 2872, 2873, 2874, 2875, + 2876, 2877, 2878, 2879, 2880, 2881, 2882, 2883, + 2884, 2885, 2886, 2887, 2888, 2889, 2890, 2891, + 2892, 2893, 2894, 2895, 2896, 2897, 2898, 2899, + 2900, 2901, 2902, 2903, 2904, 2905, 2906, 2907, + 2908, 2909, 2910, 2911, 2912, 2913, 2914, 2915, + 2916, 2917, 2918, 2919, 2920, 2921, 2922, 2923, + 2924, 2925, 2926, 2927, 2928, 2929, 2930, 2931, + 2932, 2933, 2934, 2935, 2936, 2937, 2938, 2939, + 2940, 2941, 2942, 2943, 2944, 2945, 2946, 2947, + 2948, 2949, 2950, 2951, 2952, 2953, 2954, 2955, + 2956, 2957, 2958, 2959, 2960, 2961, 2962, 2963, + 2964, 2965, 2966, 2967, 2968, 2969, 2970, 2971, + 2972, 2973, 2974, 2975, 2976, 2977, 2978, 2979, + 2980, 2981, 2982, 2983, 2984, 2985, 2986, 2987, + 2988, 2989, 2990, 2991, 2992, 2993, 2994, 2995, + 2996, 2997, 2998, 2999, 3000, 3001, 3002, 3003, + 3004, 3005, 3006, 3007, 3008, 3009, 3010, 3011, + 3012, 3013, 3014, 3015, 3016, 3017, 3018, 3019, + 3020, 3021, 3022, 3023, 3024, 3025, 3026, 3027, + 3028, 3029, 3030, 3031, 3032, 3033, 3034, 3035, + 3036, 3037, 3038, 3039, 3040, 3041, 3042, 3043, + 3044, 3045, 3046, 3047, 3048, 3049, 3050, 3051, + 3052, 3053, 3054, 3055, 3056, 3057, 3058, 3059, + 3060, 3061, 3062, 3063, 3064, 3065, 3066, 3067, + 3068, 3069, 3070, 3071, 3072, 3073, 3074, 3075, + 3076, 3077, 3078, 3079, 3080, 3081, 3082, 3083, + 3084, 3085, 3086, 3087, 3088, 3089, 3090, 3091, + 3092, 3093, 3094, 3095, 3096, 3097, 3098, 3099, + 3100, 3101, 3102, 3103, 3104, 3105, 3106, 3107, + 3108, 3109, 3110, 3111, 3112, 3113, 3114, 3115, + 3116, 3117, 3118, 3119, 3120, 3121, 3122, 3123, + 3124, 3125, 3126, 3127, 3128, 3129, 3130, 3131, + 3132, 3133, 3134, 3135, 3136, 3137, 3138, 3139, + 3140, 3141, 3142, 3143, 3144, 3145, 3146, 3147, + 3148, 3149, 3150, 3151, 3152, 3153, 3154, 3155, + 3156, 3157, 3158, 3159, 3160, 3161, 3162, 3163, + 3164, 3165, 3166, 3167, 3168, 3169, 3170, 3171, + 3172, 3173, 3174, 3175, 3176, 3177, 3178, 3179, + 3180, 3181, 3182, 3183, 3184, 3185, 3186, 3187, + 3188, 3189, 3190, 3191, 3192, 3193, 3194, 3195, + 3196, 3197, 3198, 3199, 3200, 3201, 3202, 3203, + 3204, 3205, 3206, 3207, 3208, 3209, 3210, 3211, + 3212, 3213, 3214, 3215, 3216, 3217, 3218, 3219, + 3220, 3221, 3222, 3223, 3224, 3225, 3226, 3227, + 3228, 3229, 3230, 3231, 3232, 3233, 3234, 3235, + 3236, 3237, 3238, 3239, 3240, 3241, 3242, 3243, + 3244, 3245, 3246, 3247, 3248, 3249, 3250, 3251, + 3252, 3253, 3254, 3255, 3256, 3257, 3258, 3259, + 3260, 3261, 3262, 3263, 3264, 3265, 3266, 3267, + 3268, 3269, 3270, 3271, 3272, 3273, 3274, 3275, + 3276, 3277, 3278, 3279, 3280, 3281, 3282, 3283, + 3284, 3285, 3286, 3287, 3288, 3289, 3290, 3291, + 3292, 3293, 3294, 3295, 3296, 3297, 3298, 3299, + 3300, 3301, 3302, 3303, 3304, 3305, 3306, 3307, + 3308, 3309, 3310, 3311, 3312, 3313, 3314, 3315, + 3316, 3317, 3318, 3319, 3320, 3321, 3322, 3323, + 3324, 3325, 3326, 3327, 3328, 3329, 3330, 3331, + 3332, 3333, 3334, 3335, 3336, 3337, 3338, 3339, + 3340, 3341, 3342, 3343, 3344, 3345, 3346, 3347, + 3348, 3349, 3350, 3351, 3352, 3353, 3354, 3355, + 3356, 3357, 3358, 3359, 3360, 3361, 3362, 3363, + 3364, 3365, 3366, 3367, 3368, 3369, 3370, 3371, + 3372, 3373, 3374, 3375, 3376, 3377, 3378, 3379, + 3380, 3381, 3382, 3383, 3384, 3385, 3386, 3387, + 3388, 3389, 3390, 3391, 3392, 3393, 3394, 3395, + 3396, 3397, 3398, 3399, 3400, 3401, 3402, 3403, + 3404, 3405, 3406, 3407, 3408, 3409, 3410, 3411, + 3412, 3413, 3414, 3415, 3416, 3417, 3418, 3419, + 3420, 3421, 3422, 3423, 3424, 3425, 3426, 3427, + 3428, 3429, 3430, 3431, 3432, 3433, 3434, 3435, + 3436, 3437, 3438, 3439, 3440, 3441, 3442, 3443, + 3444, 3445, 3446, 3447, 3448, 3449, 3450, 3451, + 3452, 3453, 3454, 3455, 3456, 3457, 3458, 3459, + 3460, 3461, 3462, 3463, 3464, 3465, 3466, 3467, + 3468, 3469, 3470, 3471, 3472, 3473, 3474, 3475, + 3476, 3477, 3478, 3479, 3480, 3481, 3482, 3483, + 3484, 3485, 3486, 3487, 3488, 3489, 3490, 3491, + 3492, 3493, 3494, 3495, 3496, 3497, 3498, 3499, + 3500, 3501, 3502, 3503, 3504, 3505, 3506, 3507, + 3508, 3509, 3510, 3511, 3512, 3513, 3514, 3515, + 3516, 3517, 3518, 0 }; static const int regex_start = 746; @@ -4108,15 +4109,15 @@ static const short _regex_actions[] = { static const int regex_en_readBracedUCP = 559; static const int regex_en_readUCPSingle = 818; static const int regex_en_charClassGuts = 819; - static const int regex_en_readClass = 836; - static const int regex_en_readQuotedLiteral = 838; - static const int regex_en_readQuotedClass = 843; - static const int regex_en_readComment = 848; - static const int regex_en_readNewlineTerminatedComment = 849; + static const int regex_en_readClass = 837; + static const int regex_en_readQuotedLiteral = 839; + static const int regex_en_readQuotedClass = 844; + static const int regex_en_readComment = 849; + static const int regex_en_readNewlineTerminatedComment = 850; static const int regex_en_main = 746; -#line 1911 "Parser.rl" +#line 1918 "Parser.rl" /** \brief Main parser call, returns root Component or nullptr. */ @@ -4189,7 +4190,7 @@ static const short _regex_actions[] = { try { // Embed the Ragel machine here -#line 4187 "Parser.cpp" +#line 4188 "Parser.cpp" { cs = (int)regex_start; top = 0; @@ -4198,17 +4199,17 @@ static const short _regex_actions[] = { act = 0; } -#line 1982 "Parser.rl" +#line 1989 "Parser.rl" -#line 4196 "Parser.cpp" +#line 4197 "Parser.cpp" { int _cpc; int _klen; const signed char * _ckeys; unsigned int _trans = 0; unsigned int _cond = 0; - const char * _keys; + const unsigned char * _keys; const short * _acts; unsigned int _nacts; _resume: {} @@ -4224,7 +4225,7 @@ static const short _regex_actions[] = { #line 1 "NONE" {ts = p;}} -#line 4218 "Parser.cpp" +#line 4219 "Parser.cpp" break; } @@ -4244,9 +4245,9 @@ static const short _regex_actions[] = { _klen = (int)_regex_single_lengths[cs]; if ( _klen > 0 ) { - const char *_lower = _keys; - const char *_upper = _keys + _klen - 1; - const char *_mid; + const unsigned char *_lower = _keys; + const unsigned char *_upper = _keys + _klen - 1; + const unsigned char *_mid; while ( 1 ) { if ( _upper < _lower ) { _keys += _klen; @@ -4268,9 +4269,9 @@ static const short _regex_actions[] = { _klen = (int)_regex_range_lengths[cs]; if ( _klen > 0 ) { - const char *_lower = _keys; - const char *_upper = _keys + (_klen<<1) - 2; - const char *_mid; + const unsigned char *_lower = _keys; + const unsigned char *_upper = _keys + (_klen<<1) - 2; + const unsigned char *_mid; while ( 1 ) { if ( _upper < _lower ) { _trans += (unsigned int)_klen; @@ -4300,19 +4301,19 @@ static const short _regex_actions[] = { case 0: { if ( ( mode.utf8 ) -#line 4293 "Parser.cpp" +#line 4294 "Parser.cpp" ) _cpc += 1; break; } case 1: { if ( ( mode.ignore_space ) -#line 4298 "Parser.cpp" +#line 4299 "Parser.cpp" ) _cpc += 1; break; } case 2: { if ( ( inCharClassEarly ) -#line 4303 "Parser.cpp" +#line 4304 "Parser.cpp" ) _cpc += 1; break; } @@ -4323,7 +4324,7 @@ static const short _regex_actions[] = { const signed char *_mid; while ( 1 ) { if ( _upper < _lower ) { - _cond = 3561; + _cond = 3565; break; } @@ -4352,159 +4353,159 @@ static const short _regex_actions[] = { { case 0: { { -#line 285 "Parser.rl" +#line 286 "Parser.rl" label.clear();} -#line 4345 "Parser.cpp" +#line 4346 "Parser.cpp" break; } case 1: { { -#line 286 "Parser.rl" +#line 287 "Parser.rl" label.push_back((( (*( p)))));} -#line 4353 "Parser.cpp" +#line 4354 "Parser.cpp" break; } case 2: { { -#line 287 "Parser.rl" +#line 288 "Parser.rl" octAccumulator = 0;} -#line 4361 "Parser.cpp" +#line 4362 "Parser.cpp" break; } case 3: { { -#line 288 "Parser.rl" +#line 289 "Parser.rl" accumulator = 0;} -#line 4369 "Parser.cpp" +#line 4370 "Parser.cpp" break; } case 4: { { -#line 289 "Parser.rl" +#line 290 "Parser.rl" octAccumulator = 0; pushOct(&octAccumulator, (( (*( p))))); } -#line 4380 "Parser.cpp" +#line 4381 "Parser.cpp" break; } case 5: { { -#line 293 "Parser.rl" +#line 294 "Parser.rl" accumulator = 0; pushDec(&accumulator, (( (*( p))))); } -#line 4391 "Parser.cpp" +#line 4392 "Parser.cpp" break; } case 6: { { -#line 297 "Parser.rl" +#line 298 "Parser.rl" repeatN = 0; repeatM = 0; } -#line 4399 "Parser.cpp" +#line 4400 "Parser.cpp" break; } case 7: { { -#line 298 "Parser.rl" +#line 299 "Parser.rl" pushDec(&repeatN, (( (*( p))))); } -#line 4407 "Parser.cpp" +#line 4408 "Parser.cpp" break; } case 8: { { -#line 299 "Parser.rl" +#line 300 "Parser.rl" pushDec(&repeatM, (( (*( p))))); } -#line 4415 "Parser.cpp" +#line 4416 "Parser.cpp" break; } case 9: { { -#line 300 "Parser.rl" +#line 301 "Parser.rl" pushOct(&octAccumulator, (( (*( p))))); } -#line 4423 "Parser.cpp" +#line 4424 "Parser.cpp" break; } case 10: { { -#line 301 "Parser.rl" +#line 302 "Parser.rl" pushDec(&accumulator, (( (*( p))))); } -#line 4431 "Parser.cpp" +#line 4432 "Parser.cpp" break; } case 11: { { -#line 302 "Parser.rl" +#line 303 "Parser.rl" accumulator *= 16; accumulator += (( (*( p)))) - '0'; } -#line 4442 "Parser.cpp" +#line 4443 "Parser.cpp" break; } case 12: { { -#line 306 "Parser.rl" +#line 307 "Parser.rl" accumulator *= 16; accumulator += 10 + (( (*( p)))) - 'a'; } -#line 4453 "Parser.cpp" +#line 4454 "Parser.cpp" break; } case 13: { { -#line 310 "Parser.rl" +#line 311 "Parser.rl" accumulator *= 16; accumulator += 10 + (( (*( p)))) - 'A'; } -#line 4464 "Parser.cpp" +#line 4465 "Parser.cpp" break; } case 14: { { -#line 430 "Parser.rl" +#line 431 "Parser.rl" newMode = mode; } -#line 4474 "Parser.cpp" +#line 4475 "Parser.cpp" break; } case 15: { { -#line 437 "Parser.rl" +#line 438 "Parser.rl" switch ((( (*( p))))) { case 'i': @@ -4525,13 +4526,13 @@ static const short _regex_actions[] = { } } -#line 4500 "Parser.cpp" +#line 4501 "Parser.cpp" break; } case 16: { { -#line 456 "Parser.rl" +#line 457 "Parser.rl" switch ((( (*( p))))) { case 'i': @@ -4552,42 +4553,42 @@ static const short _regex_actions[] = { } } -#line 4526 "Parser.cpp" +#line 4527 "Parser.cpp" break; } case 17: { { -#line 510 "Parser.rl" +#line 511 "Parser.rl" repeatM = repeatN;} -#line 4534 "Parser.cpp" +#line 4535 "Parser.cpp" break; } case 18: { { -#line 510 "Parser.rl" +#line 511 "Parser.rl" repeatM = ComponentRepeat::NoLimit;} -#line 4542 "Parser.cpp" +#line 4543 "Parser.cpp" break; } case 19: { { -#line 722 "Parser.rl" +#line 723 "Parser.rl" negated = !negated; } -#line 4550 "Parser.cpp" +#line 4551 "Parser.cpp" break; } case 20: { { -#line 723 "Parser.rl" +#line 724 "Parser.rl" {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -4596,30 +4597,30 @@ static const short _regex_actions[] = { } stack[top] = cs; top += 1;cs = 790;goto _again;}} -#line 4566 "Parser.cpp" +#line 4567 "Parser.cpp" break; } case 21: { { -#line 724 "Parser.rl" +#line 725 "Parser.rl" if (!inCharClass) { // not inside [..] currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } -#line 4579 "Parser.cpp" +#line 4580 "Parser.cpp" break; } case 22: { { -#line 730 "Parser.rl" +#line 731 "Parser.rl" throw LocatedParseError("Malformed property"); } -#line 4587 "Parser.cpp" +#line 4588 "Parser.cpp" break; } @@ -4628,60 +4629,60 @@ static const short _regex_actions[] = { #line 1 "NONE" {te = p+1;}} -#line 4595 "Parser.cpp" +#line 4596 "Parser.cpp" break; } case 26: { { -#line 550 "Parser.rl" +#line 551 "Parser.rl" {te = p+1;{ -#line 550 "Parser.rl" +#line 551 "Parser.rl" throw LocatedParseError("(*UTF8) must be at start of " "expression, encountered"); } }} -#line 4609 "Parser.cpp" +#line 4610 "Parser.cpp" break; } case 27: { { -#line 554 "Parser.rl" +#line 555 "Parser.rl" {te = p+1;{ -#line 554 "Parser.rl" +#line 555 "Parser.rl" throw LocatedParseError("(*UTF) must be at start of " "expression, encountered"); } }} -#line 4623 "Parser.cpp" +#line 4624 "Parser.cpp" break; } case 28: { { -#line 558 "Parser.rl" +#line 559 "Parser.rl" {te = p+1;{ -#line 558 "Parser.rl" +#line 559 "Parser.rl" throw LocatedParseError("(*UCP) must be at start of " "expression, encountered"); } }} -#line 4637 "Parser.cpp" +#line 4638 "Parser.cpp" break; } case 29: { { -#line 564 "Parser.rl" +#line 565 "Parser.rl" {te = p+1;{ -#line 564 "Parser.rl" +#line 565 "Parser.rl" ParseMode temp_mode; assert(ts - 2 >= ptr); // parser needs the '(*' at the start too. @@ -4691,2626 +4692,2626 @@ static const short _regex_actions[] = { } }} -#line 4654 "Parser.cpp" +#line 4655 "Parser.cpp" break; } case 30: { { -#line 571 "Parser.rl" +#line 572 "Parser.rl" {te = p+1;{ -#line 571 "Parser.rl" +#line 572 "Parser.rl" throw LocatedParseError("Unknown control verb"); } }} -#line 4667 "Parser.cpp" +#line 4668 "Parser.cpp" break; } case 31: { { -#line 571 "Parser.rl" +#line 572 "Parser.rl" {te = p;p = p - 1;{ -#line 571 "Parser.rl" +#line 572 "Parser.rl" throw LocatedParseError("Unknown control verb"); } }} -#line 4680 "Parser.cpp" +#line 4681 "Parser.cpp" break; } case 32: { { -#line 571 "Parser.rl" +#line 572 "Parser.rl" {p = ((te))-1; { -#line 571 "Parser.rl" +#line 572 "Parser.rl" throw LocatedParseError("Unknown control verb"); } }} -#line 4694 "Parser.cpp" +#line 4695 "Parser.cpp" break; } case 33: { { -#line 581 "Parser.rl" +#line 582 "Parser.rl" {te = p+1;{ -#line 581 "Parser.rl" +#line 582 "Parser.rl" currentCls->add(CLASS_UCP_CC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4705 "Parser.cpp" +#line 4706 "Parser.cpp" break; } case 34: { { -#line 582 "Parser.rl" +#line 583 "Parser.rl" {te = p+1;{ -#line 582 "Parser.rl" +#line 583 "Parser.rl" currentCls->add(CLASS_UCP_CF, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4716 "Parser.cpp" +#line 4717 "Parser.cpp" break; } case 35: { { -#line 583 "Parser.rl" +#line 584 "Parser.rl" {te = p+1;{ -#line 583 "Parser.rl" +#line 584 "Parser.rl" currentCls->add(CLASS_UCP_CN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4727 "Parser.cpp" +#line 4728 "Parser.cpp" break; } case 36: { { -#line 585 "Parser.rl" +#line 586 "Parser.rl" {te = p+1;{ -#line 585 "Parser.rl" +#line 586 "Parser.rl" currentCls->add(CLASS_UCP_CS, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4738 "Parser.cpp" +#line 4739 "Parser.cpp" break; } case 37: { { -#line 587 "Parser.rl" +#line 588 "Parser.rl" {te = p+1;{ -#line 587 "Parser.rl" +#line 588 "Parser.rl" currentCls->add(CLASS_UCP_LL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4749 "Parser.cpp" +#line 4750 "Parser.cpp" break; } case 38: { { -#line 588 "Parser.rl" +#line 589 "Parser.rl" {te = p+1;{ -#line 588 "Parser.rl" +#line 589 "Parser.rl" currentCls->add(CLASS_UCP_LM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4760 "Parser.cpp" +#line 4761 "Parser.cpp" break; } case 39: { { -#line 589 "Parser.rl" +#line 590 "Parser.rl" {te = p+1;{ -#line 589 "Parser.rl" +#line 590 "Parser.rl" currentCls->add(CLASS_UCP_LO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4771 "Parser.cpp" +#line 4772 "Parser.cpp" break; } case 40: { { -#line 590 "Parser.rl" +#line 591 "Parser.rl" {te = p+1;{ -#line 590 "Parser.rl" +#line 591 "Parser.rl" currentCls->add(CLASS_UCP_LT, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4782 "Parser.cpp" +#line 4783 "Parser.cpp" break; } case 41: { { -#line 591 "Parser.rl" +#line 592 "Parser.rl" {te = p+1;{ -#line 591 "Parser.rl" +#line 592 "Parser.rl" currentCls->add(CLASS_UCP_LU, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4793 "Parser.cpp" +#line 4794 "Parser.cpp" break; } case 42: { { -#line 592 "Parser.rl" +#line 593 "Parser.rl" {te = p+1;{ -#line 592 "Parser.rl" +#line 593 "Parser.rl" currentCls->add(CLASS_UCP_L_AND, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4804 "Parser.cpp" +#line 4805 "Parser.cpp" break; } case 43: { { -#line 594 "Parser.rl" +#line 595 "Parser.rl" {te = p+1;{ -#line 594 "Parser.rl" +#line 595 "Parser.rl" currentCls->add(CLASS_UCP_MC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4815 "Parser.cpp" +#line 4816 "Parser.cpp" break; } case 44: { { -#line 596 "Parser.rl" +#line 597 "Parser.rl" {te = p+1;{ -#line 596 "Parser.rl" +#line 597 "Parser.rl" currentCls->add(CLASS_UCP_MN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4826 "Parser.cpp" +#line 4827 "Parser.cpp" break; } case 45: { { -#line 598 "Parser.rl" +#line 599 "Parser.rl" {te = p+1;{ -#line 598 "Parser.rl" +#line 599 "Parser.rl" currentCls->add(CLASS_UCP_ND, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4837 "Parser.cpp" +#line 4838 "Parser.cpp" break; } case 46: { { -#line 599 "Parser.rl" +#line 600 "Parser.rl" {te = p+1;{ -#line 599 "Parser.rl" +#line 600 "Parser.rl" currentCls->add(CLASS_UCP_NL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4848 "Parser.cpp" +#line 4849 "Parser.cpp" break; } case 47: { { -#line 600 "Parser.rl" +#line 601 "Parser.rl" {te = p+1;{ -#line 600 "Parser.rl" +#line 601 "Parser.rl" currentCls->add(CLASS_UCP_NO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4859 "Parser.cpp" +#line 4860 "Parser.cpp" break; } case 48: { { -#line 602 "Parser.rl" +#line 603 "Parser.rl" {te = p+1;{ -#line 602 "Parser.rl" +#line 603 "Parser.rl" currentCls->add(CLASS_UCP_PC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4870 "Parser.cpp" +#line 4871 "Parser.cpp" break; } case 49: { { -#line 603 "Parser.rl" +#line 604 "Parser.rl" {te = p+1;{ -#line 603 "Parser.rl" +#line 604 "Parser.rl" currentCls->add(CLASS_UCP_PD, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4881 "Parser.cpp" +#line 4882 "Parser.cpp" break; } case 50: { { -#line 604 "Parser.rl" +#line 605 "Parser.rl" {te = p+1;{ -#line 604 "Parser.rl" +#line 605 "Parser.rl" currentCls->add(CLASS_UCP_PE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4892 "Parser.cpp" +#line 4893 "Parser.cpp" break; } case 51: { { -#line 605 "Parser.rl" +#line 606 "Parser.rl" {te = p+1;{ -#line 605 "Parser.rl" +#line 606 "Parser.rl" currentCls->add(CLASS_UCP_PF, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4903 "Parser.cpp" +#line 4904 "Parser.cpp" break; } case 52: { { -#line 606 "Parser.rl" +#line 607 "Parser.rl" {te = p+1;{ -#line 606 "Parser.rl" +#line 607 "Parser.rl" currentCls->add(CLASS_UCP_PI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4914 "Parser.cpp" +#line 4915 "Parser.cpp" break; } case 53: { { -#line 607 "Parser.rl" +#line 608 "Parser.rl" {te = p+1;{ -#line 607 "Parser.rl" +#line 608 "Parser.rl" currentCls->add(CLASS_UCP_PO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4925 "Parser.cpp" +#line 4926 "Parser.cpp" break; } case 54: { { -#line 608 "Parser.rl" +#line 609 "Parser.rl" {te = p+1;{ -#line 608 "Parser.rl" +#line 609 "Parser.rl" currentCls->add(CLASS_UCP_PS, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4936 "Parser.cpp" +#line 4937 "Parser.cpp" break; } case 55: { { -#line 610 "Parser.rl" +#line 611 "Parser.rl" {te = p+1;{ -#line 610 "Parser.rl" +#line 611 "Parser.rl" currentCls->add(CLASS_UCP_SC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4947 "Parser.cpp" +#line 4948 "Parser.cpp" break; } case 56: { { -#line 611 "Parser.rl" +#line 612 "Parser.rl" {te = p+1;{ -#line 611 "Parser.rl" +#line 612 "Parser.rl" currentCls->add(CLASS_UCP_SK, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4958 "Parser.cpp" +#line 4959 "Parser.cpp" break; } case 57: { { -#line 612 "Parser.rl" +#line 613 "Parser.rl" {te = p+1;{ -#line 612 "Parser.rl" +#line 613 "Parser.rl" currentCls->add(CLASS_UCP_SM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4969 "Parser.cpp" +#line 4970 "Parser.cpp" break; } case 58: { { -#line 613 "Parser.rl" +#line 614 "Parser.rl" {te = p+1;{ -#line 613 "Parser.rl" +#line 614 "Parser.rl" currentCls->add(CLASS_UCP_SO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4980 "Parser.cpp" +#line 4981 "Parser.cpp" break; } case 59: { { -#line 615 "Parser.rl" +#line 616 "Parser.rl" {te = p+1;{ -#line 615 "Parser.rl" +#line 616 "Parser.rl" currentCls->add(CLASS_UCP_ZL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 4991 "Parser.cpp" +#line 4992 "Parser.cpp" break; } case 60: { { -#line 616 "Parser.rl" +#line 617 "Parser.rl" {te = p+1;{ -#line 616 "Parser.rl" +#line 617 "Parser.rl" currentCls->add(CLASS_UCP_ZP, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5002 "Parser.cpp" +#line 5003 "Parser.cpp" break; } case 61: { { -#line 617 "Parser.rl" +#line 618 "Parser.rl" {te = p+1;{ -#line 617 "Parser.rl" +#line 618 "Parser.rl" currentCls->add(CLASS_UCP_ZS, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5013 "Parser.cpp" +#line 5014 "Parser.cpp" break; } case 62: { { -#line 618 "Parser.rl" +#line 619 "Parser.rl" {te = p+1;{ -#line 618 "Parser.rl" +#line 619 "Parser.rl" currentCls->add(CLASS_UCP_XAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5024 "Parser.cpp" +#line 5025 "Parser.cpp" break; } case 63: { { -#line 619 "Parser.rl" +#line 620 "Parser.rl" {te = p+1;{ -#line 619 "Parser.rl" +#line 620 "Parser.rl" currentCls->add(CLASS_UCP_XPS, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5035 "Parser.cpp" +#line 5036 "Parser.cpp" break; } case 64: { { -#line 620 "Parser.rl" +#line 621 "Parser.rl" {te = p+1;{ -#line 620 "Parser.rl" +#line 621 "Parser.rl" currentCls->add(CLASS_UCP_XSP, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5046 "Parser.cpp" +#line 5047 "Parser.cpp" break; } case 65: { { -#line 621 "Parser.rl" +#line 622 "Parser.rl" {te = p+1;{ -#line 621 "Parser.rl" +#line 622 "Parser.rl" currentCls->add(CLASS_UCP_XWD, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5057 "Parser.cpp" +#line 5058 "Parser.cpp" break; } case 66: { { -#line 622 "Parser.rl" +#line 623 "Parser.rl" {te = p+1;{ -#line 622 "Parser.rl" +#line 623 "Parser.rl" currentCls->add(CLASS_SCRIPT_ARABIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5068 "Parser.cpp" +#line 5069 "Parser.cpp" break; } case 67: { { -#line 623 "Parser.rl" +#line 624 "Parser.rl" {te = p+1;{ -#line 623 "Parser.rl" +#line 624 "Parser.rl" currentCls->add(CLASS_SCRIPT_ARMENIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5079 "Parser.cpp" +#line 5080 "Parser.cpp" break; } case 68: { { -#line 624 "Parser.rl" +#line 625 "Parser.rl" {te = p+1;{ -#line 624 "Parser.rl" +#line 625 "Parser.rl" currentCls->add(CLASS_SCRIPT_AVESTAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5090 "Parser.cpp" +#line 5091 "Parser.cpp" break; } case 69: { { -#line 625 "Parser.rl" +#line 626 "Parser.rl" {te = p+1;{ -#line 625 "Parser.rl" +#line 626 "Parser.rl" currentCls->add(CLASS_SCRIPT_BALINESE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5101 "Parser.cpp" +#line 5102 "Parser.cpp" break; } case 70: { { -#line 626 "Parser.rl" +#line 627 "Parser.rl" {te = p+1;{ -#line 626 "Parser.rl" +#line 627 "Parser.rl" currentCls->add(CLASS_SCRIPT_BAMUM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5112 "Parser.cpp" +#line 5113 "Parser.cpp" break; } case 71: { { -#line 627 "Parser.rl" +#line 628 "Parser.rl" {te = p+1;{ -#line 627 "Parser.rl" +#line 628 "Parser.rl" currentCls->add(CLASS_SCRIPT_BATAK, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5123 "Parser.cpp" +#line 5124 "Parser.cpp" break; } case 72: { { -#line 628 "Parser.rl" +#line 629 "Parser.rl" {te = p+1;{ -#line 628 "Parser.rl" +#line 629 "Parser.rl" currentCls->add(CLASS_SCRIPT_BENGALI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5134 "Parser.cpp" +#line 5135 "Parser.cpp" break; } case 73: { { -#line 629 "Parser.rl" +#line 630 "Parser.rl" {te = p+1;{ -#line 629 "Parser.rl" +#line 630 "Parser.rl" currentCls->add(CLASS_SCRIPT_BOPOMOFO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5145 "Parser.cpp" +#line 5146 "Parser.cpp" break; } case 74: { { -#line 630 "Parser.rl" +#line 631 "Parser.rl" {te = p+1;{ -#line 630 "Parser.rl" +#line 631 "Parser.rl" currentCls->add(CLASS_SCRIPT_BRAHMI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5156 "Parser.cpp" +#line 5157 "Parser.cpp" break; } case 75: { { -#line 631 "Parser.rl" +#line 632 "Parser.rl" {te = p+1;{ -#line 631 "Parser.rl" +#line 632 "Parser.rl" currentCls->add(CLASS_SCRIPT_BRAILLE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5167 "Parser.cpp" +#line 5168 "Parser.cpp" break; } case 76: { { -#line 632 "Parser.rl" +#line 633 "Parser.rl" {te = p+1;{ -#line 632 "Parser.rl" +#line 633 "Parser.rl" currentCls->add(CLASS_SCRIPT_BUGINESE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5178 "Parser.cpp" +#line 5179 "Parser.cpp" break; } case 77: { { -#line 633 "Parser.rl" +#line 634 "Parser.rl" {te = p+1;{ -#line 633 "Parser.rl" +#line 634 "Parser.rl" currentCls->add(CLASS_SCRIPT_BUHID, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5189 "Parser.cpp" +#line 5190 "Parser.cpp" break; } case 78: { { -#line 634 "Parser.rl" +#line 635 "Parser.rl" {te = p+1;{ -#line 634 "Parser.rl" +#line 635 "Parser.rl" currentCls->add(CLASS_SCRIPT_CANADIAN_ABORIGINAL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5200 "Parser.cpp" +#line 5201 "Parser.cpp" break; } case 79: { { -#line 635 "Parser.rl" +#line 636 "Parser.rl" {te = p+1;{ -#line 635 "Parser.rl" +#line 636 "Parser.rl" currentCls->add(CLASS_SCRIPT_CARIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5211 "Parser.cpp" +#line 5212 "Parser.cpp" break; } case 80: { { -#line 636 "Parser.rl" +#line 637 "Parser.rl" {te = p+1;{ -#line 636 "Parser.rl" +#line 637 "Parser.rl" currentCls->add(CLASS_SCRIPT_CHAM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5222 "Parser.cpp" +#line 5223 "Parser.cpp" break; } case 81: { { -#line 637 "Parser.rl" +#line 638 "Parser.rl" {te = p+1;{ -#line 637 "Parser.rl" +#line 638 "Parser.rl" currentCls->add(CLASS_SCRIPT_CHEROKEE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5233 "Parser.cpp" +#line 5234 "Parser.cpp" break; } case 82: { { -#line 638 "Parser.rl" +#line 639 "Parser.rl" {te = p+1;{ -#line 638 "Parser.rl" +#line 639 "Parser.rl" currentCls->add(CLASS_SCRIPT_COMMON, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5244 "Parser.cpp" +#line 5245 "Parser.cpp" break; } case 83: { { -#line 639 "Parser.rl" +#line 640 "Parser.rl" {te = p+1;{ -#line 639 "Parser.rl" +#line 640 "Parser.rl" currentCls->add(CLASS_SCRIPT_COPTIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5255 "Parser.cpp" +#line 5256 "Parser.cpp" break; } case 84: { { -#line 640 "Parser.rl" +#line 641 "Parser.rl" {te = p+1;{ -#line 640 "Parser.rl" +#line 641 "Parser.rl" currentCls->add(CLASS_SCRIPT_CUNEIFORM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5266 "Parser.cpp" +#line 5267 "Parser.cpp" break; } case 85: { { -#line 641 "Parser.rl" +#line 642 "Parser.rl" {te = p+1;{ -#line 641 "Parser.rl" +#line 642 "Parser.rl" currentCls->add(CLASS_SCRIPT_CYPRIOT, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5277 "Parser.cpp" +#line 5278 "Parser.cpp" break; } case 86: { { -#line 642 "Parser.rl" +#line 643 "Parser.rl" {te = p+1;{ -#line 642 "Parser.rl" +#line 643 "Parser.rl" currentCls->add(CLASS_SCRIPT_CYRILLIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5288 "Parser.cpp" +#line 5289 "Parser.cpp" break; } case 87: { { -#line 643 "Parser.rl" +#line 644 "Parser.rl" {te = p+1;{ -#line 643 "Parser.rl" +#line 644 "Parser.rl" currentCls->add(CLASS_SCRIPT_DESERET, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5299 "Parser.cpp" +#line 5300 "Parser.cpp" break; } case 88: { { -#line 644 "Parser.rl" +#line 645 "Parser.rl" {te = p+1;{ -#line 644 "Parser.rl" +#line 645 "Parser.rl" currentCls->add(CLASS_SCRIPT_DEVANAGARI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5310 "Parser.cpp" +#line 5311 "Parser.cpp" break; } case 89: { { -#line 645 "Parser.rl" +#line 646 "Parser.rl" {te = p+1;{ -#line 645 "Parser.rl" +#line 646 "Parser.rl" currentCls->add(CLASS_SCRIPT_EGYPTIAN_HIEROGLYPHS, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5321 "Parser.cpp" +#line 5322 "Parser.cpp" break; } case 90: { { -#line 646 "Parser.rl" +#line 647 "Parser.rl" {te = p+1;{ -#line 646 "Parser.rl" +#line 647 "Parser.rl" currentCls->add(CLASS_SCRIPT_ETHIOPIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5332 "Parser.cpp" +#line 5333 "Parser.cpp" break; } case 91: { { -#line 647 "Parser.rl" +#line 648 "Parser.rl" {te = p+1;{ -#line 647 "Parser.rl" +#line 648 "Parser.rl" currentCls->add(CLASS_SCRIPT_GEORGIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5343 "Parser.cpp" +#line 5344 "Parser.cpp" break; } case 92: { { -#line 648 "Parser.rl" +#line 649 "Parser.rl" {te = p+1;{ -#line 648 "Parser.rl" +#line 649 "Parser.rl" currentCls->add(CLASS_SCRIPT_GLAGOLITIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5354 "Parser.cpp" +#line 5355 "Parser.cpp" break; } case 93: { { -#line 649 "Parser.rl" +#line 650 "Parser.rl" {te = p+1;{ -#line 649 "Parser.rl" +#line 650 "Parser.rl" currentCls->add(CLASS_SCRIPT_GOTHIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5365 "Parser.cpp" +#line 5366 "Parser.cpp" break; } case 94: { { -#line 650 "Parser.rl" +#line 651 "Parser.rl" {te = p+1;{ -#line 650 "Parser.rl" +#line 651 "Parser.rl" currentCls->add(CLASS_SCRIPT_GREEK, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5376 "Parser.cpp" +#line 5377 "Parser.cpp" break; } case 95: { { -#line 651 "Parser.rl" +#line 652 "Parser.rl" {te = p+1;{ -#line 651 "Parser.rl" +#line 652 "Parser.rl" currentCls->add(CLASS_SCRIPT_GUJARATI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5387 "Parser.cpp" +#line 5388 "Parser.cpp" break; } case 96: { { -#line 652 "Parser.rl" +#line 653 "Parser.rl" {te = p+1;{ -#line 652 "Parser.rl" +#line 653 "Parser.rl" currentCls->add(CLASS_SCRIPT_GURMUKHI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5398 "Parser.cpp" +#line 5399 "Parser.cpp" break; } case 97: { { -#line 654 "Parser.rl" +#line 655 "Parser.rl" {te = p+1;{ -#line 654 "Parser.rl" +#line 655 "Parser.rl" currentCls->add(CLASS_SCRIPT_HANGUL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5409 "Parser.cpp" +#line 5410 "Parser.cpp" break; } case 98: { { -#line 655 "Parser.rl" +#line 656 "Parser.rl" {te = p+1;{ -#line 655 "Parser.rl" +#line 656 "Parser.rl" currentCls->add(CLASS_SCRIPT_HANUNOO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5420 "Parser.cpp" +#line 5421 "Parser.cpp" break; } case 99: { { -#line 656 "Parser.rl" +#line 657 "Parser.rl" {te = p+1;{ -#line 656 "Parser.rl" +#line 657 "Parser.rl" currentCls->add(CLASS_SCRIPT_HEBREW, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5431 "Parser.cpp" +#line 5432 "Parser.cpp" break; } case 100: { { -#line 657 "Parser.rl" +#line 658 "Parser.rl" {te = p+1;{ -#line 657 "Parser.rl" +#line 658 "Parser.rl" currentCls->add(CLASS_SCRIPT_HIRAGANA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5442 "Parser.cpp" +#line 5443 "Parser.cpp" break; } case 101: { { -#line 658 "Parser.rl" +#line 659 "Parser.rl" {te = p+1;{ -#line 658 "Parser.rl" +#line 659 "Parser.rl" currentCls->add(CLASS_SCRIPT_IMPERIAL_ARAMAIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5453 "Parser.cpp" +#line 5454 "Parser.cpp" break; } case 102: { { -#line 659 "Parser.rl" +#line 660 "Parser.rl" {te = p+1;{ -#line 659 "Parser.rl" +#line 660 "Parser.rl" currentCls->add(CLASS_SCRIPT_INHERITED, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5464 "Parser.cpp" +#line 5465 "Parser.cpp" break; } case 103: { { -#line 660 "Parser.rl" +#line 661 "Parser.rl" {te = p+1;{ -#line 660 "Parser.rl" +#line 661 "Parser.rl" currentCls->add(CLASS_SCRIPT_INSCRIPTIONAL_PAHLAVI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5475 "Parser.cpp" +#line 5476 "Parser.cpp" break; } case 104: { { -#line 661 "Parser.rl" +#line 662 "Parser.rl" {te = p+1;{ -#line 661 "Parser.rl" +#line 662 "Parser.rl" currentCls->add(CLASS_SCRIPT_INSCRIPTIONAL_PARTHIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5486 "Parser.cpp" +#line 5487 "Parser.cpp" break; } case 105: { { -#line 662 "Parser.rl" +#line 663 "Parser.rl" {te = p+1;{ -#line 662 "Parser.rl" +#line 663 "Parser.rl" currentCls->add(CLASS_SCRIPT_JAVANESE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5497 "Parser.cpp" +#line 5498 "Parser.cpp" break; } case 106: { { -#line 663 "Parser.rl" +#line 664 "Parser.rl" {te = p+1;{ -#line 663 "Parser.rl" +#line 664 "Parser.rl" currentCls->add(CLASS_SCRIPT_KAITHI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5508 "Parser.cpp" +#line 5509 "Parser.cpp" break; } case 107: { { -#line 664 "Parser.rl" +#line 665 "Parser.rl" {te = p+1;{ -#line 664 "Parser.rl" +#line 665 "Parser.rl" currentCls->add(CLASS_SCRIPT_KANNADA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5519 "Parser.cpp" +#line 5520 "Parser.cpp" break; } case 108: { { -#line 665 "Parser.rl" +#line 666 "Parser.rl" {te = p+1;{ -#line 665 "Parser.rl" +#line 666 "Parser.rl" currentCls->add(CLASS_SCRIPT_KATAKANA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5530 "Parser.cpp" +#line 5531 "Parser.cpp" break; } case 109: { { -#line 666 "Parser.rl" +#line 667 "Parser.rl" {te = p+1;{ -#line 666 "Parser.rl" +#line 667 "Parser.rl" currentCls->add(CLASS_SCRIPT_KAYAH_LI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5541 "Parser.cpp" +#line 5542 "Parser.cpp" break; } case 110: { { -#line 667 "Parser.rl" +#line 668 "Parser.rl" {te = p+1;{ -#line 667 "Parser.rl" +#line 668 "Parser.rl" currentCls->add(CLASS_SCRIPT_KHAROSHTHI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5552 "Parser.cpp" +#line 5553 "Parser.cpp" break; } case 111: { { -#line 668 "Parser.rl" +#line 669 "Parser.rl" {te = p+1;{ -#line 668 "Parser.rl" +#line 669 "Parser.rl" currentCls->add(CLASS_SCRIPT_KHMER, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5563 "Parser.cpp" +#line 5564 "Parser.cpp" break; } case 112: { { -#line 669 "Parser.rl" +#line 670 "Parser.rl" {te = p+1;{ -#line 669 "Parser.rl" +#line 670 "Parser.rl" currentCls->add(CLASS_SCRIPT_LAO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5574 "Parser.cpp" +#line 5575 "Parser.cpp" break; } case 113: { { -#line 670 "Parser.rl" +#line 671 "Parser.rl" {te = p+1;{ -#line 670 "Parser.rl" +#line 671 "Parser.rl" currentCls->add(CLASS_SCRIPT_LATIN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5585 "Parser.cpp" +#line 5586 "Parser.cpp" break; } case 114: { { -#line 671 "Parser.rl" +#line 672 "Parser.rl" {te = p+1;{ -#line 671 "Parser.rl" +#line 672 "Parser.rl" currentCls->add(CLASS_SCRIPT_LEPCHA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5596 "Parser.cpp" +#line 5597 "Parser.cpp" break; } case 115: { { -#line 672 "Parser.rl" +#line 673 "Parser.rl" {te = p+1;{ -#line 672 "Parser.rl" +#line 673 "Parser.rl" currentCls->add(CLASS_SCRIPT_LIMBU, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5607 "Parser.cpp" +#line 5608 "Parser.cpp" break; } case 116: { { -#line 673 "Parser.rl" +#line 674 "Parser.rl" {te = p+1;{ -#line 673 "Parser.rl" +#line 674 "Parser.rl" currentCls->add(CLASS_SCRIPT_LINEAR_B, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5618 "Parser.cpp" +#line 5619 "Parser.cpp" break; } case 117: { { -#line 674 "Parser.rl" +#line 675 "Parser.rl" {te = p+1;{ -#line 674 "Parser.rl" +#line 675 "Parser.rl" currentCls->add(CLASS_SCRIPT_LISU, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5629 "Parser.cpp" +#line 5630 "Parser.cpp" break; } case 118: { { -#line 675 "Parser.rl" +#line 676 "Parser.rl" {te = p+1;{ -#line 675 "Parser.rl" +#line 676 "Parser.rl" currentCls->add(CLASS_SCRIPT_LYCIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5640 "Parser.cpp" +#line 5641 "Parser.cpp" break; } case 119: { { -#line 676 "Parser.rl" +#line 677 "Parser.rl" {te = p+1;{ -#line 676 "Parser.rl" +#line 677 "Parser.rl" currentCls->add(CLASS_SCRIPT_LYDIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5651 "Parser.cpp" +#line 5652 "Parser.cpp" break; } case 120: { { -#line 677 "Parser.rl" +#line 678 "Parser.rl" {te = p+1;{ -#line 677 "Parser.rl" +#line 678 "Parser.rl" currentCls->add(CLASS_SCRIPT_MALAYALAM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5662 "Parser.cpp" +#line 5663 "Parser.cpp" break; } case 121: { { -#line 678 "Parser.rl" +#line 679 "Parser.rl" {te = p+1;{ -#line 678 "Parser.rl" +#line 679 "Parser.rl" currentCls->add(CLASS_SCRIPT_MANDAIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5673 "Parser.cpp" +#line 5674 "Parser.cpp" break; } case 122: { { -#line 679 "Parser.rl" +#line 680 "Parser.rl" {te = p+1;{ -#line 679 "Parser.rl" +#line 680 "Parser.rl" currentCls->add(CLASS_SCRIPT_MEETEI_MAYEK, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5684 "Parser.cpp" +#line 5685 "Parser.cpp" break; } case 123: { { -#line 680 "Parser.rl" +#line 681 "Parser.rl" {te = p+1;{ -#line 680 "Parser.rl" +#line 681 "Parser.rl" currentCls->add(CLASS_SCRIPT_MONGOLIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5695 "Parser.cpp" +#line 5696 "Parser.cpp" break; } case 124: { { -#line 681 "Parser.rl" +#line 682 "Parser.rl" {te = p+1;{ -#line 681 "Parser.rl" +#line 682 "Parser.rl" currentCls->add(CLASS_SCRIPT_MYANMAR, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5706 "Parser.cpp" +#line 5707 "Parser.cpp" break; } case 125: { { -#line 682 "Parser.rl" +#line 683 "Parser.rl" {te = p+1;{ -#line 682 "Parser.rl" +#line 683 "Parser.rl" currentCls->add(CLASS_SCRIPT_NEW_TAI_LUE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5717 "Parser.cpp" +#line 5718 "Parser.cpp" break; } case 126: { { -#line 683 "Parser.rl" +#line 684 "Parser.rl" {te = p+1;{ -#line 683 "Parser.rl" +#line 684 "Parser.rl" currentCls->add(CLASS_SCRIPT_NKO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5728 "Parser.cpp" +#line 5729 "Parser.cpp" break; } case 127: { { -#line 684 "Parser.rl" +#line 685 "Parser.rl" {te = p+1;{ -#line 684 "Parser.rl" +#line 685 "Parser.rl" currentCls->add(CLASS_SCRIPT_OGHAM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5739 "Parser.cpp" +#line 5740 "Parser.cpp" break; } case 128: { { -#line 685 "Parser.rl" +#line 686 "Parser.rl" {te = p+1;{ -#line 685 "Parser.rl" +#line 686 "Parser.rl" currentCls->add(CLASS_SCRIPT_OL_CHIKI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5750 "Parser.cpp" +#line 5751 "Parser.cpp" break; } case 129: { { -#line 686 "Parser.rl" +#line 687 "Parser.rl" {te = p+1;{ -#line 686 "Parser.rl" +#line 687 "Parser.rl" currentCls->add(CLASS_SCRIPT_OLD_ITALIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5761 "Parser.cpp" +#line 5762 "Parser.cpp" break; } case 130: { { -#line 687 "Parser.rl" +#line 688 "Parser.rl" {te = p+1;{ -#line 687 "Parser.rl" +#line 688 "Parser.rl" currentCls->add(CLASS_SCRIPT_OLD_PERSIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5772 "Parser.cpp" +#line 5773 "Parser.cpp" break; } case 131: { { -#line 688 "Parser.rl" +#line 689 "Parser.rl" {te = p+1;{ -#line 688 "Parser.rl" +#line 689 "Parser.rl" currentCls->add(CLASS_SCRIPT_OLD_SOUTH_ARABIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5783 "Parser.cpp" +#line 5784 "Parser.cpp" break; } case 132: { { -#line 689 "Parser.rl" +#line 690 "Parser.rl" {te = p+1;{ -#line 689 "Parser.rl" +#line 690 "Parser.rl" currentCls->add(CLASS_SCRIPT_OLD_TURKIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5794 "Parser.cpp" +#line 5795 "Parser.cpp" break; } case 133: { { -#line 690 "Parser.rl" +#line 691 "Parser.rl" {te = p+1;{ -#line 690 "Parser.rl" +#line 691 "Parser.rl" currentCls->add(CLASS_SCRIPT_ORIYA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5805 "Parser.cpp" +#line 5806 "Parser.cpp" break; } case 134: { { -#line 691 "Parser.rl" +#line 692 "Parser.rl" {te = p+1;{ -#line 691 "Parser.rl" +#line 692 "Parser.rl" currentCls->add(CLASS_SCRIPT_OSMANYA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5816 "Parser.cpp" +#line 5817 "Parser.cpp" break; } case 135: { { -#line 692 "Parser.rl" +#line 693 "Parser.rl" {te = p+1;{ -#line 692 "Parser.rl" +#line 693 "Parser.rl" currentCls->add(CLASS_SCRIPT_PHAGS_PA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5827 "Parser.cpp" +#line 5828 "Parser.cpp" break; } case 136: { { -#line 693 "Parser.rl" +#line 694 "Parser.rl" {te = p+1;{ -#line 693 "Parser.rl" +#line 694 "Parser.rl" currentCls->add(CLASS_SCRIPT_PHOENICIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5838 "Parser.cpp" +#line 5839 "Parser.cpp" break; } case 137: { { -#line 694 "Parser.rl" +#line 695 "Parser.rl" {te = p+1;{ -#line 694 "Parser.rl" +#line 695 "Parser.rl" currentCls->add(CLASS_SCRIPT_REJANG, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5849 "Parser.cpp" +#line 5850 "Parser.cpp" break; } case 138: { { -#line 695 "Parser.rl" +#line 696 "Parser.rl" {te = p+1;{ -#line 695 "Parser.rl" +#line 696 "Parser.rl" currentCls->add(CLASS_SCRIPT_RUNIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5860 "Parser.cpp" +#line 5861 "Parser.cpp" break; } case 139: { { -#line 696 "Parser.rl" +#line 697 "Parser.rl" {te = p+1;{ -#line 696 "Parser.rl" +#line 697 "Parser.rl" currentCls->add(CLASS_SCRIPT_SAMARITAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5871 "Parser.cpp" +#line 5872 "Parser.cpp" break; } case 140: { { -#line 697 "Parser.rl" +#line 698 "Parser.rl" {te = p+1;{ -#line 697 "Parser.rl" +#line 698 "Parser.rl" currentCls->add(CLASS_SCRIPT_SAURASHTRA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5882 "Parser.cpp" +#line 5883 "Parser.cpp" break; } case 141: { { -#line 698 "Parser.rl" +#line 699 "Parser.rl" {te = p+1;{ -#line 698 "Parser.rl" +#line 699 "Parser.rl" currentCls->add(CLASS_SCRIPT_SHAVIAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5893 "Parser.cpp" +#line 5894 "Parser.cpp" break; } case 142: { { -#line 699 "Parser.rl" +#line 700 "Parser.rl" {te = p+1;{ -#line 699 "Parser.rl" +#line 700 "Parser.rl" currentCls->add(CLASS_SCRIPT_SINHALA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5904 "Parser.cpp" +#line 5905 "Parser.cpp" break; } case 143: { { -#line 700 "Parser.rl" +#line 701 "Parser.rl" {te = p+1;{ -#line 700 "Parser.rl" +#line 701 "Parser.rl" currentCls->add(CLASS_SCRIPT_SUNDANESE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5915 "Parser.cpp" +#line 5916 "Parser.cpp" break; } case 144: { { -#line 701 "Parser.rl" +#line 702 "Parser.rl" {te = p+1;{ -#line 701 "Parser.rl" +#line 702 "Parser.rl" currentCls->add(CLASS_SCRIPT_SYLOTI_NAGRI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5926 "Parser.cpp" +#line 5927 "Parser.cpp" break; } case 145: { { -#line 702 "Parser.rl" +#line 703 "Parser.rl" {te = p+1;{ -#line 702 "Parser.rl" +#line 703 "Parser.rl" currentCls->add(CLASS_SCRIPT_SYRIAC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5937 "Parser.cpp" +#line 5938 "Parser.cpp" break; } case 146: { { -#line 703 "Parser.rl" +#line 704 "Parser.rl" {te = p+1;{ -#line 703 "Parser.rl" +#line 704 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAGALOG, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5948 "Parser.cpp" +#line 5949 "Parser.cpp" break; } case 147: { { -#line 704 "Parser.rl" +#line 705 "Parser.rl" {te = p+1;{ -#line 704 "Parser.rl" +#line 705 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAGBANWA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5959 "Parser.cpp" +#line 5960 "Parser.cpp" break; } case 148: { { -#line 705 "Parser.rl" +#line 706 "Parser.rl" {te = p+1;{ -#line 705 "Parser.rl" +#line 706 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAI_LE, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5970 "Parser.cpp" +#line 5971 "Parser.cpp" break; } case 149: { { -#line 706 "Parser.rl" +#line 707 "Parser.rl" {te = p+1;{ -#line 706 "Parser.rl" +#line 707 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAI_THAM, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5981 "Parser.cpp" +#line 5982 "Parser.cpp" break; } case 150: { { -#line 707 "Parser.rl" +#line 708 "Parser.rl" {te = p+1;{ -#line 707 "Parser.rl" +#line 708 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAI_VIET, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 5992 "Parser.cpp" +#line 5993 "Parser.cpp" break; } case 151: { { -#line 708 "Parser.rl" +#line 709 "Parser.rl" {te = p+1;{ -#line 708 "Parser.rl" +#line 709 "Parser.rl" currentCls->add(CLASS_SCRIPT_TAMIL, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6003 "Parser.cpp" +#line 6004 "Parser.cpp" break; } case 152: { { -#line 709 "Parser.rl" +#line 710 "Parser.rl" {te = p+1;{ -#line 709 "Parser.rl" +#line 710 "Parser.rl" currentCls->add(CLASS_SCRIPT_TELUGU, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6014 "Parser.cpp" +#line 6015 "Parser.cpp" break; } case 153: { { -#line 710 "Parser.rl" +#line 711 "Parser.rl" {te = p+1;{ -#line 710 "Parser.rl" +#line 711 "Parser.rl" currentCls->add(CLASS_SCRIPT_THAANA, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6025 "Parser.cpp" +#line 6026 "Parser.cpp" break; } case 154: { { -#line 711 "Parser.rl" +#line 712 "Parser.rl" {te = p+1;{ -#line 711 "Parser.rl" +#line 712 "Parser.rl" currentCls->add(CLASS_SCRIPT_THAI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6036 "Parser.cpp" +#line 6037 "Parser.cpp" break; } case 155: { { -#line 712 "Parser.rl" +#line 713 "Parser.rl" {te = p+1;{ -#line 712 "Parser.rl" +#line 713 "Parser.rl" currentCls->add(CLASS_SCRIPT_TIBETAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6047 "Parser.cpp" +#line 6048 "Parser.cpp" break; } case 156: { { -#line 713 "Parser.rl" +#line 714 "Parser.rl" {te = p+1;{ -#line 713 "Parser.rl" +#line 714 "Parser.rl" currentCls->add(CLASS_SCRIPT_TIFINAGH, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6058 "Parser.cpp" +#line 6059 "Parser.cpp" break; } case 157: { { -#line 714 "Parser.rl" +#line 715 "Parser.rl" {te = p+1;{ -#line 714 "Parser.rl" +#line 715 "Parser.rl" currentCls->add(CLASS_SCRIPT_UGARITIC, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6069 "Parser.cpp" +#line 6070 "Parser.cpp" break; } case 158: { { -#line 715 "Parser.rl" +#line 716 "Parser.rl" {te = p+1;{ -#line 715 "Parser.rl" +#line 716 "Parser.rl" currentCls->add(CLASS_SCRIPT_VAI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6080 "Parser.cpp" +#line 6081 "Parser.cpp" break; } case 159: { { -#line 716 "Parser.rl" +#line 717 "Parser.rl" {te = p+1;{ -#line 716 "Parser.rl" +#line 717 "Parser.rl" currentCls->add(CLASS_SCRIPT_YI, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6091 "Parser.cpp" +#line 6092 "Parser.cpp" break; } case 160: { { -#line 717 "Parser.rl" +#line 718 "Parser.rl" {te = p+1;{ -#line 717 "Parser.rl" +#line 718 "Parser.rl" currentCls->add(CLASS_UCP_ANY, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6102 "Parser.cpp" +#line 6103 "Parser.cpp" break; } case 161: { { -#line 718 "Parser.rl" +#line 719 "Parser.rl" {te = p+1;{ -#line 718 "Parser.rl" +#line 719 "Parser.rl" throw LocatedParseError("Unknown property"); } }} -#line 6113 "Parser.cpp" +#line 6114 "Parser.cpp" break; } case 162: { { -#line 580 "Parser.rl" +#line 581 "Parser.rl" {te = p;p = p - 1;{ -#line 580 "Parser.rl" +#line 581 "Parser.rl" currentCls->add(CLASS_UCP_C, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6124 "Parser.cpp" +#line 6125 "Parser.cpp" break; } case 163: { { -#line 584 "Parser.rl" +#line 585 "Parser.rl" {te = p;p = p - 1;{ -#line 584 "Parser.rl" +#line 585 "Parser.rl" currentCls->add(CLASS_UCP_CO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6135 "Parser.cpp" +#line 6136 "Parser.cpp" break; } case 164: { { -#line 586 "Parser.rl" +#line 587 "Parser.rl" {te = p;p = p - 1;{ -#line 586 "Parser.rl" +#line 587 "Parser.rl" currentCls->add(CLASS_UCP_L, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6146 "Parser.cpp" +#line 6147 "Parser.cpp" break; } case 165: { { -#line 593 "Parser.rl" +#line 594 "Parser.rl" {te = p;p = p - 1;{ -#line 593 "Parser.rl" +#line 594 "Parser.rl" currentCls->add(CLASS_UCP_M, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6157 "Parser.cpp" +#line 6158 "Parser.cpp" break; } case 166: { { -#line 595 "Parser.rl" +#line 596 "Parser.rl" {te = p;p = p - 1;{ -#line 595 "Parser.rl" +#line 596 "Parser.rl" currentCls->add(CLASS_UCP_ME, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6168 "Parser.cpp" +#line 6169 "Parser.cpp" break; } case 167: { { -#line 597 "Parser.rl" +#line 598 "Parser.rl" {te = p;p = p - 1;{ -#line 597 "Parser.rl" +#line 598 "Parser.rl" currentCls->add(CLASS_UCP_N, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6179 "Parser.cpp" +#line 6180 "Parser.cpp" break; } case 168: { { -#line 601 "Parser.rl" +#line 602 "Parser.rl" {te = p;p = p - 1;{ -#line 601 "Parser.rl" +#line 602 "Parser.rl" currentCls->add(CLASS_UCP_P, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6190 "Parser.cpp" +#line 6191 "Parser.cpp" break; } case 169: { { -#line 609 "Parser.rl" +#line 610 "Parser.rl" {te = p;p = p - 1;{ -#line 609 "Parser.rl" +#line 610 "Parser.rl" currentCls->add(CLASS_UCP_S, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6201 "Parser.cpp" +#line 6202 "Parser.cpp" break; } case 170: { { -#line 614 "Parser.rl" +#line 615 "Parser.rl" {te = p;p = p - 1;{ -#line 614 "Parser.rl" +#line 615 "Parser.rl" currentCls->add(CLASS_UCP_Z, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6212 "Parser.cpp" +#line 6213 "Parser.cpp" break; } case 171: { { -#line 653 "Parser.rl" +#line 654 "Parser.rl" {te = p;p = p - 1;{ -#line 653 "Parser.rl" +#line 654 "Parser.rl" currentCls->add(CLASS_SCRIPT_HAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6223 "Parser.cpp" +#line 6224 "Parser.cpp" break; } case 172: { { -#line 718 "Parser.rl" +#line 719 "Parser.rl" {te = p;p = p - 1;{ -#line 718 "Parser.rl" +#line 719 "Parser.rl" throw LocatedParseError("Unknown property"); } }} -#line 6234 "Parser.cpp" +#line 6235 "Parser.cpp" break; } case 173: { { -#line 580 "Parser.rl" +#line 581 "Parser.rl" {p = ((te))-1; { -#line 580 "Parser.rl" +#line 581 "Parser.rl" currentCls->add(CLASS_UCP_C, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6246 "Parser.cpp" +#line 6247 "Parser.cpp" break; } case 174: { { -#line 584 "Parser.rl" +#line 585 "Parser.rl" {p = ((te))-1; { -#line 584 "Parser.rl" +#line 585 "Parser.rl" currentCls->add(CLASS_UCP_CO, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6258 "Parser.cpp" +#line 6259 "Parser.cpp" break; } case 175: { { -#line 586 "Parser.rl" +#line 587 "Parser.rl" {p = ((te))-1; { -#line 586 "Parser.rl" +#line 587 "Parser.rl" currentCls->add(CLASS_UCP_L, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6270 "Parser.cpp" +#line 6271 "Parser.cpp" break; } case 176: { { -#line 593 "Parser.rl" +#line 594 "Parser.rl" {p = ((te))-1; { -#line 593 "Parser.rl" +#line 594 "Parser.rl" currentCls->add(CLASS_UCP_M, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6282 "Parser.cpp" +#line 6283 "Parser.cpp" break; } case 177: { { -#line 595 "Parser.rl" +#line 596 "Parser.rl" {p = ((te))-1; { -#line 595 "Parser.rl" +#line 596 "Parser.rl" currentCls->add(CLASS_UCP_ME, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6294 "Parser.cpp" +#line 6295 "Parser.cpp" break; } case 178: { { -#line 597 "Parser.rl" +#line 598 "Parser.rl" {p = ((te))-1; { -#line 597 "Parser.rl" +#line 598 "Parser.rl" currentCls->add(CLASS_UCP_N, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6306 "Parser.cpp" +#line 6307 "Parser.cpp" break; } case 179: { { -#line 601 "Parser.rl" +#line 602 "Parser.rl" {p = ((te))-1; { -#line 601 "Parser.rl" +#line 602 "Parser.rl" currentCls->add(CLASS_UCP_P, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6318 "Parser.cpp" +#line 6319 "Parser.cpp" break; } case 180: { { -#line 609 "Parser.rl" +#line 610 "Parser.rl" {p = ((te))-1; { -#line 609 "Parser.rl" +#line 610 "Parser.rl" currentCls->add(CLASS_UCP_S, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6330 "Parser.cpp" +#line 6331 "Parser.cpp" break; } case 181: { { -#line 653 "Parser.rl" +#line 654 "Parser.rl" {p = ((te))-1; { -#line 653 "Parser.rl" +#line 654 "Parser.rl" currentCls->add(CLASS_SCRIPT_HAN, negated); {top -= 1;cs = stack[top];goto _again;} } }} -#line 6342 "Parser.cpp" +#line 6343 "Parser.cpp" break; } case 182: { { -#line 718 "Parser.rl" +#line 719 "Parser.rl" {p = ((te))-1; { -#line 718 "Parser.rl" +#line 719 "Parser.rl" throw LocatedParseError("Unknown property"); } }} -#line 6354 "Parser.cpp" +#line 6355 "Parser.cpp" break; } case 183: { { -#line 733 "Parser.rl" +#line 734 "Parser.rl" {te = p+1;{ -#line 733 "Parser.rl" +#line 734 "Parser.rl" currentCls->add(CLASS_UCP_C, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6372 "Parser.cpp" +#line 6373 "Parser.cpp" break; } case 184: { { -#line 741 "Parser.rl" +#line 742 "Parser.rl" {te = p+1;{ -#line 741 "Parser.rl" +#line 742 "Parser.rl" currentCls->add(CLASS_UCP_L, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6390 "Parser.cpp" +#line 6391 "Parser.cpp" break; } case 185: { { -#line 749 "Parser.rl" +#line 750 "Parser.rl" {te = p+1;{ -#line 749 "Parser.rl" +#line 750 "Parser.rl" currentCls->add(CLASS_UCP_M, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6408 "Parser.cpp" +#line 6409 "Parser.cpp" break; } case 186: { { -#line 757 "Parser.rl" +#line 758 "Parser.rl" {te = p+1;{ -#line 757 "Parser.rl" +#line 758 "Parser.rl" currentCls->add(CLASS_UCP_N, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6426 "Parser.cpp" +#line 6427 "Parser.cpp" break; } case 187: { { -#line 765 "Parser.rl" +#line 766 "Parser.rl" {te = p+1;{ -#line 765 "Parser.rl" +#line 766 "Parser.rl" currentCls->add(CLASS_UCP_P, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6444 "Parser.cpp" +#line 6445 "Parser.cpp" break; } case 188: { { -#line 773 "Parser.rl" +#line 774 "Parser.rl" {te = p+1;{ -#line 773 "Parser.rl" +#line 774 "Parser.rl" currentCls->add(CLASS_UCP_S, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6462 "Parser.cpp" +#line 6463 "Parser.cpp" break; } case 189: { { -#line 781 "Parser.rl" +#line 782 "Parser.rl" {te = p+1;{ -#line 781 "Parser.rl" +#line 782 "Parser.rl" currentCls->add(CLASS_UCP_Z, negated); if (!inCharClass) { currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); } {top -= 1;cs = stack[top];goto _again;} } }} -#line 6480 "Parser.cpp" +#line 6481 "Parser.cpp" break; } case 190: { { -#line 790 "Parser.rl" +#line 791 "Parser.rl" {te = p+1;{ -#line 790 "Parser.rl" +#line 791 "Parser.rl" throw LocatedParseError("Unknown property"); } }} -#line 6491 "Parser.cpp" +#line 6492 "Parser.cpp" break; } case 191: { { -#line 796 "Parser.rl" +#line 797 "Parser.rl" {te = p+1;{ -#line 796 "Parser.rl" +#line 797 "Parser.rl" throw LocatedParseError("Unsupported POSIX collating " "element"); } }} -#line 6505 "Parser.cpp" +#line 6506 "Parser.cpp" break; } case 192: { { -#line 803 "Parser.rl" +#line 804 "Parser.rl" {te = p+1;{ -#line 803 "Parser.rl" +#line 804 "Parser.rl" currentCls->add(CLASS_ALNUM, false); } }} -#line 6518 "Parser.cpp" +#line 6519 "Parser.cpp" break; } case 193: { { -#line 806 "Parser.rl" +#line 807 "Parser.rl" {te = p+1;{ -#line 806 "Parser.rl" +#line 807 "Parser.rl" currentCls->add(CLASS_ALNUM, true); } }} -#line 6531 "Parser.cpp" +#line 6532 "Parser.cpp" break; } case 194: { { -#line 809 "Parser.rl" +#line 810 "Parser.rl" {te = p+1;{ -#line 809 "Parser.rl" +#line 810 "Parser.rl" currentCls->add(CLASS_ALPHA, false); } }} -#line 6544 "Parser.cpp" +#line 6545 "Parser.cpp" break; } case 195: { { -#line 812 "Parser.rl" +#line 813 "Parser.rl" {te = p+1;{ -#line 812 "Parser.rl" +#line 813 "Parser.rl" currentCls->add(CLASS_ALPHA, true); } }} -#line 6557 "Parser.cpp" +#line 6558 "Parser.cpp" break; } case 196: { { -#line 815 "Parser.rl" +#line 816 "Parser.rl" {te = p+1;{ -#line 815 "Parser.rl" +#line 816 "Parser.rl" currentCls->add(CLASS_ASCII, false); } }} -#line 6570 "Parser.cpp" +#line 6571 "Parser.cpp" break; } case 197: { { -#line 818 "Parser.rl" +#line 819 "Parser.rl" {te = p+1;{ -#line 818 "Parser.rl" +#line 819 "Parser.rl" currentCls->add(CLASS_ASCII, true); } }} -#line 6583 "Parser.cpp" +#line 6584 "Parser.cpp" break; } case 198: { { -#line 821 "Parser.rl" +#line 822 "Parser.rl" {te = p+1;{ -#line 821 "Parser.rl" +#line 822 "Parser.rl" currentCls->add(CLASS_BLANK, false); } }} -#line 6596 "Parser.cpp" +#line 6597 "Parser.cpp" break; } case 199: { { -#line 824 "Parser.rl" +#line 825 "Parser.rl" {te = p+1;{ -#line 824 "Parser.rl" +#line 825 "Parser.rl" currentCls->add(CLASS_BLANK, true); } }} -#line 6609 "Parser.cpp" +#line 6610 "Parser.cpp" break; } case 200: { { -#line 827 "Parser.rl" +#line 828 "Parser.rl" {te = p+1;{ -#line 827 "Parser.rl" +#line 828 "Parser.rl" currentCls->add(CLASS_CNTRL, false); } }} -#line 6622 "Parser.cpp" +#line 6623 "Parser.cpp" break; } case 201: { { -#line 830 "Parser.rl" +#line 831 "Parser.rl" {te = p+1;{ -#line 830 "Parser.rl" +#line 831 "Parser.rl" currentCls->add(CLASS_CNTRL, true); } }} -#line 6635 "Parser.cpp" +#line 6636 "Parser.cpp" break; } case 202: { { -#line 833 "Parser.rl" +#line 834 "Parser.rl" {te = p+1;{ -#line 833 "Parser.rl" +#line 834 "Parser.rl" currentCls->add(CLASS_DIGIT, false); } }} -#line 6648 "Parser.cpp" +#line 6649 "Parser.cpp" break; } case 203: { { -#line 836 "Parser.rl" +#line 837 "Parser.rl" {te = p+1;{ -#line 836 "Parser.rl" +#line 837 "Parser.rl" currentCls->add(CLASS_DIGIT, true); } }} -#line 6661 "Parser.cpp" +#line 6662 "Parser.cpp" break; } case 204: { { -#line 839 "Parser.rl" +#line 840 "Parser.rl" {te = p+1;{ -#line 839 "Parser.rl" +#line 840 "Parser.rl" currentCls->add(CLASS_GRAPH, false); } }} -#line 6674 "Parser.cpp" +#line 6675 "Parser.cpp" break; } case 205: { { -#line 842 "Parser.rl" +#line 843 "Parser.rl" {te = p+1;{ -#line 842 "Parser.rl" +#line 843 "Parser.rl" currentCls->add(CLASS_GRAPH, true); } }} -#line 6687 "Parser.cpp" +#line 6688 "Parser.cpp" break; } case 206: { { -#line 845 "Parser.rl" +#line 846 "Parser.rl" {te = p+1;{ -#line 845 "Parser.rl" +#line 846 "Parser.rl" currentCls->add(CLASS_LOWER, false); } }} -#line 6700 "Parser.cpp" +#line 6701 "Parser.cpp" break; } case 207: { { -#line 848 "Parser.rl" +#line 849 "Parser.rl" {te = p+1;{ -#line 848 "Parser.rl" +#line 849 "Parser.rl" currentCls->add(CLASS_LOWER, true); } }} -#line 6713 "Parser.cpp" +#line 6714 "Parser.cpp" break; } case 208: { { -#line 851 "Parser.rl" +#line 852 "Parser.rl" {te = p+1;{ -#line 851 "Parser.rl" +#line 852 "Parser.rl" currentCls->add(CLASS_PRINT, false); } }} -#line 6726 "Parser.cpp" +#line 6727 "Parser.cpp" break; } case 209: { { -#line 854 "Parser.rl" +#line 855 "Parser.rl" {te = p+1;{ -#line 854 "Parser.rl" +#line 855 "Parser.rl" currentCls->add(CLASS_PRINT, true); } }} -#line 6739 "Parser.cpp" +#line 6740 "Parser.cpp" break; } case 210: { { -#line 857 "Parser.rl" +#line 858 "Parser.rl" {te = p+1;{ -#line 857 "Parser.rl" +#line 858 "Parser.rl" currentCls->add(CLASS_PUNCT, false); } }} -#line 6752 "Parser.cpp" +#line 6753 "Parser.cpp" break; } case 211: { { -#line 860 "Parser.rl" +#line 861 "Parser.rl" {te = p+1;{ -#line 860 "Parser.rl" +#line 861 "Parser.rl" currentCls->add(CLASS_PUNCT, true); } }} -#line 6765 "Parser.cpp" +#line 6766 "Parser.cpp" break; } case 212: { { -#line 864 "Parser.rl" +#line 865 "Parser.rl" {te = p+1;{ -#line 864 "Parser.rl" +#line 865 "Parser.rl" currentCls->add(CLASS_SPACE, false); } }} -#line 6778 "Parser.cpp" +#line 6779 "Parser.cpp" break; } case 213: { { -#line 867 "Parser.rl" +#line 868 "Parser.rl" {te = p+1;{ -#line 867 "Parser.rl" +#line 868 "Parser.rl" currentCls->add(CLASS_SPACE, true); } }} -#line 6791 "Parser.cpp" +#line 6792 "Parser.cpp" break; } case 214: { { -#line 870 "Parser.rl" +#line 871 "Parser.rl" {te = p+1;{ -#line 870 "Parser.rl" +#line 871 "Parser.rl" currentCls->add(CLASS_UPPER, false); } }} -#line 6804 "Parser.cpp" +#line 6805 "Parser.cpp" break; } case 215: { { -#line 873 "Parser.rl" +#line 874 "Parser.rl" {te = p+1;{ -#line 873 "Parser.rl" +#line 874 "Parser.rl" currentCls->add(CLASS_UPPER, true); } }} -#line 6817 "Parser.cpp" +#line 6818 "Parser.cpp" break; } case 216: { { -#line 876 "Parser.rl" +#line 877 "Parser.rl" {te = p+1;{ -#line 876 "Parser.rl" +#line 877 "Parser.rl" currentCls->add(CLASS_WORD, false); } }} -#line 6830 "Parser.cpp" +#line 6831 "Parser.cpp" break; } case 217: { { -#line 879 "Parser.rl" +#line 880 "Parser.rl" {te = p+1;{ -#line 879 "Parser.rl" +#line 880 "Parser.rl" currentCls->add(CLASS_WORD, true); } }} -#line 6843 "Parser.cpp" +#line 6844 "Parser.cpp" break; } case 218: { { -#line 882 "Parser.rl" +#line 883 "Parser.rl" {te = p+1;{ -#line 882 "Parser.rl" +#line 883 "Parser.rl" currentCls->add(CLASS_XDIGIT, false); } }} -#line 6856 "Parser.cpp" +#line 6857 "Parser.cpp" break; } case 219: { { -#line 885 "Parser.rl" +#line 886 "Parser.rl" {te = p+1;{ -#line 885 "Parser.rl" +#line 886 "Parser.rl" currentCls->add(CLASS_XDIGIT, true); } }} -#line 6869 "Parser.cpp" +#line 6870 "Parser.cpp" break; } case 220: { { -#line 890 "Parser.rl" +#line 891 "Parser.rl" {te = p+1;{ -#line 890 "Parser.rl" +#line 891 "Parser.rl" throw LocatedParseError("Invalid POSIX named class"); } }} -#line 6882 "Parser.cpp" +#line 6883 "Parser.cpp" break; } case 221: { { -#line 893 "Parser.rl" +#line 894 "Parser.rl" {te = p+1;{ -#line 893 "Parser.rl" +#line 894 "Parser.rl" {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { stack.resize(2 * (top + 1)); } } - stack[top] = cs; top += 1;cs = 843;goto _again;}} + stack[top] = cs; top += 1;cs = 844;goto _again;}} }} -#line 6902 "Parser.cpp" +#line 6903 "Parser.cpp" break; } case 222: { { -#line 896 "Parser.rl" +#line 897 "Parser.rl" {te = p+1;{ -#line 896 "Parser.rl" +#line 897 "Parser.rl" /*noop*/} }} -#line 6913 "Parser.cpp" +#line 6914 "Parser.cpp" break; } case 223: { { -#line 898 "Parser.rl" +#line 899 "Parser.rl" {te = p+1;{ -#line 898 "Parser.rl" +#line 899 "Parser.rl" currentCls->add('\x08'); } }} -#line 6926 "Parser.cpp" +#line 6927 "Parser.cpp" break; } case 224: { { -#line 902 "Parser.rl" +#line 903 "Parser.rl" {te = p+1;{ -#line 902 "Parser.rl" +#line 903 "Parser.rl" currentCls->add('\x09'); } }} -#line 6939 "Parser.cpp" +#line 6940 "Parser.cpp" break; } case 225: { { -#line 906 "Parser.rl" +#line 907 "Parser.rl" {te = p+1;{ -#line 906 "Parser.rl" +#line 907 "Parser.rl" currentCls->add('\x0a'); } }} -#line 6952 "Parser.cpp" +#line 6953 "Parser.cpp" break; } case 226: { { -#line 910 "Parser.rl" +#line 911 "Parser.rl" {te = p+1;{ -#line 910 "Parser.rl" +#line 911 "Parser.rl" currentCls->add('\x0d'); } }} -#line 6965 "Parser.cpp" +#line 6966 "Parser.cpp" break; } case 227: { { -#line 914 "Parser.rl" +#line 915 "Parser.rl" {te = p+1;{ -#line 914 "Parser.rl" +#line 915 "Parser.rl" currentCls->add('\x0c'); } }} -#line 6978 "Parser.cpp" +#line 6979 "Parser.cpp" break; } case 228: { { -#line 918 "Parser.rl" +#line 919 "Parser.rl" {te = p+1;{ -#line 918 "Parser.rl" +#line 919 "Parser.rl" currentCls->add('\x07'); } }} -#line 6991 "Parser.cpp" +#line 6992 "Parser.cpp" break; } case 229: { { -#line 922 "Parser.rl" +#line 923 "Parser.rl" {te = p+1;{ -#line 922 "Parser.rl" +#line 923 "Parser.rl" currentCls->add('\x1b'); } }} -#line 7004 "Parser.cpp" +#line 7005 "Parser.cpp" break; } case 230: { { -#line 926 "Parser.rl" +#line 927 "Parser.rl" {te = p+1;{ -#line 926 "Parser.rl" +#line 927 "Parser.rl" currentCls->add(CLASS_HORZ, false); } }} -#line 7017 "Parser.cpp" +#line 7018 "Parser.cpp" break; } case 231: { { -#line 930 "Parser.rl" +#line 931 "Parser.rl" {te = p+1;{ -#line 930 "Parser.rl" +#line 931 "Parser.rl" currentCls->add(CLASS_HORZ, true); } }} -#line 7030 "Parser.cpp" +#line 7031 "Parser.cpp" break; } case 232: { { -#line 934 "Parser.rl" +#line 935 "Parser.rl" {te = p+1;{ -#line 934 "Parser.rl" +#line 935 "Parser.rl" currentCls->add(CLASS_VERT, false); } }} -#line 7043 "Parser.cpp" +#line 7044 "Parser.cpp" break; } case 233: { { -#line 938 "Parser.rl" +#line 939 "Parser.rl" {te = p+1;{ -#line 938 "Parser.rl" +#line 939 "Parser.rl" currentCls->add(CLASS_VERT, true); } }} -#line 7056 "Parser.cpp" +#line 7057 "Parser.cpp" break; } case 234: { { -#line 942 "Parser.rl" +#line 943 "Parser.rl" {te = p+1;{ -#line 942 "Parser.rl" +#line 943 "Parser.rl" negated = false; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -7320,20 +7321,20 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 559;goto _again;}} }} -#line 7078 "Parser.cpp" +#line 7079 "Parser.cpp" break; } case 235: { { -#line 948 "Parser.rl" +#line 949 "Parser.rl" {te = p+1;{ -#line 948 "Parser.rl" +#line 949 "Parser.rl" negated = false; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -7343,20 +7344,20 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 818;goto _again;}} }} -#line 7100 "Parser.cpp" +#line 7101 "Parser.cpp" break; } case 236: { { -#line 954 "Parser.rl" +#line 955 "Parser.rl" {te = p+1;{ -#line 954 "Parser.rl" +#line 955 "Parser.rl" negated = true; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -7366,20 +7367,20 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 559;goto _again;}} }} -#line 7122 "Parser.cpp" +#line 7123 "Parser.cpp" break; } case 237: { { -#line 960 "Parser.rl" +#line 961 "Parser.rl" {te = p+1;{ -#line 960 "Parser.rl" +#line 961 "Parser.rl" negated = true; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -7389,43 +7390,43 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 818;goto _again;}} }} -#line 7144 "Parser.cpp" +#line 7145 "Parser.cpp" break; } case 238: { { -#line 970 "Parser.rl" +#line 971 "Parser.rl" {te = p+1;{ -#line 970 "Parser.rl" +#line 971 "Parser.rl" currentCls->add(octAccumulator); } }} -#line 7157 "Parser.cpp" +#line 7158 "Parser.cpp" break; } case 239: { { -#line 973 "Parser.rl" +#line 974 "Parser.rl" {te = p+1;{ -#line 973 "Parser.rl" +#line 974 "Parser.rl" currentCls->add(octAccumulator); } }} -#line 7170 "Parser.cpp" +#line 7171 "Parser.cpp" break; } case 240: { { -#line 977 "Parser.rl" +#line 978 "Parser.rl" {te = p+1;{ -#line 977 "Parser.rl" +#line 978 "Parser.rl" string oct(ts + 3, te - ts - 4); unsigned long val; @@ -7441,29 +7442,29 @@ static const short _regex_actions[] = { } }} -#line 7193 "Parser.cpp" +#line 7194 "Parser.cpp" break; } case 241: { { -#line 997 "Parser.rl" +#line 998 "Parser.rl" {te = p+1;{ -#line 997 "Parser.rl" +#line 998 "Parser.rl" currentCls->add(accumulator); } }} -#line 7206 "Parser.cpp" +#line 7207 "Parser.cpp" break; } case 242: { { -#line 1001 "Parser.rl" +#line 1002 "Parser.rl" {te = p+1;{ -#line 1001 "Parser.rl" +#line 1002 "Parser.rl" // whatever we found here currentCls->add(*(ts + 1)); @@ -7471,15 +7472,15 @@ static const short _regex_actions[] = { } }} -#line 7221 "Parser.cpp" +#line 7222 "Parser.cpp" break; } case 243: { { -#line 1007 "Parser.rl" +#line 1008 "Parser.rl" {te = p+1;{ -#line 1007 "Parser.rl" +#line 1008 "Parser.rl" string hex(ts + 3, te - ts - 4); unsigned long val; @@ -7495,15 +7496,15 @@ static const short _regex_actions[] = { } }} -#line 7244 "Parser.cpp" +#line 7245 "Parser.cpp" break; } case 244: { { -#line 1025 "Parser.rl" +#line 1026 "Parser.rl" {te = p+1;{ -#line 1025 "Parser.rl" +#line 1026 "Parser.rl" if (te - ts < 3) { assert(te - ts == 2); @@ -7515,113 +7516,113 @@ static const short _regex_actions[] = { } }} -#line 7263 "Parser.cpp" +#line 7264 "Parser.cpp" break; } case 245: { { -#line 1035 "Parser.rl" +#line 1036 "Parser.rl" {te = p+1;{ -#line 1035 "Parser.rl" +#line 1036 "Parser.rl" currentCls->add(CLASS_WORD, false); } }} -#line 7276 "Parser.cpp" +#line 7277 "Parser.cpp" break; } case 246: { { -#line 1039 "Parser.rl" +#line 1040 "Parser.rl" {te = p+1;{ -#line 1039 "Parser.rl" +#line 1040 "Parser.rl" currentCls->add(CLASS_WORD, true); } }} -#line 7289 "Parser.cpp" +#line 7290 "Parser.cpp" break; } case 247: { { -#line 1043 "Parser.rl" +#line 1044 "Parser.rl" {te = p+1;{ -#line 1043 "Parser.rl" +#line 1044 "Parser.rl" currentCls->add(CLASS_SPACE, false); } }} -#line 7302 "Parser.cpp" +#line 7303 "Parser.cpp" break; } case 248: { { -#line 1047 "Parser.rl" +#line 1048 "Parser.rl" {te = p+1;{ -#line 1047 "Parser.rl" +#line 1048 "Parser.rl" currentCls->add(CLASS_SPACE, true); } }} -#line 7315 "Parser.cpp" +#line 7316 "Parser.cpp" break; } case 249: { { -#line 1051 "Parser.rl" +#line 1052 "Parser.rl" {te = p+1;{ -#line 1051 "Parser.rl" +#line 1052 "Parser.rl" currentCls->add(CLASS_DIGIT, false); } }} -#line 7328 "Parser.cpp" +#line 7329 "Parser.cpp" break; } case 250: { { -#line 1055 "Parser.rl" +#line 1056 "Parser.rl" {te = p+1;{ -#line 1055 "Parser.rl" +#line 1056 "Parser.rl" currentCls->add(CLASS_DIGIT, true); } }} -#line 7341 "Parser.cpp" +#line 7342 "Parser.cpp" break; } case 251: { { -#line 1058 "Parser.rl" +#line 1059 "Parser.rl" {te = p+1;{ -#line 1058 "Parser.rl" +#line 1059 "Parser.rl" currentCls->addDash(); } }} -#line 7354 "Parser.cpp" +#line 7355 "Parser.cpp" break; } case 252: { { -#line 276 "Parser.rl" +#line 277 "Parser.rl" {te = p+1;{ -#line 276 "Parser.rl" +#line 277 "Parser.rl" ostringstream str; str << "'\\" << *(ts + 1) << "' at index " << ts - ptr @@ -7630,15 +7631,15 @@ static const short _regex_actions[] = { } }} -#line 7370 "Parser.cpp" +#line 7371 "Parser.cpp" break; } case 253: { { -#line 276 "Parser.rl" +#line 277 "Parser.rl" {te = p+1;{ -#line 276 "Parser.rl" +#line 277 "Parser.rl" ostringstream str; str << "'\\" << *(ts + 1) << "' at index " << ts - ptr @@ -7647,15 +7648,15 @@ static const short _regex_actions[] = { } }} -#line 7386 "Parser.cpp" +#line 7387 "Parser.cpp" break; } case 254: { { -#line 276 "Parser.rl" +#line 277 "Parser.rl" {te = p+1;{ -#line 276 "Parser.rl" +#line 277 "Parser.rl" ostringstream str; str << "'\\" << *(ts + 1) << "' at index " << ts - ptr @@ -7664,214 +7665,226 @@ static const short _regex_actions[] = { } }} -#line 7402 "Parser.cpp" +#line 7403 "Parser.cpp" break; } case 255: { { -#line 1075 "Parser.rl" +#line 1076 "Parser.rl" {te = p+1;{ -#line 1075 "Parser.rl" +#line 1076 "Parser.rl" // add the literal char currentCls->add(*(ts + 1)); } }} -#line 7416 "Parser.cpp" +#line 7417 "Parser.cpp" break; } case 256: { { -#line 1081 "Parser.rl" +#line 1082 "Parser.rl" {te = p+1;{ -#line 1081 "Parser.rl" +#line 1082 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint2c(ts)); } }} -#line 7430 "Parser.cpp" +#line 7431 "Parser.cpp" break; } case 257: { { -#line 1086 "Parser.rl" +#line 1087 "Parser.rl" {te = p+1;{ -#line 1086 "Parser.rl" +#line 1087 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint3c(ts)); } }} -#line 7444 "Parser.cpp" +#line 7445 "Parser.cpp" break; } case 258: { { -#line 1091 "Parser.rl" +#line 1092 "Parser.rl" {te = p+1;{ -#line 1091 "Parser.rl" +#line 1092 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint4c(ts)); } }} -#line 7458 "Parser.cpp" +#line 7459 "Parser.cpp" break; } case 259: { { -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" {te = p+1;{ -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7472 "Parser.cpp" +#line 7473 "Parser.cpp" break; } case 260: { { -#line 1102 "Parser.rl" +#line 1106 "Parser.rl" + {te = p+1;{ +#line 1106 "Parser.rl" + throw LocatedParseError("Character class intersection/subtraction is not supported"); } + }} + +#line 7484 "Parser.cpp" + + break; + } + case 261: { + { +#line 1109 "Parser.rl" {te = p+1;{ -#line 1102 "Parser.rl" +#line 1109 "Parser.rl" currentCls->add((u8)*ts); } }} -#line 7485 "Parser.cpp" +#line 7497 "Parser.cpp" break; } - case 261: { + case 262: { { -#line 1106 "Parser.rl" +#line 1113 "Parser.rl" {te = p+1;{ -#line 1106 "Parser.rl" +#line 1113 "Parser.rl" currentCls->finalize(); - currentSeq->addComponent(move(currentCls)); + currentSeq->addComponent(std::move(currentCls)); inCharClass = false; {cs = 746;goto _again;}} }} -#line 7500 "Parser.cpp" +#line 7512 "Parser.cpp" break; } - case 262: { + case 263: { { -#line 966 "Parser.rl" +#line 967 "Parser.rl" {te = p;p = p - 1;{ -#line 966 "Parser.rl" +#line 967 "Parser.rl" throw LocatedParseError("Malformed property"); } }} -#line 7511 "Parser.cpp" +#line 7523 "Parser.cpp" break; } - case 263: { + case 264: { { -#line 967 "Parser.rl" +#line 968 "Parser.rl" {te = p;p = p - 1;{ -#line 967 "Parser.rl" +#line 968 "Parser.rl" throw LocatedParseError("Malformed property"); } }} -#line 7522 "Parser.cpp" +#line 7534 "Parser.cpp" break; } - case 264: { + case 265: { { -#line 970 "Parser.rl" +#line 971 "Parser.rl" {te = p;p = p - 1;{ -#line 970 "Parser.rl" +#line 971 "Parser.rl" currentCls->add(octAccumulator); } }} -#line 7535 "Parser.cpp" +#line 7547 "Parser.cpp" break; } - case 265: { + case 266: { { -#line 973 "Parser.rl" +#line 974 "Parser.rl" {te = p;p = p - 1;{ -#line 973 "Parser.rl" +#line 974 "Parser.rl" currentCls->add(octAccumulator); } }} -#line 7548 "Parser.cpp" +#line 7560 "Parser.cpp" break; } - case 266: { + case 267: { { -#line 992 "Parser.rl" +#line 993 "Parser.rl" {te = p;p = p - 1;{ -#line 992 "Parser.rl" +#line 993 "Parser.rl" throw LocatedParseError("Value in \\o{...} sequence is non-octal or missing braces"); } }} -#line 7561 "Parser.cpp" +#line 7573 "Parser.cpp" break; } - case 267: { + case 268: { { -#line 997 "Parser.rl" +#line 998 "Parser.rl" {te = p;p = p - 1;{ -#line 997 "Parser.rl" +#line 998 "Parser.rl" currentCls->add(accumulator); } }} -#line 7574 "Parser.cpp" +#line 7586 "Parser.cpp" break; } - case 268: { + case 269: { { -#line 1021 "Parser.rl" +#line 1022 "Parser.rl" {te = p;p = p - 1;{ -#line 1021 "Parser.rl" +#line 1022 "Parser.rl" throw LocatedParseError("Value in \\x{...} sequence is non-hex or missing }"); } }} -#line 7587 "Parser.cpp" +#line 7599 "Parser.cpp" break; } - case 269: { + case 270: { { -#line 1025 "Parser.rl" +#line 1026 "Parser.rl" {te = p;p = p - 1;{ -#line 1025 "Parser.rl" +#line 1026 "Parser.rl" if (te - ts < 3) { assert(te - ts == 2); @@ -7883,105 +7896,115 @@ static const short _regex_actions[] = { } }} -#line 7606 "Parser.cpp" +#line 7618 "Parser.cpp" break; } - case 270: { + case 271: { { -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" {te = p;p = p - 1;{ -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7620 "Parser.cpp" +#line 7632 "Parser.cpp" break; } - case 271: { + case 272: { + { +#line 1103 "Parser.rl" + {te = p;p = p - 1;{ +#line 1103 "Parser.rl" + throw LocatedParseError("Nested character classes are not supported"); } + }} + +#line 7643 "Parser.cpp" + + break; + } + case 273: { { -#line 1102 "Parser.rl" +#line 1109 "Parser.rl" {te = p;p = p - 1;{ -#line 1102 "Parser.rl" +#line 1109 "Parser.rl" currentCls->add((u8)*ts); } }} -#line 7633 "Parser.cpp" +#line 7656 "Parser.cpp" break; } - case 272: { + case 274: { { -#line 992 "Parser.rl" +#line 993 "Parser.rl" {p = ((te))-1; { -#line 992 "Parser.rl" +#line 993 "Parser.rl" throw LocatedParseError("Value in \\o{...} sequence is non-octal or missing braces"); } }} -#line 7647 "Parser.cpp" +#line 7670 "Parser.cpp" break; } - case 273: { + case 275: { { -#line 1021 "Parser.rl" +#line 1022 "Parser.rl" {p = ((te))-1; { -#line 1021 "Parser.rl" +#line 1022 "Parser.rl" throw LocatedParseError("Value in \\x{...} sequence is non-hex or missing }"); } }} -#line 7661 "Parser.cpp" +#line 7684 "Parser.cpp" break; } - case 274: { + case 276: { { -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" {p = ((te))-1; { -#line 1096 "Parser.rl" +#line 1097 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7676 "Parser.cpp" +#line 7699 "Parser.cpp" break; } - case 275: { + case 277: { { -#line 1102 "Parser.rl" +#line 1103 "Parser.rl" {p = ((te))-1; { -#line 1102 "Parser.rl" - - currentCls->add((u8)*ts); - } +#line 1103 "Parser.rl" + throw LocatedParseError("Nested character classes are not supported"); } }} -#line 7690 "Parser.cpp" +#line 7711 "Parser.cpp" break; } - case 276: { + case 278: { { -#line 1120 "Parser.rl" +#line 1127 "Parser.rl" {te = p+1;{ -#line 1120 "Parser.rl" +#line 1127 "Parser.rl" if (currentCls->isNegated()) { // Already seen a caret; the second one is not a meta-character. @@ -7994,250 +8017,250 @@ static const short _regex_actions[] = { } }} -#line 7710 "Parser.cpp" +#line 7731 "Parser.cpp" break; } - case 277: { + case 279: { { -#line 1133 "Parser.rl" +#line 1140 "Parser.rl" {te = p+1;{ -#line 1133 "Parser.rl" +#line 1140 "Parser.rl" currentCls->add(']'); inCharClassEarly = false; } }} -#line 7724 "Parser.cpp" +#line 7745 "Parser.cpp" break; } - case 278: { + case 280: { { -#line 1138 "Parser.rl" +#line 1145 "Parser.rl" {te = p+1;{ -#line 1138 "Parser.rl" +#line 1145 "Parser.rl" {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { stack.resize(2 * (top + 1)); } } - stack[top] = cs; top += 1;cs = 843;goto _again;}} + stack[top] = cs; top += 1;cs = 844;goto _again;}} }} -#line 7743 "Parser.cpp" +#line 7764 "Parser.cpp" break; } - case 279: { + case 281: { { -#line 1139 "Parser.rl" +#line 1146 "Parser.rl" {te = p+1;{ -#line 1139 "Parser.rl" +#line 1146 "Parser.rl" /*noop*/} }} -#line 7754 "Parser.cpp" +#line 7775 "Parser.cpp" break; } - case 280: { + case 282: { { -#line 1142 "Parser.rl" +#line 1149 "Parser.rl" {te = p+1;{ -#line 1142 "Parser.rl" +#line 1149 "Parser.rl" inCharClassEarly = false; {p = p - 1; } {cs = 819;goto _again;}} }} -#line 7768 "Parser.cpp" +#line 7789 "Parser.cpp" break; } - case 281: { + case 283: { { -#line 1142 "Parser.rl" +#line 1149 "Parser.rl" {te = p;p = p - 1;{ -#line 1142 "Parser.rl" +#line 1149 "Parser.rl" inCharClassEarly = false; {p = p - 1; } {cs = 819;goto _again;}} }} -#line 7782 "Parser.cpp" +#line 7803 "Parser.cpp" break; } - case 282: { + case 284: { { -#line 1154 "Parser.rl" +#line 1161 "Parser.rl" {te = p+1;{ -#line 1154 "Parser.rl" +#line 1161 "Parser.rl" {cs = 746;goto _again;}} }} -#line 7794 "Parser.cpp" +#line 7815 "Parser.cpp" break; } - case 283: { + case 285: { { -#line 1159 "Parser.rl" +#line 1166 "Parser.rl" {te = p+1;{ -#line 1159 "Parser.rl" +#line 1166 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint2c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 7812 "Parser.cpp" +#line 7833 "Parser.cpp" break; } - case 284: { + case 286: { { -#line 1168 "Parser.rl" +#line 1175 "Parser.rl" {te = p+1;{ -#line 1168 "Parser.rl" +#line 1175 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint3c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 7830 "Parser.cpp" +#line 7851 "Parser.cpp" break; } - case 285: { + case 287: { { -#line 1177 "Parser.rl" +#line 1184 "Parser.rl" {te = p+1;{ -#line 1177 "Parser.rl" +#line 1184 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint4c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 7848 "Parser.cpp" +#line 7869 "Parser.cpp" break; } - case 286: { + case 288: { { -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" {te = p+1;{ -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7862 "Parser.cpp" +#line 7883 "Parser.cpp" break; } - case 287: { + case 289: { { -#line 1192 "Parser.rl" +#line 1199 "Parser.rl" {te = p+1;{ -#line 1192 "Parser.rl" +#line 1199 "Parser.rl" addLiteral(currentSeq, *ts, mode); } }} -#line 7875 "Parser.cpp" +#line 7896 "Parser.cpp" break; } - case 288: { + case 290: { { -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" {te = p;p = p - 1;{ -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7889 "Parser.cpp" +#line 7910 "Parser.cpp" break; } - case 289: { + case 291: { { -#line 1192 "Parser.rl" +#line 1199 "Parser.rl" {te = p;p = p - 1;{ -#line 1192 "Parser.rl" +#line 1199 "Parser.rl" addLiteral(currentSeq, *ts, mode); } }} -#line 7902 "Parser.cpp" +#line 7923 "Parser.cpp" break; } - case 290: { + case 292: { { -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" {p = ((te))-1; { -#line 1186 "Parser.rl" +#line 1193 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7917 "Parser.cpp" +#line 7938 "Parser.cpp" break; } - case 291: { + case 293: { { -#line 1202 "Parser.rl" +#line 1209 "Parser.rl" {te = p+1;{ -#line 1202 "Parser.rl" +#line 1209 "Parser.rl" {top -= 1;cs = stack[top];goto _again;} } }} -#line 7930 "Parser.cpp" +#line 7951 "Parser.cpp" break; } - case 292: { + case 294: { { -#line 1207 "Parser.rl" +#line 1214 "Parser.rl" {te = p+1;{ -#line 1207 "Parser.rl" +#line 1214 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint2c(ts)); @@ -8245,15 +8268,15 @@ static const short _regex_actions[] = { } }} -#line 7945 "Parser.cpp" +#line 7966 "Parser.cpp" break; } - case 293: { + case 295: { { -#line 1213 "Parser.rl" +#line 1220 "Parser.rl" {te = p+1;{ -#line 1213 "Parser.rl" +#line 1220 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint3c(ts)); @@ -8261,15 +8284,15 @@ static const short _regex_actions[] = { } }} -#line 7960 "Parser.cpp" +#line 7981 "Parser.cpp" break; } - case 294: { + case 296: { { -#line 1219 "Parser.rl" +#line 1226 "Parser.rl" {te = p+1;{ -#line 1219 "Parser.rl" +#line 1226 "Parser.rl" assert(mode.utf8); currentCls->add(readUtf8CodePoint4c(ts)); @@ -8277,160 +8300,160 @@ static const short _regex_actions[] = { } }} -#line 7975 "Parser.cpp" +#line 7996 "Parser.cpp" break; } - case 295: { + case 297: { { -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" {te = p+1;{ -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 7989 "Parser.cpp" +#line 8010 "Parser.cpp" break; } - case 296: { + case 298: { { -#line 1231 "Parser.rl" +#line 1238 "Parser.rl" {te = p+1;{ -#line 1231 "Parser.rl" +#line 1238 "Parser.rl" currentCls->add(*ts); inCharClassEarly = false; } }} -#line 8003 "Parser.cpp" +#line 8024 "Parser.cpp" break; } - case 297: { + case 299: { { -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" {te = p;p = p - 1;{ -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 8017 "Parser.cpp" +#line 8038 "Parser.cpp" break; } - case 298: { + case 300: { { -#line 1231 "Parser.rl" +#line 1238 "Parser.rl" {te = p;p = p - 1;{ -#line 1231 "Parser.rl" +#line 1238 "Parser.rl" currentCls->add(*ts); inCharClassEarly = false; } }} -#line 8031 "Parser.cpp" +#line 8052 "Parser.cpp" break; } - case 299: { + case 301: { { -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" {p = ((te))-1; { -#line 1225 "Parser.rl" +#line 1232 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 8046 "Parser.cpp" +#line 8067 "Parser.cpp" break; } - case 300: { + case 302: { { -#line 1243 "Parser.rl" +#line 1250 "Parser.rl" {te = p+1;{ -#line 1243 "Parser.rl" +#line 1250 "Parser.rl" inComment = false; {cs = 746;goto _again;}} }} -#line 8057 "Parser.cpp" +#line 8078 "Parser.cpp" break; } - case 301: { + case 303: { { #line 1 "-" {te = p+1;}} -#line 8065 "Parser.cpp" +#line 8086 "Parser.cpp" break; } - case 302: { + case 304: { { -#line 1255 "Parser.rl" +#line 1262 "Parser.rl" {te = p+1;{ -#line 1255 "Parser.rl" +#line 1262 "Parser.rl" inComment = false; {cs = 746;goto _again;}} }} -#line 8076 "Parser.cpp" +#line 8097 "Parser.cpp" break; } - case 303: { + case 305: { { #line 1 "-" {te = p+1;}} -#line 8084 "Parser.cpp" +#line 8105 "Parser.cpp" break; } - case 304: { + case 306: { { -#line 1491 "Parser.rl" - {act = 288;}} +#line 1498 "Parser.rl" + {act = 290;}} -#line 8092 "Parser.cpp" +#line 8113 "Parser.cpp" break; } - case 305: { + case 307: { { -#line 1508 "Parser.rl" - {act = 290;}} +#line 1515 "Parser.rl" + {act = 292;}} -#line 8100 "Parser.cpp" +#line 8121 "Parser.cpp" break; } - case 306: { + case 308: { { -#line 1737 "Parser.rl" - {act = 330;}} +#line 1744 "Parser.rl" + {act = 332;}} -#line 8108 "Parser.cpp" +#line 8129 "Parser.cpp" break; } - case 307: { + case 309: { { -#line 362 "Parser.rl" +#line 363 "Parser.rl" {te = p+1;{ -#line 362 "Parser.rl" +#line 363 "Parser.rl" if (sequences.empty()) { throw LocatedParseError("Unmatched parentheses"); @@ -8440,98 +8463,98 @@ static const short _regex_actions[] = { } }} -#line 8125 "Parser.cpp" +#line 8146 "Parser.cpp" break; } - case 308: { + case 310: { { -#line 1274 "Parser.rl" +#line 1281 "Parser.rl" {te = p+1;{ -#line 1274 "Parser.rl" +#line 1281 "Parser.rl" currentSeq->addAlternation(); } }} -#line 8138 "Parser.cpp" +#line 8159 "Parser.cpp" break; } - case 309: { + case 311: { { -#line 1279 "Parser.rl" +#line 1286 "Parser.rl" {te = p+1;{ -#line 1279 "Parser.rl" +#line 1286 "Parser.rl" throw LocatedParseError("POSIX named classes are only " "supported inside a class"); } }} -#line 8152 "Parser.cpp" +#line 8173 "Parser.cpp" break; } - case 310: { + case 312: { { -#line 1286 "Parser.rl" +#line 1293 "Parser.rl" {te = p+1;{ -#line 1286 "Parser.rl" +#line 1293 "Parser.rl" throw LocatedParseError("Unsupported POSIX collating " "element"); } }} -#line 8166 "Parser.cpp" +#line 8187 "Parser.cpp" break; } - case 311: { + case 313: { { -#line 1293 "Parser.rl" +#line 1300 "Parser.rl" {te = p+1;{ -#line 1293 "Parser.rl" +#line 1300 "Parser.rl" - {cs = 838;goto _again;}} + {cs = 839;goto _again;}} }} -#line 8178 "Parser.cpp" +#line 8199 "Parser.cpp" break; } - case 312: { + case 314: { { -#line 1297 "Parser.rl" +#line 1304 "Parser.rl" {te = p+1;{ -#line 1297 "Parser.rl" +#line 1304 "Parser.rl" /* noop */ } }} -#line 8189 "Parser.cpp" +#line 8210 "Parser.cpp" break; } - case 313: { + case 315: { { -#line 1299 "Parser.rl" +#line 1306 "Parser.rl" {te = p+1;{ -#line 1299 "Parser.rl" +#line 1306 "Parser.rl" currentSeq->addComponent(generateComponent(CLASS_ANY, false, mode)); } }} -#line 8202 "Parser.cpp" +#line 8223 "Parser.cpp" break; } - case 314: { + case 316: { { -#line 1303 "Parser.rl" +#line 1310 "Parser.rl" {te = p+1;{ -#line 1303 "Parser.rl" +#line 1310 "Parser.rl" if (mode.utf8) { throw LocatedParseError("\\C is unsupported in UTF8"); @@ -8540,15 +8563,15 @@ static const short _regex_actions[] = { } }} -#line 8218 "Parser.cpp" +#line 8239 "Parser.cpp" break; } - case 315: { + case 317: { { -#line 1317 "Parser.rl" +#line 1324 "Parser.rl" {te = p+1;{ -#line 1317 "Parser.rl" +#line 1324 "Parser.rl" if (!currentSeq->addRepeat(0, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_NONGREEDY)) { @@ -8557,15 +8580,15 @@ static const short _regex_actions[] = { } }} -#line 8234 "Parser.cpp" +#line 8255 "Parser.cpp" break; } - case 316: { + case 318: { { -#line 1324 "Parser.rl" +#line 1331 "Parser.rl" {te = p+1;{ -#line 1324 "Parser.rl" +#line 1331 "Parser.rl" if (!currentSeq->addRepeat(0, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_POSSESSIVE)) { @@ -8574,15 +8597,15 @@ static const short _regex_actions[] = { } }} -#line 8250 "Parser.cpp" +#line 8271 "Parser.cpp" break; } - case 317: { + case 319: { { -#line 1338 "Parser.rl" +#line 1345 "Parser.rl" {te = p+1;{ -#line 1338 "Parser.rl" +#line 1345 "Parser.rl" if (!currentSeq->addRepeat(1, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_NONGREEDY)) { @@ -8591,15 +8614,15 @@ static const short _regex_actions[] = { } }} -#line 8266 "Parser.cpp" +#line 8287 "Parser.cpp" break; } - case 318: { + case 320: { { -#line 1345 "Parser.rl" +#line 1352 "Parser.rl" {te = p+1;{ -#line 1345 "Parser.rl" +#line 1352 "Parser.rl" if (!currentSeq->addRepeat(1, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_POSSESSIVE)) { @@ -8608,15 +8631,15 @@ static const short _regex_actions[] = { } }} -#line 8282 "Parser.cpp" +#line 8303 "Parser.cpp" break; } - case 319: { + case 321: { { -#line 1359 "Parser.rl" +#line 1366 "Parser.rl" {te = p+1;{ -#line 1359 "Parser.rl" +#line 1366 "Parser.rl" if (!currentSeq->addRepeat( 0, 1, ComponentRepeat::REPEAT_NONGREEDY)) { @@ -8625,15 +8648,15 @@ static const short _regex_actions[] = { } }} -#line 8298 "Parser.cpp" +#line 8319 "Parser.cpp" break; } - case 320: { + case 322: { { -#line 1366 "Parser.rl" +#line 1373 "Parser.rl" {te = p+1;{ -#line 1366 "Parser.rl" +#line 1373 "Parser.rl" if (!currentSeq->addRepeat( 0, 1, ComponentRepeat::REPEAT_POSSESSIVE)) { @@ -8642,15 +8665,15 @@ static const short _regex_actions[] = { } }} -#line 8314 "Parser.cpp" +#line 8335 "Parser.cpp" break; } - case 321: { + case 323: { { -#line 1383 "Parser.rl" +#line 1390 "Parser.rl" {te = p+1;{ -#line 1383 "Parser.rl" +#line 1390 "Parser.rl" if (repeatN > repeatM || repeatM == 0) { throwInvalidRepeat(); @@ -8662,15 +8685,15 @@ static const short _regex_actions[] = { } }} -#line 8333 "Parser.cpp" +#line 8354 "Parser.cpp" break; } - case 322: { + case 324: { { -#line 1393 "Parser.rl" +#line 1400 "Parser.rl" {te = p+1;{ -#line 1393 "Parser.rl" +#line 1400 "Parser.rl" if (repeatN > repeatM || repeatM == 0) { throwInvalidRepeat(); @@ -8682,31 +8705,31 @@ static const short _regex_actions[] = { } }} -#line 8352 "Parser.cpp" +#line 8373 "Parser.cpp" break; } - case 323: { + case 325: { { -#line 322 "Parser.rl" +#line 323 "Parser.rl" {te = p+1;{ -#line 322 "Parser.rl" +#line 323 "Parser.rl" inComment = true; - {cs = 849;goto _again;}} + {cs = 850;goto _again;}} }} -#line 8365 "Parser.cpp" +#line 8386 "Parser.cpp" break; } - case 324: { + case 326: { { -#line 1410 "Parser.rl" +#line 1417 "Parser.rl" {te = p+1;{ -#line 1410 "Parser.rl" +#line 1417 "Parser.rl" {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -8716,27 +8739,27 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 787;goto _again;}} }} -#line 8384 "Parser.cpp" +#line 8405 "Parser.cpp" break; } - case 325: { + case 327: { { -#line 1414 "Parser.rl" +#line 1421 "Parser.rl" {te = p+1;{ -#line 1414 "Parser.rl" +#line 1421 "Parser.rl" assert(0); {p += 1; goto _out; } } }} -#line 8395 "Parser.cpp" +#line 8416 "Parser.cpp" break; } - case 326: { + case 328: { { -#line 1421 "Parser.rl" +#line 1428 "Parser.rl" {te = p+1;{ -#line 1421 "Parser.rl" +#line 1428 "Parser.rl" auto bound = mode.multiline ? ComponentBoundary::BEGIN_LINE : ComponentBoundary::BEGIN_STRING; @@ -8744,15 +8767,15 @@ static const short _regex_actions[] = { } }} -#line 8410 "Parser.cpp" +#line 8431 "Parser.cpp" break; } - case 327: { + case 329: { { -#line 1428 "Parser.rl" +#line 1435 "Parser.rl" {te = p+1;{ -#line 1428 "Parser.rl" +#line 1435 "Parser.rl" auto bound = mode.multiline ? ComponentBoundary::END_LINE : ComponentBoundary::END_STRING_OPTIONAL_LF; @@ -8760,188 +8783,188 @@ static const short _regex_actions[] = { } }} -#line 8425 "Parser.cpp" +#line 8446 "Parser.cpp" break; } - case 328: { + case 330: { { -#line 1434 "Parser.rl" +#line 1441 "Parser.rl" {te = p+1;{ -#line 1434 "Parser.rl" +#line 1441 "Parser.rl" auto bound = ComponentBoundary::BEGIN_STRING; currentSeq->addComponent(std::make_unique(bound)); } }} -#line 8439 "Parser.cpp" +#line 8460 "Parser.cpp" break; } - case 329: { + case 331: { { -#line 1439 "Parser.rl" +#line 1446 "Parser.rl" {te = p+1;{ -#line 1439 "Parser.rl" +#line 1446 "Parser.rl" auto bound = ComponentBoundary::END_STRING_OPTIONAL_LF; currentSeq->addComponent(std::make_unique(bound)); } }} -#line 8453 "Parser.cpp" +#line 8474 "Parser.cpp" break; } - case 330: { + case 332: { { -#line 1444 "Parser.rl" +#line 1451 "Parser.rl" {te = p+1;{ -#line 1444 "Parser.rl" +#line 1451 "Parser.rl" auto bound = ComponentBoundary::END_STRING; currentSeq->addComponent(std::make_unique(bound)); } }} -#line 8467 "Parser.cpp" +#line 8488 "Parser.cpp" break; } - case 331: { + case 333: { { -#line 1449 "Parser.rl" +#line 1456 "Parser.rl" {te = p+1;{ -#line 1449 "Parser.rl" +#line 1456 "Parser.rl" currentSeq->addComponent( std::make_unique(ts - ptr, false, mode)); } }} -#line 8481 "Parser.cpp" +#line 8502 "Parser.cpp" break; } - case 332: { + case 334: { { -#line 1454 "Parser.rl" +#line 1461 "Parser.rl" {te = p+1;{ -#line 1454 "Parser.rl" +#line 1461 "Parser.rl" currentSeq->addComponent( std::make_unique(ts - ptr, true, mode)); } }} -#line 8495 "Parser.cpp" +#line 8516 "Parser.cpp" break; } - case 333: { + case 335: { { -#line 1464 "Parser.rl" +#line 1471 "Parser.rl" {te = p+1;{ -#line 1464 "Parser.rl" +#line 1471 "Parser.rl" addLiteral(currentSeq, '\x09', mode); } }} -#line 8508 "Parser.cpp" +#line 8529 "Parser.cpp" break; } - case 334: { + case 336: { { -#line 1468 "Parser.rl" +#line 1475 "Parser.rl" {te = p+1;{ -#line 1468 "Parser.rl" +#line 1475 "Parser.rl" addLiteral(currentSeq, '\x0a', mode); } }} -#line 8521 "Parser.cpp" +#line 8542 "Parser.cpp" break; } - case 335: { + case 337: { { -#line 1472 "Parser.rl" +#line 1479 "Parser.rl" {te = p+1;{ -#line 1472 "Parser.rl" +#line 1479 "Parser.rl" addLiteral(currentSeq, '\x0d', mode); } }} -#line 8534 "Parser.cpp" +#line 8555 "Parser.cpp" break; } - case 336: { + case 338: { { -#line 1476 "Parser.rl" +#line 1483 "Parser.rl" {te = p+1;{ -#line 1476 "Parser.rl" +#line 1483 "Parser.rl" addLiteral(currentSeq, '\x0c', mode); } }} -#line 8547 "Parser.cpp" +#line 8568 "Parser.cpp" break; } - case 337: { + case 339: { { -#line 1480 "Parser.rl" +#line 1487 "Parser.rl" {te = p+1;{ -#line 1480 "Parser.rl" +#line 1487 "Parser.rl" addLiteral(currentSeq, '\x07', mode); } }} -#line 8560 "Parser.cpp" +#line 8581 "Parser.cpp" break; } - case 338: { + case 340: { { -#line 1484 "Parser.rl" +#line 1491 "Parser.rl" {te = p+1;{ -#line 1484 "Parser.rl" +#line 1491 "Parser.rl" addLiteral(currentSeq, '\x1b', mode); } }} -#line 8573 "Parser.cpp" +#line 8594 "Parser.cpp" break; } - case 339: { + case 341: { { -#line 1488 "Parser.rl" +#line 1495 "Parser.rl" {te = p+1;{ -#line 1488 "Parser.rl" +#line 1495 "Parser.rl" addLiteral(currentSeq, octAccumulator, mode); } }} -#line 8586 "Parser.cpp" +#line 8607 "Parser.cpp" break; } - case 340: { + case 342: { { -#line 479 "Parser.rl" +#line 480 "Parser.rl" {te = p+1;{ -#line 479 "Parser.rl" +#line 480 "Parser.rl" if (accumulator == 0) { throw LocatedParseError("Numbered reference cannot be zero"); @@ -8950,15 +8973,15 @@ static const short _regex_actions[] = { } }} -#line 8602 "Parser.cpp" +#line 8623 "Parser.cpp" break; } - case 341: { + case 343: { { -#line 486 "Parser.rl" +#line 487 "Parser.rl" {te = p+1;{ -#line 486 "Parser.rl" +#line 487 "Parser.rl" // Accumulator is a negative offset. if (accumulator == 0) { @@ -8972,15 +8995,15 @@ static const short _regex_actions[] = { } }} -#line 8623 "Parser.cpp" +#line 8644 "Parser.cpp" break; } - case 342: { + case 344: { { -#line 479 "Parser.rl" +#line 480 "Parser.rl" {te = p+1;{ -#line 479 "Parser.rl" +#line 480 "Parser.rl" if (accumulator == 0) { throw LocatedParseError("Numbered reference cannot be zero"); @@ -8989,15 +9012,15 @@ static const short _regex_actions[] = { } }} -#line 8639 "Parser.cpp" +#line 8660 "Parser.cpp" break; } - case 343: { + case 345: { { -#line 486 "Parser.rl" +#line 487 "Parser.rl" {te = p+1;{ -#line 486 "Parser.rl" +#line 487 "Parser.rl" // Accumulator is a negative offset. if (accumulator == 0) { @@ -9011,85 +9034,85 @@ static const short _regex_actions[] = { } }} -#line 8660 "Parser.cpp" +#line 8681 "Parser.cpp" break; } - case 344: { + case 346: { { -#line 498 "Parser.rl" +#line 499 "Parser.rl" {te = p+1;{ -#line 498 "Parser.rl" +#line 499 "Parser.rl" currentSeq->addComponent(std::make_unique(label)); } }} -#line 8673 "Parser.cpp" +#line 8694 "Parser.cpp" break; } - case 345: { + case 347: { { -#line 498 "Parser.rl" +#line 499 "Parser.rl" {te = p+1;{ -#line 498 "Parser.rl" +#line 499 "Parser.rl" currentSeq->addComponent(std::make_unique(label)); } }} -#line 8686 "Parser.cpp" +#line 8707 "Parser.cpp" break; } - case 346: { + case 348: { { -#line 498 "Parser.rl" +#line 499 "Parser.rl" {te = p+1;{ -#line 498 "Parser.rl" +#line 499 "Parser.rl" currentSeq->addComponent(std::make_unique(label)); } }} -#line 8699 "Parser.cpp" +#line 8720 "Parser.cpp" break; } - case 347: { + case 349: { { -#line 498 "Parser.rl" +#line 499 "Parser.rl" {te = p+1;{ -#line 498 "Parser.rl" +#line 499 "Parser.rl" currentSeq->addComponent(std::make_unique(label)); } }} -#line 8712 "Parser.cpp" +#line 8733 "Parser.cpp" break; } - case 348: { + case 350: { { -#line 498 "Parser.rl" +#line 499 "Parser.rl" {te = p+1;{ -#line 498 "Parser.rl" +#line 499 "Parser.rl" currentSeq->addComponent(std::make_unique(label)); } }} -#line 8725 "Parser.cpp" +#line 8746 "Parser.cpp" break; } - case 349: { + case 351: { { -#line 1549 "Parser.rl" +#line 1556 "Parser.rl" {te = p+1;{ -#line 1549 "Parser.rl" +#line 1556 "Parser.rl" ostringstream str; str << "Onigiruma subroutine call at index " << ts - ptr << @@ -9098,15 +9121,15 @@ static const short _regex_actions[] = { } }} -#line 8741 "Parser.cpp" +#line 8762 "Parser.cpp" break; } - case 350: { + case 352: { { -#line 1560 "Parser.rl" +#line 1567 "Parser.rl" {te = p+1;{ -#line 1560 "Parser.rl" +#line 1567 "Parser.rl" string oct(ts + 3, te - ts - 4); unsigned long val; @@ -9122,29 +9145,29 @@ static const short _regex_actions[] = { } }} -#line 8764 "Parser.cpp" +#line 8785 "Parser.cpp" break; } - case 351: { + case 353: { { -#line 1578 "Parser.rl" +#line 1585 "Parser.rl" {te = p+1;{ -#line 1578 "Parser.rl" +#line 1585 "Parser.rl" addEscapedHex(currentSeq, accumulator, mode); } }} -#line 8777 "Parser.cpp" +#line 8798 "Parser.cpp" break; } - case 352: { + case 354: { { -#line 1582 "Parser.rl" +#line 1589 "Parser.rl" {te = p+1;{ -#line 1582 "Parser.rl" +#line 1589 "Parser.rl" string hex(ts + 3, te - ts - 4); unsigned long val; @@ -9160,15 +9183,15 @@ static const short _regex_actions[] = { } }} -#line 8800 "Parser.cpp" +#line 8821 "Parser.cpp" break; } - case 353: { + case 355: { { -#line 1600 "Parser.rl" +#line 1607 "Parser.rl" {te = p+1;{ -#line 1600 "Parser.rl" +#line 1607 "Parser.rl" if (te - ts < 3) { assert(te - ts == 2); @@ -9180,15 +9203,15 @@ static const short _regex_actions[] = { } }} -#line 8819 "Parser.cpp" +#line 8840 "Parser.cpp" break; } - case 354: { + case 356: { { -#line 1610 "Parser.rl" +#line 1617 "Parser.rl" {te = p+1;{ -#line 1610 "Parser.rl" +#line 1617 "Parser.rl" ostringstream str; str << "'\\" << *(ts + 1) << "' at index " << ts - ptr @@ -9197,172 +9220,172 @@ static const short _regex_actions[] = { } }} -#line 8835 "Parser.cpp" +#line 8856 "Parser.cpp" break; } - case 355: { + case 357: { { -#line 1618 "Parser.rl" +#line 1625 "Parser.rl" {te = p+1;{ -#line 1618 "Parser.rl" +#line 1625 "Parser.rl" auto cc = generateComponent(CLASS_WORD, false, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8849 "Parser.cpp" +#line 8870 "Parser.cpp" break; } - case 356: { + case 358: { { -#line 1623 "Parser.rl" +#line 1630 "Parser.rl" {te = p+1;{ -#line 1623 "Parser.rl" +#line 1630 "Parser.rl" auto cc = generateComponent(CLASS_WORD, true, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8863 "Parser.cpp" +#line 8884 "Parser.cpp" break; } - case 357: { + case 359: { { -#line 1628 "Parser.rl" +#line 1635 "Parser.rl" {te = p+1;{ -#line 1628 "Parser.rl" +#line 1635 "Parser.rl" auto cc = generateComponent(CLASS_SPACE, false, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8877 "Parser.cpp" +#line 8898 "Parser.cpp" break; } - case 358: { + case 360: { { -#line 1633 "Parser.rl" +#line 1640 "Parser.rl" {te = p+1;{ -#line 1633 "Parser.rl" +#line 1640 "Parser.rl" auto cc = generateComponent(CLASS_SPACE, true, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8891 "Parser.cpp" +#line 8912 "Parser.cpp" break; } - case 359: { + case 361: { { -#line 1638 "Parser.rl" +#line 1645 "Parser.rl" {te = p+1;{ -#line 1638 "Parser.rl" +#line 1645 "Parser.rl" auto cc = generateComponent(CLASS_DIGIT, false, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8905 "Parser.cpp" +#line 8926 "Parser.cpp" break; } - case 360: { + case 362: { { -#line 1643 "Parser.rl" +#line 1650 "Parser.rl" {te = p+1;{ -#line 1643 "Parser.rl" +#line 1650 "Parser.rl" auto cc = generateComponent(CLASS_DIGIT, true, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8919 "Parser.cpp" +#line 8940 "Parser.cpp" break; } - case 361: { + case 363: { { -#line 1648 "Parser.rl" +#line 1655 "Parser.rl" {te = p+1;{ -#line 1648 "Parser.rl" +#line 1655 "Parser.rl" auto cc = generateComponent(CLASS_HORZ, false, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8933 "Parser.cpp" +#line 8954 "Parser.cpp" break; } - case 362: { + case 364: { { -#line 1653 "Parser.rl" +#line 1660 "Parser.rl" {te = p+1;{ -#line 1653 "Parser.rl" +#line 1660 "Parser.rl" auto cc = generateComponent(CLASS_HORZ, true, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8947 "Parser.cpp" +#line 8968 "Parser.cpp" break; } - case 363: { + case 365: { { -#line 1658 "Parser.rl" +#line 1665 "Parser.rl" {te = p+1;{ -#line 1658 "Parser.rl" +#line 1665 "Parser.rl" auto cc = generateComponent(CLASS_VERT, false, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8961 "Parser.cpp" +#line 8982 "Parser.cpp" break; } - case 364: { + case 366: { { -#line 1663 "Parser.rl" +#line 1670 "Parser.rl" {te = p+1;{ -#line 1663 "Parser.rl" +#line 1670 "Parser.rl" auto cc = generateComponent(CLASS_VERT, true, mode); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 8975 "Parser.cpp" +#line 8996 "Parser.cpp" break; } - case 365: { + case 367: { { -#line 1668 "Parser.rl" +#line 1675 "Parser.rl" {te = p+1;{ -#line 1668 "Parser.rl" +#line 1675 "Parser.rl" assert(!currentCls && !inCharClass); currentCls = getComponentClass(mode); negated = false; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -9372,22 +9395,22 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 559;goto _again;}} }} -#line 8999 "Parser.cpp" +#line 9020 "Parser.cpp" break; } - case 366: { + case 368: { { -#line 1676 "Parser.rl" +#line 1683 "Parser.rl" {te = p+1;{ -#line 1676 "Parser.rl" +#line 1683 "Parser.rl" assert(!currentCls && !inCharClass); currentCls = getComponentClass(mode); negated = false; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -9397,22 +9420,22 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 818;goto _again;}} }} -#line 9023 "Parser.cpp" +#line 9044 "Parser.cpp" break; } - case 367: { + case 369: { { -#line 1684 "Parser.rl" +#line 1691 "Parser.rl" {te = p+1;{ -#line 1684 "Parser.rl" +#line 1691 "Parser.rl" assert(!currentCls && !inCharClass); currentCls = getComponentClass(mode); negated = true; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -9422,22 +9445,22 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 559;goto _again;}} }} -#line 9047 "Parser.cpp" +#line 9068 "Parser.cpp" break; } - case 368: { + case 370: { { -#line 1692 "Parser.rl" +#line 1699 "Parser.rl" {te = p+1;{ -#line 1692 "Parser.rl" +#line 1699 "Parser.rl" assert(!currentCls && !inCharClass); currentCls = getComponentClass(mode); negated = true; {p = p - 1; } {{ -#line 1903 "Parser.rl" +#line 1910 "Parser.rl" DEBUG_PRINTF("stack %zu top %d\n", stack.size(), top); if ((int)stack.size() == top) { @@ -9447,15 +9470,15 @@ static const short _regex_actions[] = { stack[top] = cs; top += 1;cs = 818;goto _again;}} }} -#line 9071 "Parser.cpp" +#line 9092 "Parser.cpp" break; } - case 369: { + case 371: { { -#line 1704 "Parser.rl" +#line 1711 "Parser.rl" {te = p+1;{ -#line 1704 "Parser.rl" +#line 1711 "Parser.rl" ostringstream str; str << "\\R at index " << ts - ptr << " not supported."; @@ -9463,15 +9486,15 @@ static const short _regex_actions[] = { } }} -#line 9086 "Parser.cpp" +#line 9107 "Parser.cpp" break; } - case 370: { + case 372: { { -#line 1711 "Parser.rl" +#line 1718 "Parser.rl" {te = p+1;{ -#line 1711 "Parser.rl" +#line 1718 "Parser.rl" ostringstream str; str << "\\K at index " << ts - ptr << " not supported."; @@ -9479,15 +9502,15 @@ static const short _regex_actions[] = { } }} -#line 9101 "Parser.cpp" +#line 9122 "Parser.cpp" break; } - case 371: { + case 373: { { -#line 1726 "Parser.rl" +#line 1733 "Parser.rl" {te = p+1;{ -#line 1726 "Parser.rl" +#line 1733 "Parser.rl" ostringstream str; str << "\\G at index " << ts - ptr << " not supported."; @@ -9495,72 +9518,72 @@ static const short _regex_actions[] = { } }} -#line 9116 "Parser.cpp" +#line 9137 "Parser.cpp" break; } - case 372: { + case 374: { { -#line 1732 "Parser.rl" +#line 1739 "Parser.rl" {te = p+1;{ -#line 1732 "Parser.rl" +#line 1739 "Parser.rl" currentSeq->addComponent(std::make_unique(ts - ptr, mode)); } }} -#line 9129 "Parser.cpp" +#line 9150 "Parser.cpp" break; } - case 373: { + case 375: { { -#line 1737 "Parser.rl" +#line 1744 "Parser.rl" {te = p+1;{ -#line 1737 "Parser.rl" +#line 1744 "Parser.rl" addLiteral(currentSeq, *(ts + 1), mode); } }} -#line 9142 "Parser.cpp" +#line 9163 "Parser.cpp" break; } - case 374: { + case 376: { { -#line 316 "Parser.rl" +#line 317 "Parser.rl" {te = p+1;{ -#line 316 "Parser.rl" +#line 317 "Parser.rl" inComment = true; - {cs = 848;goto _again;}} + {cs = 849;goto _again;}} }} -#line 9155 "Parser.cpp" +#line 9176 "Parser.cpp" break; } - case 375: { + case 377: { { -#line 433 "Parser.rl" +#line 434 "Parser.rl" {te = p+1;{ -#line 433 "Parser.rl" +#line 434 "Parser.rl" mode = newMode; currentSeq->addComponent(std::make_unique()); } }} -#line 9169 "Parser.cpp" +#line 9190 "Parser.cpp" break; } - case 376: { + case 378: { { -#line 355 "Parser.rl" +#line 356 "Parser.rl" {te = p+1;{ -#line 355 "Parser.rl" +#line 356 "Parser.rl" PUSH_SEQUENCE; mode = newMode; @@ -9569,15 +9592,15 @@ static const short _regex_actions[] = { } }} -#line 9185 "Parser.cpp" +#line 9206 "Parser.cpp" break; } - case 377: { + case 379: { { -#line 369 "Parser.rl" +#line 370 "Parser.rl" {te = p+1;{ -#line 369 "Parser.rl" +#line 370 "Parser.rl" PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, @@ -9586,15 +9609,15 @@ static const short _regex_actions[] = { } }} -#line 9201 "Parser.cpp" +#line 9222 "Parser.cpp" break; } - case 378: { + case 380: { { -#line 375 "Parser.rl" +#line 376 "Parser.rl" {te = p+1;{ -#line 375 "Parser.rl" +#line 376 "Parser.rl" PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, @@ -9603,15 +9626,15 @@ static const short _regex_actions[] = { } }} -#line 9217 "Parser.cpp" +#line 9238 "Parser.cpp" break; } - case 379: { + case 381: { { -#line 381 "Parser.rl" +#line 382 "Parser.rl" {te = p+1;{ -#line 381 "Parser.rl" +#line 382 "Parser.rl" PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, @@ -9620,15 +9643,15 @@ static const short _regex_actions[] = { } }} -#line 9233 "Parser.cpp" +#line 9254 "Parser.cpp" break; } - case 380: { + case 382: { { -#line 387 "Parser.rl" +#line 388 "Parser.rl" {te = p+1;{ -#line 387 "Parser.rl" +#line 388 "Parser.rl" PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, @@ -9637,43 +9660,43 @@ static const short _regex_actions[] = { } }} -#line 9249 "Parser.cpp" +#line 9270 "Parser.cpp" break; } - case 381: { + case 383: { { -#line 393 "Parser.rl" +#line 394 "Parser.rl" {te = p+1;{ -#line 393 "Parser.rl" +#line 394 "Parser.rl" throw LocatedParseError("Embedded code is not supported"); } }} -#line 9262 "Parser.cpp" +#line 9283 "Parser.cpp" break; } - case 382: { + case 384: { { -#line 393 "Parser.rl" +#line 394 "Parser.rl" {te = p+1;{ -#line 393 "Parser.rl" +#line 394 "Parser.rl" throw LocatedParseError("Embedded code is not supported"); } }} -#line 9275 "Parser.cpp" +#line 9296 "Parser.cpp" break; } - case 383: { + case 385: { { -#line 416 "Parser.rl" +#line 417 "Parser.rl" {te = p+1;{ -#line 416 "Parser.rl" +#line 417 "Parser.rl" PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, @@ -9681,15 +9704,15 @@ static const short _regex_actions[] = { } }} -#line 9290 "Parser.cpp" +#line 9311 "Parser.cpp" break; } - case 384: { + case 386: { { -#line 336 "Parser.rl" +#line 337 "Parser.rl" {te = p+1;{ -#line 336 "Parser.rl" +#line 337 "Parser.rl" assert(!label.empty()); // should be guaranteed by machine char c = *label.begin(); @@ -9703,145 +9726,145 @@ static const short _regex_actions[] = { auto seq = std::make_unique(); seq->setCaptureIndex(groupIndex++); seq->setCaptureName(label); - currentSeq = enterSequence(currentSeq, move(seq)); + currentSeq = enterSequence(currentSeq, std::move(seq)); } }} -#line 9315 "Parser.cpp" +#line 9336 "Parser.cpp" break; } - case 385: { + case 387: { { -#line 399 "Parser.rl" +#line 400 "Parser.rl" {te = p+1;{ -#line 399 "Parser.rl" +#line 400 "Parser.rl" throw LocatedParseError("Subpattern reference unsupported"); } }} -#line 9328 "Parser.cpp" +#line 9349 "Parser.cpp" break; } - case 386: { + case 388: { { -#line 399 "Parser.rl" +#line 400 "Parser.rl" {te = p+1;{ -#line 399 "Parser.rl" +#line 400 "Parser.rl" throw LocatedParseError("Subpattern reference unsupported"); } }} -#line 9341 "Parser.cpp" +#line 9362 "Parser.cpp" break; } - case 387: { + case 389: { { -#line 1783 "Parser.rl" +#line 1790 "Parser.rl" {te = p+1;{ -#line 1783 "Parser.rl" +#line 1790 "Parser.rl" auto a = std::make_unique( ComponentAssertion::LOOKAHEAD, ComponentAssertion::POS); ComponentAssertion *a_seq = a.get(); PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, - std::make_unique(move(a))); + std::make_unique(std::move(a))); PUSH_SEQUENCE; currentSeq = a_seq; } }} -#line 9361 "Parser.cpp" +#line 9382 "Parser.cpp" break; } - case 388: { + case 390: { { -#line 1794 "Parser.rl" +#line 1801 "Parser.rl" {te = p+1;{ -#line 1794 "Parser.rl" +#line 1801 "Parser.rl" auto a = std::make_unique( ComponentAssertion::LOOKAHEAD, ComponentAssertion::NEG); ComponentAssertion *a_seq = a.get(); PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, - std::make_unique(move(a))); + std::make_unique(std::move(a))); PUSH_SEQUENCE; currentSeq = a_seq; } }} -#line 9381 "Parser.cpp" +#line 9402 "Parser.cpp" break; } - case 389: { + case 391: { { -#line 1805 "Parser.rl" +#line 1812 "Parser.rl" {te = p+1;{ -#line 1805 "Parser.rl" +#line 1812 "Parser.rl" auto a = std::make_unique( ComponentAssertion::LOOKBEHIND, ComponentAssertion::POS); ComponentAssertion *a_seq = a.get(); PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, - std::make_unique(move(a))); + std::make_unique(std::move(a))); PUSH_SEQUENCE; currentSeq = a_seq; } }} -#line 9401 "Parser.cpp" +#line 9422 "Parser.cpp" break; } - case 390: { + case 392: { { -#line 1816 "Parser.rl" +#line 1823 "Parser.rl" {te = p+1;{ -#line 1816 "Parser.rl" +#line 1823 "Parser.rl" auto a = std::make_unique( ComponentAssertion::LOOKBEHIND, ComponentAssertion::NEG); ComponentAssertion *a_seq = a.get(); PUSH_SEQUENCE; currentSeq = enterSequence(currentSeq, - std::make_unique(move(a))); + std::make_unique(std::move(a))); PUSH_SEQUENCE; currentSeq = a_seq; } }} -#line 9421 "Parser.cpp" +#line 9442 "Parser.cpp" break; } - case 391: { + case 393: { { -#line 1828 "Parser.rl" +#line 1835 "Parser.rl" {te = p+1;{ -#line 1828 "Parser.rl" +#line 1835 "Parser.rl" throw LocatedParseError("Pattern recursion not supported"); } }} -#line 9434 "Parser.cpp" +#line 9455 "Parser.cpp" break; } - case 392: { + case 394: { { -#line 402 "Parser.rl" +#line 403 "Parser.rl" {te = p+1;{ -#line 402 "Parser.rl" +#line 403 "Parser.rl" if (accumulator == 0) { throw LocatedParseError("Numbered reference cannot be zero"); @@ -9852,15 +9875,15 @@ static const short _regex_actions[] = { } }} -#line 9452 "Parser.cpp" +#line 9473 "Parser.cpp" break; } - case 393: { + case 395: { { -#line 410 "Parser.rl" +#line 411 "Parser.rl" {te = p+1;{ -#line 410 "Parser.rl" +#line 411 "Parser.rl" PUSH_SEQUENCE; assert(!label.empty()); @@ -9869,15 +9892,15 @@ static const short _regex_actions[] = { } }} -#line 9468 "Parser.cpp" +#line 9489 "Parser.cpp" break; } - case 394: { + case 396: { { -#line 1844 "Parser.rl" +#line 1851 "Parser.rl" {te = p+1;{ -#line 1844 "Parser.rl" +#line 1851 "Parser.rl" ostringstream str; str << "Callout at index " << ts - ptr << " not supported."; @@ -9885,101 +9908,101 @@ static const short _regex_actions[] = { } }} -#line 9483 "Parser.cpp" +#line 9504 "Parser.cpp" break; } - case 395: { + case 397: { { -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" {te = p+1;{ -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" throw LocatedParseError("Unrecognised character after (?"); } }} -#line 9496 "Parser.cpp" +#line 9517 "Parser.cpp" break; } - case 396: { + case 398: { { -#line 1857 "Parser.rl" +#line 1864 "Parser.rl" {te = p+1;{ -#line 1857 "Parser.rl" +#line 1864 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint2c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 9514 "Parser.cpp" +#line 9535 "Parser.cpp" break; } - case 397: { + case 399: { { -#line 1866 "Parser.rl" +#line 1873 "Parser.rl" {te = p+1;{ -#line 1866 "Parser.rl" +#line 1873 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint3c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 9532 "Parser.cpp" +#line 9553 "Parser.cpp" break; } - case 398: { + case 400: { { -#line 1875 "Parser.rl" +#line 1882 "Parser.rl" {te = p+1;{ -#line 1875 "Parser.rl" +#line 1882 "Parser.rl" assert(mode.utf8); /* leverage ComponentClass to generate the vertices */ auto cc = getComponentClass(mode); cc->add(readUtf8CodePoint4c(ts)); cc->finalize(); - currentSeq->addComponent(move(cc)); + currentSeq->addComponent(std::move(cc)); } }} -#line 9550 "Parser.cpp" +#line 9571 "Parser.cpp" break; } - case 399: { + case 401: { { -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" {te = p+1;{ -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 9564 "Parser.cpp" +#line 9585 "Parser.cpp" break; } - case 400: { + case 402: { { -#line 1893 "Parser.rl" +#line 1900 "Parser.rl" {te = p+1;{ -#line 1893 "Parser.rl" +#line 1900 "Parser.rl" if (mode.ignore_space == false) { addLiteral(currentSeq, *ts, mode); @@ -9987,46 +10010,46 @@ static const short _regex_actions[] = { } }} -#line 9579 "Parser.cpp" +#line 9600 "Parser.cpp" break; } - case 401: { + case 403: { { -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" {te = p+1;{ -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" addLiteral(currentSeq, *ts, mode); } }} -#line 9592 "Parser.cpp" +#line 9613 "Parser.cpp" break; } - case 402: { + case 404: { { -#line 328 "Parser.rl" +#line 329 "Parser.rl" {te = p;p = p - 1;{ -#line 328 "Parser.rl" +#line 329 "Parser.rl" PUSH_SEQUENCE; auto seq = std::make_unique(); seq->setCaptureIndex(groupIndex++); - currentSeq = enterSequence(currentSeq, move(seq)); + currentSeq = enterSequence(currentSeq, std::move(seq)); } }} -#line 9608 "Parser.cpp" +#line 9629 "Parser.cpp" break; } - case 403: { + case 405: { { -#line 421 "Parser.rl" +#line 422 "Parser.rl" {te = p;p = p - 1;{ -#line 421 "Parser.rl" +#line 422 "Parser.rl" assert(!currentCls); assert(!inCharClass); // not reentrant @@ -10034,18 +10057,18 @@ static const short _regex_actions[] = { inCharClass = true; inCharClassEarly = true; currentClsBegin = ts; - {cs = 836;goto _again;}} + {cs = 837;goto _again;}} }} -#line 9626 "Parser.cpp" +#line 9647 "Parser.cpp" break; } - case 404: { + case 406: { { -#line 1310 "Parser.rl" +#line 1317 "Parser.rl" {te = p;p = p - 1;{ -#line 1310 "Parser.rl" +#line 1317 "Parser.rl" if (!currentSeq->addRepeat(0, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_GREEDY)) { @@ -10054,15 +10077,15 @@ static const short _regex_actions[] = { } }} -#line 9642 "Parser.cpp" +#line 9663 "Parser.cpp" break; } - case 405: { + case 407: { { -#line 1331 "Parser.rl" +#line 1338 "Parser.rl" {te = p;p = p - 1;{ -#line 1331 "Parser.rl" +#line 1338 "Parser.rl" if (!currentSeq->addRepeat(1, ComponentRepeat::NoLimit, ComponentRepeat::REPEAT_GREEDY)) { @@ -10071,15 +10094,15 @@ static const short _regex_actions[] = { } }} -#line 9658 "Parser.cpp" +#line 9679 "Parser.cpp" break; } - case 406: { + case 408: { { -#line 1352 "Parser.rl" +#line 1359 "Parser.rl" {te = p;p = p - 1;{ -#line 1352 "Parser.rl" +#line 1359 "Parser.rl" if (!currentSeq->addRepeat( 0, 1, ComponentRepeat::REPEAT_GREEDY)) { @@ -10088,15 +10111,15 @@ static const short _regex_actions[] = { } }} -#line 9674 "Parser.cpp" +#line 9695 "Parser.cpp" break; } - case 407: { + case 409: { { -#line 1373 "Parser.rl" +#line 1380 "Parser.rl" {te = p;p = p - 1;{ -#line 1373 "Parser.rl" +#line 1380 "Parser.rl" if (repeatN > repeatM || repeatM == 0) { throwInvalidRepeat(); @@ -10108,29 +10131,29 @@ static const short _regex_actions[] = { } }} -#line 9693 "Parser.cpp" +#line 9714 "Parser.cpp" break; } - case 408: { + case 410: { { -#line 1488 "Parser.rl" +#line 1495 "Parser.rl" {te = p;p = p - 1;{ -#line 1488 "Parser.rl" +#line 1495 "Parser.rl" addLiteral(currentSeq, octAccumulator, mode); } }} -#line 9706 "Parser.cpp" +#line 9727 "Parser.cpp" break; } - case 409: { + case 411: { { -#line 1491 "Parser.rl" +#line 1498 "Parser.rl" {te = p;p = p - 1;{ -#line 1491 "Parser.rl" +#line 1498 "Parser.rl" // If there are enough capturing sub expressions, this may be // a back reference @@ -10143,15 +10166,15 @@ static const short _regex_actions[] = { } }} -#line 9726 "Parser.cpp" +#line 9747 "Parser.cpp" break; } - case 410: { + case 412: { { -#line 479 "Parser.rl" +#line 480 "Parser.rl" {te = p;p = p - 1;{ -#line 479 "Parser.rl" +#line 480 "Parser.rl" if (accumulator == 0) { throw LocatedParseError("Numbered reference cannot be zero"); @@ -10160,15 +10183,15 @@ static const short _regex_actions[] = { } }} -#line 9742 "Parser.cpp" +#line 9763 "Parser.cpp" break; } - case 411: { + case 413: { { -#line 479 "Parser.rl" +#line 480 "Parser.rl" {te = p;p = p - 1;{ -#line 479 "Parser.rl" +#line 480 "Parser.rl" if (accumulator == 0) { throw LocatedParseError("Numbered reference cannot be zero"); @@ -10177,15 +10200,15 @@ static const short _regex_actions[] = { } }} -#line 9758 "Parser.cpp" +#line 9779 "Parser.cpp" break; } - case 412: { + case 414: { { -#line 486 "Parser.rl" +#line 487 "Parser.rl" {te = p;p = p - 1;{ -#line 486 "Parser.rl" +#line 487 "Parser.rl" // Accumulator is a negative offset. if (accumulator == 0) { @@ -10199,71 +10222,71 @@ static const short _regex_actions[] = { } }} -#line 9779 "Parser.cpp" +#line 9800 "Parser.cpp" break; } - case 413: { + case 415: { { -#line 1557 "Parser.rl" +#line 1564 "Parser.rl" {te = p;p = p - 1;{ -#line 1557 "Parser.rl" +#line 1564 "Parser.rl" throw LocatedParseError("Invalid reference after \\g"); } }} -#line 9792 "Parser.cpp" +#line 9813 "Parser.cpp" break; } - case 414: { + case 416: { { -#line 1574 "Parser.rl" +#line 1581 "Parser.rl" {te = p;p = p - 1;{ -#line 1574 "Parser.rl" +#line 1581 "Parser.rl" throw LocatedParseError("Value in \\o{...} sequence is non-octal or missing braces"); } }} -#line 9805 "Parser.cpp" +#line 9826 "Parser.cpp" break; } - case 415: { + case 417: { { -#line 1578 "Parser.rl" +#line 1585 "Parser.rl" {te = p;p = p - 1;{ -#line 1578 "Parser.rl" +#line 1585 "Parser.rl" addEscapedHex(currentSeq, accumulator, mode); } }} -#line 9818 "Parser.cpp" +#line 9839 "Parser.cpp" break; } - case 416: { + case 418: { { -#line 1596 "Parser.rl" +#line 1603 "Parser.rl" {te = p;p = p - 1;{ -#line 1596 "Parser.rl" +#line 1603 "Parser.rl" throw LocatedParseError("Value in \\x{...} sequence is non-hex or missing }"); } }} -#line 9831 "Parser.cpp" +#line 9852 "Parser.cpp" break; } - case 417: { + case 419: { { -#line 1600 "Parser.rl" +#line 1607 "Parser.rl" {te = p;p = p - 1;{ -#line 1600 "Parser.rl" +#line 1607 "Parser.rl" if (te - ts < 3) { assert(te - ts == 2); @@ -10275,39 +10298,39 @@ static const short _regex_actions[] = { } }} -#line 9850 "Parser.cpp" +#line 9871 "Parser.cpp" break; } - case 418: { + case 420: { { -#line 1700 "Parser.rl" +#line 1707 "Parser.rl" {te = p;p = p - 1;{ -#line 1700 "Parser.rl" +#line 1707 "Parser.rl" throw LocatedParseError("Malformed property"); } }} -#line 9861 "Parser.cpp" +#line 9882 "Parser.cpp" break; } - case 419: { + case 421: { { -#line 1701 "Parser.rl" +#line 1708 "Parser.rl" {te = p;p = p - 1;{ -#line 1701 "Parser.rl" +#line 1708 "Parser.rl" throw LocatedParseError("Malformed property"); } }} -#line 9872 "Parser.cpp" +#line 9893 "Parser.cpp" break; } - case 420: { + case 422: { { -#line 1719 "Parser.rl" +#line 1726 "Parser.rl" {te = p;p = p - 1;{ -#line 1719 "Parser.rl" +#line 1726 "Parser.rl" ostringstream str; str << "\\k at index " << ts - ptr << " not supported."; @@ -10315,15 +10338,15 @@ static const short _regex_actions[] = { } }} -#line 9887 "Parser.cpp" +#line 9908 "Parser.cpp" break; } - case 421: { + case 423: { { -#line 1742 "Parser.rl" +#line 1749 "Parser.rl" {te = p;p = p - 1;{ -#line 1742 "Parser.rl" +#line 1749 "Parser.rl" assert(ts + 1 == pe); ostringstream str; @@ -10332,91 +10355,91 @@ static const short _regex_actions[] = { } }} -#line 9903 "Parser.cpp" +#line 9924 "Parser.cpp" break; } - case 422: { + case 424: { { -#line 396 "Parser.rl" +#line 397 "Parser.rl" {te = p;p = p - 1;{ -#line 396 "Parser.rl" +#line 397 "Parser.rl" throw LocatedParseError("Conditional subpattern unsupported"); } }} -#line 9916 "Parser.cpp" +#line 9937 "Parser.cpp" break; } - case 423: { + case 425: { { -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" {te = p;p = p - 1;{ -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" throw LocatedParseError("Unrecognised character after (?"); } }} -#line 9929 "Parser.cpp" +#line 9950 "Parser.cpp" break; } - case 424: { + case 426: { { -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" {te = p;p = p - 1;{ -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 9943 "Parser.cpp" +#line 9964 "Parser.cpp" break; } - case 425: { + case 427: { { -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" {te = p;p = p - 1;{ -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" addLiteral(currentSeq, *ts, mode); } }} -#line 9956 "Parser.cpp" +#line 9977 "Parser.cpp" break; } - case 426: { + case 428: { { -#line 328 "Parser.rl" +#line 329 "Parser.rl" {p = ((te))-1; { -#line 328 "Parser.rl" +#line 329 "Parser.rl" PUSH_SEQUENCE; auto seq = std::make_unique(); seq->setCaptureIndex(groupIndex++); - currentSeq = enterSequence(currentSeq, move(seq)); + currentSeq = enterSequence(currentSeq, std::move(seq)); } }} -#line 9973 "Parser.cpp" +#line 9994 "Parser.cpp" break; } - case 427: { + case 429: { { -#line 421 "Parser.rl" +#line 422 "Parser.rl" {p = ((te))-1; { -#line 421 "Parser.rl" +#line 422 "Parser.rl" assert(!currentCls); assert(!inCharClass); // not reentrant @@ -10424,64 +10447,64 @@ static const short _regex_actions[] = { inCharClass = true; inCharClassEarly = true; currentClsBegin = ts; - {cs = 836;goto _again;}} + {cs = 837;goto _again;}} }} -#line 9992 "Parser.cpp" +#line 10013 "Parser.cpp" break; } - case 428: { + case 430: { { -#line 1557 "Parser.rl" +#line 1564 "Parser.rl" {p = ((te))-1; { -#line 1557 "Parser.rl" +#line 1564 "Parser.rl" throw LocatedParseError("Invalid reference after \\g"); } }} -#line 10006 "Parser.cpp" +#line 10027 "Parser.cpp" break; } - case 429: { + case 431: { { -#line 1574 "Parser.rl" +#line 1581 "Parser.rl" {p = ((te))-1; { -#line 1574 "Parser.rl" +#line 1581 "Parser.rl" throw LocatedParseError("Value in \\o{...} sequence is non-octal or missing braces"); } }} -#line 10020 "Parser.cpp" +#line 10041 "Parser.cpp" break; } - case 430: { + case 432: { { -#line 1596 "Parser.rl" +#line 1603 "Parser.rl" {p = ((te))-1; { -#line 1596 "Parser.rl" +#line 1603 "Parser.rl" throw LocatedParseError("Value in \\x{...} sequence is non-hex or missing }"); } }} -#line 10034 "Parser.cpp" +#line 10055 "Parser.cpp" break; } - case 431: { + case 433: { { -#line 1719 "Parser.rl" +#line 1726 "Parser.rl" {p = ((te))-1; { -#line 1719 "Parser.rl" +#line 1726 "Parser.rl" ostringstream str; str << "\\k at index " << ts - ptr << " not supported."; @@ -10489,79 +10512,79 @@ static const short _regex_actions[] = { } }} -#line 10050 "Parser.cpp" +#line 10071 "Parser.cpp" break; } - case 432: { + case 434: { { -#line 396 "Parser.rl" +#line 397 "Parser.rl" {p = ((te))-1; { -#line 396 "Parser.rl" +#line 397 "Parser.rl" throw LocatedParseError("Conditional subpattern unsupported"); } }} -#line 10064 "Parser.cpp" +#line 10085 "Parser.cpp" break; } - case 433: { + case 435: { { -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" {p = ((te))-1; { -#line 1852 "Parser.rl" +#line 1859 "Parser.rl" throw LocatedParseError("Unrecognised character after (?"); } }} -#line 10078 "Parser.cpp" +#line 10099 "Parser.cpp" break; } - case 434: { + case 436: { { -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" {p = ((te))-1; { -#line 1884 "Parser.rl" +#line 1891 "Parser.rl" assert(mode.utf8); throwInvalidUtf8(); } }} -#line 10093 "Parser.cpp" +#line 10114 "Parser.cpp" break; } - case 435: { + case 437: { { -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" {p = ((te))-1; { -#line 1898 "Parser.rl" +#line 1905 "Parser.rl" addLiteral(currentSeq, *ts, mode); } }} -#line 10107 "Parser.cpp" +#line 10128 "Parser.cpp" break; } - case 436: { + case 438: { { #line 1 "NONE" {switch( act ) { - case 288: { + case 290: { p = ((te))-1; { -#line 1491 "Parser.rl" +#line 1498 "Parser.rl" // If there are enough capturing sub expressions, this may be // a back reference @@ -10574,10 +10597,10 @@ static const short _regex_actions[] = { } break; } - case 290: { + case 292: { p = ((te))-1; { -#line 1508 "Parser.rl" +#line 1515 "Parser.rl" // if there are enough left parens to this point, back ref if (accumulator < groupIndex) { @@ -10611,10 +10634,10 @@ static const short _regex_actions[] = { } break; } - case 330: { + case 332: { p = ((te))-1; { -#line 1737 "Parser.rl" +#line 1744 "Parser.rl" addLiteral(currentSeq, *(ts + 1), mode); } @@ -10623,7 +10646,7 @@ static const short _regex_actions[] = { }} } -#line 10179 "Parser.cpp" +#line 10200 "Parser.cpp" break; } @@ -10650,7 +10673,7 @@ else { #line 1 "NONE" {ts = 0;}} -#line 10205 "Parser.cpp" +#line 10226 "Parser.cpp" break; } @@ -10667,7 +10690,7 @@ else { _out: {} } -#line 1983 "Parser.rl" +#line 1990 "Parser.rl" if (p != pe && *p != '\0') { @@ -10711,7 +10734,7 @@ assert(currentSeq == rootSeq.get()); // Ensure that all references are valid. checkReferences(*rootSeq, groupIndex, groupNames); -return move(rootSeq); +return rootSeq; } catch (LocatedParseError &error) { if (ts >= ptr && ts <= pe) { error.locate(ts - ptr); diff --git a/contrib/vectorscan-cmake/rageled_files/amd64/control_verbs.cpp b/contrib/vectorscan-cmake/rageled_files/amd64/control_verbs.cpp index e0393e4935561..57cea09266693 100644 --- a/contrib/vectorscan-cmake/rageled_files/amd64/control_verbs.cpp +++ b/contrib/vectorscan-cmake/rageled_files/amd64/control_verbs.cpp @@ -74,28 +74,28 @@ static const signed char _ControlVerbs_actions[] = { 155, 157, 159, 160, 161, 0 }; - static const char _ControlVerbs_trans_keys[] = { - 41, 65, 66, 67, 76, 78, 85, 41, - 41, 78, 41, 89, 41, 67, 41, 82, - 41, 76, 41, 70, 41, 41, 83, 41, - 82, 41, 95, 41, 65, 85, 41, 78, - 41, 89, 41, 67, 41, 78, 41, 73, - 41, 67, 41, 79, 41, 68, 41, 69, - 41, 82, 41, 76, 41, 70, 73, 41, - 77, 41, 73, 41, 84, 41, 95, 41, - 77, 82, 41, 65, 41, 84, 41, 67, - 41, 72, 41, 61, 41, 48, 57, 41, - 48, 57, 41, 69, 41, 67, 41, 85, - 41, 82, 41, 83, 41, 73, 41, 79, - 41, 78, 41, 79, 41, 95, 41, 65, - 83, 41, 85, 41, 84, 41, 79, 41, - 95, 41, 80, 41, 79, 41, 83, 41, - 83, 41, 69, 41, 83, 41, 83, 41, - 84, 41, 65, 41, 82, 41, 84, 41, - 95, 41, 79, 41, 80, 41, 84, 41, - 67, 84, 41, 80, 41, 41, 70, 41, - 49, 51, 56, 41, 54, 41, 50, 41, - 40, 42, 0 + static const unsigned char _ControlVerbs_trans_keys[] = { + 41u, 65u, 66u, 67u, 76u, 78u, 85u, 41u, + 41u, 78u, 41u, 89u, 41u, 67u, 41u, 82u, + 41u, 76u, 41u, 70u, 41u, 41u, 83u, 41u, + 82u, 41u, 95u, 41u, 65u, 85u, 41u, 78u, + 41u, 89u, 41u, 67u, 41u, 78u, 41u, 73u, + 41u, 67u, 41u, 79u, 41u, 68u, 41u, 69u, + 41u, 82u, 41u, 76u, 41u, 70u, 73u, 41u, + 77u, 41u, 73u, 41u, 84u, 41u, 95u, 41u, + 77u, 82u, 41u, 65u, 41u, 84u, 41u, 67u, + 41u, 72u, 41u, 61u, 41u, 48u, 57u, 41u, + 48u, 57u, 41u, 69u, 41u, 67u, 41u, 85u, + 41u, 82u, 41u, 83u, 41u, 73u, 41u, 79u, + 41u, 78u, 41u, 79u, 41u, 95u, 41u, 65u, + 83u, 41u, 85u, 41u, 84u, 41u, 79u, 41u, + 95u, 41u, 80u, 41u, 79u, 41u, 83u, 41u, + 83u, 41u, 69u, 41u, 83u, 41u, 83u, 41u, + 84u, 41u, 65u, 41u, 82u, 41u, 84u, 41u, + 95u, 41u, 79u, 41u, 80u, 41u, 84u, 41u, + 67u, 84u, 41u, 80u, 41u, 41u, 70u, 41u, + 49u, 51u, 56u, 41u, 54u, 41u, 50u, 41u, + 40u, 42u, 0u }; static const signed char _ControlVerbs_single_lengths[] = { @@ -276,7 +276,7 @@ static const signed char _ControlVerbs_actions[] = { te = 0; } -#line 105 "control_verbs.rl" +#line 106 "control_verbs.rl" try { @@ -285,7 +285,7 @@ static const signed char _ControlVerbs_actions[] = { { int _klen; unsigned int _trans = 0; - const char * _keys; + const unsigned char * _keys; const signed char * _acts; unsigned int _nacts; _resume: {} @@ -321,9 +321,9 @@ static const signed char _ControlVerbs_actions[] = { _klen = (int)_ControlVerbs_single_lengths[cs]; if ( _klen > 0 ) { - const char *_lower = _keys; - const char *_upper = _keys + _klen - 1; - const char *_mid; + const unsigned char *_lower = _keys; + const unsigned char *_upper = _keys + _klen - 1; + const unsigned char *_mid; while ( 1 ) { if ( _upper < _lower ) { _keys += _klen; @@ -345,9 +345,9 @@ static const signed char _ControlVerbs_actions[] = { _klen = (int)_ControlVerbs_range_lengths[cs]; if ( _klen > 0 ) { - const char *_lower = _keys; - const char *_upper = _keys + (_klen<<1) - 2; - const char *_mid; + const unsigned char *_lower = _keys; + const unsigned char *_upper = _keys + (_klen<<1) - 2; + const unsigned char *_mid; while ( 1 ) { if ( _upper < _lower ) { _trans += (unsigned int)_klen; @@ -389,9 +389,9 @@ static const signed char _ControlVerbs_actions[] = { } case 3: { { -#line 76 "control_verbs.rl" +#line 77 "control_verbs.rl" {te = p+1;{ -#line 76 "control_verbs.rl" +#line 77 "control_verbs.rl" mode.utf8 = true; } @@ -403,9 +403,9 @@ static const signed char _ControlVerbs_actions[] = { } case 4: { { -#line 80 "control_verbs.rl" +#line 81 "control_verbs.rl" {te = p+1;{ -#line 80 "control_verbs.rl" +#line 81 "control_verbs.rl" mode.ucp = true; } @@ -417,9 +417,9 @@ static const signed char _ControlVerbs_actions[] = { } case 5: { { -#line 84 "control_verbs.rl" +#line 85 "control_verbs.rl" {te = p+1;{ -#line 84 "control_verbs.rl" +#line 85 "control_verbs.rl" ostringstream str; str << "Unsupported control verb " << string(ts, te - ts); @@ -433,9 +433,9 @@ static const signed char _ControlVerbs_actions[] = { } case 6: { { -#line 90 "control_verbs.rl" +#line 91 "control_verbs.rl" {te = p+1;{ -#line 90 "control_verbs.rl" +#line 91 "control_verbs.rl" ostringstream str; str << "Unknown control verb " << string(ts, te - ts); @@ -449,9 +449,9 @@ static const signed char _ControlVerbs_actions[] = { } case 7: { { -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {te = p+1;{ -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {p = p - 1; } {p += 1; goto _out; } @@ -464,9 +464,9 @@ static const signed char _ControlVerbs_actions[] = { } case 8: { { -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {te = p;p = p - 1;{ -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {p = p - 1; } {p += 1; goto _out; } @@ -479,10 +479,10 @@ static const signed char _ControlVerbs_actions[] = { } case 9: { { -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {p = ((te))-1; { -#line 97 "control_verbs.rl" +#line 98 "control_verbs.rl" {p = p - 1; } {p += 1; goto _out; } @@ -530,9 +530,11 @@ static const signed char _ControlVerbs_actions[] = { _out: {} } -#line 108 "control_verbs.rl" +#line 109 "control_verbs.rl" } catch (LocatedParseError &error) { + // cppcheck-suppress pointerLessThanZero + // cppcheck-suppress pointerPositive if (ts >= ptr && ts <= pe) { error.locate(ts - ptr + start); } else { diff --git a/docker/keeper/Dockerfile.distroless b/docker/keeper/Dockerfile.distroless index 4b30438a93ade..99642eae67581 100644 --- a/docker/keeper/Dockerfile.distroless +++ b/docker/keeper/Dockerfile.distroless @@ -3,8 +3,8 @@ # The entrypoint is the compiled `clickhouse docker-init --keeper` subcommand. # # Build targets: -# production — gcr.io/distroless/cc-debian13:nonroot (default) -# debug — gcr.io/distroless/cc-debian13:debug-nonroot (includes busybox shell) +# production — gcr.io/distroless/base-nossl-debian13:nonroot (default) +# debug — gcr.io/distroless/base-nossl-debian13:debug-nonroot (includes busybox shell) # # Usage: # docker build -f Dockerfile.distroless --target production -t clickhouse/clickhouse-keeper:distroless . @@ -143,10 +143,11 @@ RUN mkdir -p \ } > /output/etc/group # ────────────────────────────────────────────────────────────────────────────── -# Stage 2: Production distroless image. +# Stage 2: Debug image — same as production but includes the busybox shell +# at /busybox/sh for interactive troubleshooting. # ────────────────────────────────────────────────────────────────────────────── -# Pinned 2026-04-02. Refresh: docker pull gcr.io/distroless/cc-debian13:nonroot && docker inspect --format='{{index .RepoDigests 0}}' gcr.io/distroless/cc-debian13:nonroot -FROM gcr.io/distroless/cc-debian13:nonroot@sha256:d3cda6e91129130d7229a1806b6a73d292ef245ab032da7851907798024cefba AS production +# Pinned 2026-08-28. Refresh: docker pull gcr.io/distroless/base-nossl-debian13:debug-nonroot && docker inspect --format='{{index .RepoDigests 0}}' gcr.io/distroless/base-nossl-debian13:debug-nonroot +FROM gcr.io/distroless/base-nossl-debian13:debug-nonroot@sha256:12224264396c6637ae850ff7f82fb5c759115d9aa4c8b55f4aa7d40059c2e6f1 AS debug COPY --from=ch-builder /output/ / @@ -165,11 +166,10 @@ WORKDIR /var/lib/clickhouse ENTRYPOINT ["/usr/bin/clickhouse", "docker-init", "--keeper"] # ────────────────────────────────────────────────────────────────────────────── -# Stage 3: Debug image — same as production but includes the busybox shell -# at /busybox/sh for interactive troubleshooting. +# Stage 3: Production distroless image. # ────────────────────────────────────────────────────────────────────────────── -# Pinned 2026-04-02. Refresh: docker pull gcr.io/distroless/cc-debian13:debug-nonroot && docker inspect --format='{{index .RepoDigests 0}}' gcr.io/distroless/cc-debian13:debug-nonroot -FROM gcr.io/distroless/cc-debian13:debug-nonroot@sha256:61e8df8909b98d2cf65fa8f620ee8292be40becd9de3b85d42852bee3f86fdb0 AS debug +# Pinned 2026-08-28. Refresh: docker pull gcr.io/distroless/base-nossl-debian13:nonroot && docker inspect --format='{{index .RepoDigests 0}}' gcr.io/distroless/base-nossl-debian13:nonroot +FROM gcr.io/distroless/base-nossl-debian13:nonroot@sha256:5cab74e7f8a5e7c5f1c8a9e6268b1f352f053c36c656f493308340bcecbc636c AS production COPY --from=ch-builder /output/ / diff --git a/docker/server/Dockerfile.distroless b/docker/server/Dockerfile.distroless index 5db48b7559144..4899e22af66a8 100644 --- a/docker/server/Dockerfile.distroless +++ b/docker/server/Dockerfile.distroless @@ -3,8 +3,8 @@ # The entrypoint is the compiled `clickhouse docker-init` subcommand. # # Build targets: -# production — gcr.io/distroless/cc-debian13:nonroot (default) -# debug — gcr.io/distroless/cc-debian13:debug-nonroot (includes busybox shell) +# production — gcr.io/distroless/base-nossl-debian13:nonroot (default) +# debug — gcr.io/distroless/base-nossl-debian13:debug-nonroot (includes busybox shell) # # Usage: # docker build -f Dockerfile.distroless --target production -t clickhouse/clickhouse-server:distroless . @@ -161,10 +161,11 @@ RUN { \ } > /output/etc/group # ────────────────────────────────────────────────────────────────────────────── -# Stage 2: Production distroless image. +# Stage 2: Debug image — same as production but includes the busybox shell +# at /busybox/sh for interactive troubleshooting. # ────────────────────────────────────────────────────────────────────────────── -# Pinned 2026-04-02. Refresh: docker pull gcr.io/distroless/cc-debian13:nonroot && docker inspect --format='{{index .RepoDigests 0}}' gcr.io/distroless/cc-debian13:nonroot -FROM gcr.io/distroless/cc-debian13:nonroot@sha256:d3cda6e91129130d7229a1806b6a73d292ef245ab032da7851907798024cefba AS production +# Pinned 2026-08-28. Refresh: docker pull gcr.io/distroless/base-nossl-debian13:debug-nonroot && docker inspect --format='{{index .RepoDigests 0}}' gcr.io/distroless/base-nossl-debian13:debug-nonroot +FROM gcr.io/distroless/base-nossl-debian13:debug-nonroot@sha256:12224264396c6637ae850ff7f82fb5c759115d9aa4c8b55f4aa7d40059c2e6f1 AS debug COPY --from=ch-builder /output/ / @@ -183,11 +184,10 @@ WORKDIR /var/lib/clickhouse ENTRYPOINT ["/usr/bin/clickhouse", "docker-init"] # ────────────────────────────────────────────────────────────────────────────── -# Stage 3: Debug image — same as production but includes the busybox shell -# at /busybox/sh for interactive troubleshooting. +# Stage 3: Production distroless image. # ────────────────────────────────────────────────────────────────────────────── -# Pinned 2026-04-02. Refresh: docker pull gcr.io/distroless/cc-debian13:debug-nonroot && docker inspect --format='{{index .RepoDigests 0}}' gcr.io/distroless/cc-debian13:debug-nonroot -FROM gcr.io/distroless/cc-debian13:debug-nonroot@sha256:61e8df8909b98d2cf65fa8f620ee8292be40becd9de3b85d42852bee3f86fdb0 AS debug +# Pinned 2026-08-28. Refresh: docker pull gcr.io/distroless/base-nossl-debian13:nonroot && docker inspect --format='{{index .RepoDigests 0}}' gcr.io/distroless/base-nossl-debian13:nonroot +FROM gcr.io/distroless/base-nossl-debian13:nonroot@sha256:5cab74e7f8a5e7c5f1c8a9e6268b1f352f053c36c656f493308340bcecbc636c AS production COPY --from=ch-builder /output/ / diff --git a/docs/en/engines/table-engines/special/join.md b/docs/en/engines/table-engines/special/join.md index c0c2020d16f1a..6f877a1f9a6b9 100644 --- a/docs/en/engines/table-engines/special/join.md +++ b/docs/en/engines/table-engines/special/join.md @@ -104,6 +104,8 @@ Default value: `1`. The `Join`-engine tables can't be used in `GLOBAL JOIN` operations. +A [row policy](/sql-reference/statements/create/row-policy) on a `Join`-engine table filters a plain `SELECT` from it, but a `JOIN` or `joinGet` reads the prepared hash table as is and cannot filter its rows, so while a policy applies to the table such queries fail with `ACCESS_DENIED`. + The `Join`-engine allows to specify [join_use_nulls](/operations/settings/settings.md/#join_use_nulls) setting in the `CREATE TABLE` statement. [SELECT](/sql-reference/statements/select/index.md) query should have the same `join_use_nulls` value. ## Usage examples {#example} diff --git a/docs/en/operations/named-collections.md b/docs/en/operations/named-collections.md index 54b411ef91940..b84729638a882 100644 --- a/docs/en/operations/named-collections.md +++ b/docs/en/operations/named-collections.md @@ -23,16 +23,15 @@ The keys in a named collection must match the parameter names of the correspondi function, table engine, database, etc. In the examples below the parameter list is linked to for each type. -Parameters set in a named collection can be overridden in SQL, this is shown in the examples -below. This ability can be limited using `[NOT] OVERRIDABLE` keywords and XML attributes -and/or the configuration option `allow_named_collection_override_by_default`. - -:::warning -If override is allowed, it may be possible for users without administrative access to -figure out the credentials that you are trying to hide. -If you are using named collections with that purpose, you should disable -`allow_named_collection_override_by_default` (which is enabled by default). -::: +Overriding a parameter stored in a named collection requires the `SHOW NAMED COLLECTIONS SECRETS` privilege on that collection, including when the parameter is marked `OVERRIDABLE`. Keys marked `NOT OVERRIDABLE` in SQL or `overridable="false"` in XML cannot be overridden. Adding a parameter that the collection does not store does not require this additional privilege. Aliases, such as `addresses_expr` for `host`, count as overrides of the stored parameter. + +```sql +GRANT SHOW NAMED COLLECTIONS SECRETS ON my_collection TO my_user; +``` + +This check requires the privilege alone; it does not depend on the settings that control displaying secrets. The `allow_named_collection_override_by_default` setting is obsolete and has no effect. + +Dictionary sources follow the same rule. The privilege is checked when the dictionary is created, attached, or restored. When the dictionary is loaded, only keys marked `NOT OVERRIDABLE` are enforced. ## Storing named collections in the system database {#storing-named-collections-in-the-system-database} @@ -47,9 +46,9 @@ url = 'https://connection.url/' In the above example: -* `key_1` can always be overridden. +* `key_1` can be overridden by users with `SHOW NAMED COLLECTIONS SECRETS` on the collection. * `key_2` can never be overridden. -* `url` can be overridden or not depending on the value of `allow_named_collection_override_by_default`. +* `url` can be overridden by users with `SHOW NAMED COLLECTIONS SECRETS` on the collection. ### Permissions to create named collections with DDL {#permissions-to-create-named-collections-with-ddl} @@ -126,9 +125,9 @@ Changing the storage type requires a server restart; `SYSTEM RELOAD CONFIG` does In the above example: -* `key_1` can always be overridden. +* `key_1` can be overridden by users with `SHOW NAMED COLLECTIONS SECRETS` on the collection. * `key_2` can never be overridden. -* `url` can be overridden or not depending on the value of `allow_named_collection_override_by_default`. +* `url` can be overridden by users with `SHOW NAMED COLLECTIONS SECRETS` on the collection. ## Modifying named collections {#modifying-named-collections} @@ -489,9 +488,16 @@ SELECT * FROM remote(remote1, database = default, table = test); ### Example of using named collections with a dictionary with source ClickHouse {#example-of-using-named-collections-with-a-dictionary-with-source-clickhouse} ```sql +CREATE NAMED COLLECTION remote_dict AS +host = 'localhost', +port = 9000, +database = 'default', +user = 'default', +password = ''; + CREATE DICTIONARY dict(a Int64, b String) PRIMARY KEY a -SOURCE(CLICKHOUSE(NAME remote1 TABLE test DB default)) +SOURCE(CLICKHOUSE(NAME remote_dict TABLE test)) LIFETIME(MIN 1 MAX 2) LAYOUT(HASHED()); @@ -639,7 +645,7 @@ CREATE DICTIONARY dict `b` String ) PRIMARY KEY a -SOURCE(MONGODB(NAME mymongo COLLECTION my_dict)) +SOURCE(MONGODB(NAME mymongo)) LIFETIME(MIN 1 MAX 2) LAYOUT(HASHED()) @@ -651,5 +657,5 @@ SELECT dictGet('dict', 'b', 2); ``` :::note -The named collection specifies `my_collection` for the collection name. In the function call it is overwritten by `collection = 'my_dict'` to select another collection. +The dictionary uses `my_collection`, as specified in the named collection. To use another MongoDB collection, define a separate named collection with that value. ::: diff --git a/docs/en/operations/server-configuration-parameters/_server_settings_outside_source.md b/docs/en/operations/server-configuration-parameters/_server_settings_outside_source.md index 2580d649f7310..ad6c5526870e3 100644 --- a/docs/en/operations/server-configuration-parameters/_server_settings_outside_source.md +++ b/docs/en/operations/server-configuration-parameters/_server_settings_outside_source.md @@ -1698,11 +1698,22 @@ When adding a host with the `\` xml tag: - if the host is specified as an IP address, then it is checked as specified in the URL. For example: `[2a02:6b8:a::a]`. - if there are redirects and support for redirects is enabled, then every redirect (the location field) is checked. +When adding a regular expression with the `\` xml tag, the RE2 engine is used and the whole host is matched. For example: `^.*\.clickhouse\.com$` + +When adding a single S3 bucket with the `\` xml tag, as `host/bucket` or `host:port/bucket`: +- it is useful when the endpoint is shared by many parties, for example the path-style AWS endpoint `s3.us-west-2.amazonaws.com`, and allowing the whole host would allow every bucket on it. +- the bucket is taken from the parsed S3 URL, so the entry matches both `https://s3.us-west-2.amazonaws.com/my-bucket/...` and `https://my-bucket.s3.us-west-2.amazonaws.com/...`. +- it applies to S3 URLs only: the `s3` table functions and engines, the `S3` database engine, `BACKUP`/`RESTORE` to `S3`, and S3 disks created in SQL with the `disk` function. +- an S3 disk created in SQL is checked when the query creates it. A disk that already existed when the list was set, or that was created by a version without this check, still loads at server start, including a table that was dropped but not yet deleted, so `UNDROP TABLE` keeps working; the server reports it in the log and in `system.warnings`. +- a permanent (301) redirect returned by S3 is checked against the bucket entries as well, but a temporary (307) redirect is checked by host only, so it is rejected when the list contains only `\` entries. + For example: ```sql clickhouse.com + ^.*\.clickhouse\.com$ + s3.us-west-2.amazonaws.com/my-bucket ``` @@ -1826,6 +1837,7 @@ Settings for optional improvements in the access control system. | Setting | Description | Default | |-------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------| | `on_cluster_queries_require_cluster_grant` | Sets whether `ON CLUSTER` queries require the `CLUSTER` grant. | `true` | +| `readonly_restricts_set_role` | Sets whether the `SET ROLE` statement is refused when the session or the query is readonly. When `false`, `readonly` does not restrict `SET ROLE`. The `role` HTTP query parameter is not affected. | `false` | | `role_cache_expiration_time_seconds` | Sets the number of seconds since last access, that a role is stored in the Role Cache. | `600` | | `select_from_information_schema_requires_grant` | Sets whether `SELECT * FROM information_schema.` requires any grants and can be executed by any user. If set to true, then this query requires `GRANT SELECT ON information_schema.
`, just as for ordinary tables. | `true` | | `select_from_system_db_requires_grant` | Sets whether `SELECT * FROM system.
` requires any grants and can be executed by any user. If set to true then this query requires `GRANT SELECT ON system.
` just as for non-system tables. Exceptions: a few system tables (`tables`, `columns`, `databases`, and some constant tables like `one`, `contributors`) are still accessible for everyone; and if there is a `SHOW` privilege (e.g. `SHOW USERS`) granted then the corresponding system table (i.e. `system.users`) will be accessible. | `true` | @@ -1845,6 +1857,7 @@ Example: truetruefalse + false600 ``` diff --git a/docs/en/operations/settings/settings-users.md b/docs/en/operations/settings/settings-users.md index 1f8f682be2a79..d7d61ff6ee9ce 100644 --- a/docs/en/operations/settings/settings-users.md +++ b/docs/en/operations/settings/settings-users.md @@ -111,6 +111,9 @@ It's not yet supported in SQL-driven Access Control. To authenticate using TOTP, users must provide a primary password along with a one-time password generated by their TOTP application via the `--one-time-password` command line option or concatenated to main password together with a '+' character. For example if the primary password is `some_password` and the generated TOTP code is `345123`, user may specify `--password some_password+345123` or `--password some_password --one-time-password 345123` when connecting to ClickHouse. If no password specified `clickhouse-client` will prompt for it interactively. +Each code is accepted at most once, as required by [RFC 6238](https://datatracker.ietf.org/doc/html/rfc6238#section-5.2), so a user can successfully authenticate at most once per `period`. +Keep this in mind for clients that authenticate on every request or connection (for example, scripts using the HTTP interface): each authentication needs a fresh code. + To enable TOTP authentication for a user, configure the `time_based_one_time_password` section in `users.xml`. This section defines the TOTP settings, such as secret, validity period, number of digits, and hash algorithm. **Example** diff --git a/docs/en/sql-reference/statements/create/named-collection.md b/docs/en/sql-reference/statements/create/named-collection.md index 59a285e92345f..928a35fbe0cba 100644 --- a/docs/en/sql-reference/statements/create/named-collection.md +++ b/docs/en/sql-reference/statements/create/named-collection.md @@ -24,6 +24,11 @@ key_name3 = 'some value' [[NOT] OVERRIDABLE], ... ``` +Overriding a stored key when using the collection requires `SHOW NAMED COLLECTIONS SECRETS` on that collection, +including keys marked `OVERRIDABLE`. Keys marked `NOT OVERRIDABLE` cannot be overridden. +Dictionary sources follow the same rule. The privilege is checked when the dictionary is created, attached, or restored. +When the dictionary is loaded, only keys marked `NOT OVERRIDABLE` are enforced. + **Example** ```sql diff --git a/docs/en/sql-reference/statements/create/row-policy.md b/docs/en/sql-reference/statements/create/row-policy.md index 115298580fcbc..e844df93ad6f0 100644 --- a/docs/en/sql-reference/statements/create/row-policy.md +++ b/docs/en/sql-reference/statements/create/row-policy.md @@ -79,6 +79,10 @@ CREATE ROW POLICY pol2 ON mydb.table1 USING c=2 AS RESTRICTIVE TO peter, antonio enable the user `peter` to see table1 rows only if both `b=1` AND `c=2`, although any other table in mydb would have only `b=1` policy applied for the user. +## Join tables {#join-tables} + +A [Join](../../../engines/table-engines/special/join.md) table is a prepared hash table that a `JOIN` or `joinGet` reads as is, so its rows cannot be filtered there. A policy on such a table, including a database-wide `ON db.*` policy, filters a plain `SELECT` from the table, but while it applies, `JOIN` and `joinGet` queries against the table fail with `ACCESS_DENIED`. + ## ON CLUSTER Clause {#on-cluster-clause} Allows creating row policies on a cluster, see [Distributed DDL](../../../sql-reference/distributed-ddl.md). diff --git a/programs/keeper/Keeper.cpp b/programs/keeper/Keeper.cpp index 2f6ff040fab72..006c49fa998fc 100644 --- a/programs/keeper/Keeper.cpp +++ b/programs/keeper/Keeper.cpp @@ -362,33 +362,6 @@ try if (!config().has("keeper_server")) throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "Keeper configuration ( section) not found in config"); - auto update_memory_soft_limit_in_config = [&](Poco::Util::AbstractConfiguration & config) - { - UInt64 memory_soft_limit = 0; - if (config.has("keeper_server.max_memory_usage_soft_limit")) - { - memory_soft_limit = config.getUInt64("keeper_server.max_memory_usage_soft_limit"); - } - - /// if memory soft limit is not set, we will use default value - if (memory_soft_limit == 0) - { - Float64 ratio = 0.9; - if (config.has("keeper_server.max_memory_usage_soft_limit_ratio")) - ratio = config.getDouble("keeper_server.max_memory_usage_soft_limit_ratio"); - - size_t physical_server_memory = getMemoryAmount(); - if (ratio > 0 && physical_server_memory > 0) - { - memory_soft_limit = static_cast(static_cast(physical_server_memory) * ratio); - config.setUInt64("keeper_server.max_memory_usage_soft_limit", memory_soft_limit); - } - } - LOG_INFO(log, "keeper_server.max_memory_usage_soft_limit is set to {}", formatReadableSizeWithBinarySuffix(memory_soft_limit)); - }; - - update_memory_soft_limit_in_config(config()); - std::string path = getKeeperPath(config()); std::filesystem::create_directories(path); @@ -652,7 +625,6 @@ try config().replace("default", loaded_config, PRIO_DEFAULT, true); updateLevels(config(), logger()); - update_memory_soft_limit_in_config(config()); if (config().has("keeper_server")) global_context->updateKeeperConfiguration(config()); diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 5e7e188c77df6..1fe389aeb18a6 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -355,7 +355,9 @@ static DatabasePtr createClickHouseLocalDatabaseOverlay(const String & name_, Co DatabaseCatalog::getStoreDirPath(default_database_uuid); overlay->registerNextDatabase(std::make_shared(name_, default_database_metadata_path, default_database_uuid, context)); - overlay->registerNextDatabase(std::make_shared(name_, "", context)); + /// Not a metadata replay: this database is registered fresh on every start, so a missing path + /// should still be refused here. Master reaches this through `DatabaseURL` and has no such call. + overlay->registerNextDatabase(std::make_shared(name_, "", context, /*is_internal_metadata_replay=*/false)); return overlay; } @@ -398,7 +400,7 @@ void LocalServer::tryInitPath() LOG_DEBUG(log, "Can not get temporary folder: {}", e.what()); parent_folder = std::filesystem::current_path(); - std::filesystem::is_directory(parent_folder); // that will throw an exception if it's not a directory + (void)std::filesystem::is_directory(parent_folder); // checks the path is accessible (may throw on I/O error) LOG_DEBUG(log, "Will create working directory inside current directory: {}", parent_folder.string()); } diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index f2688947447d4..38cb123df1f27 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -977,7 +977,7 @@ void loadStartupScripts(const Poco::Util::AbstractConfiguration & config, const auto condition_read_buffer = ReadBufferFromString(condition); auto condition_write_buffer = WriteBufferFromOwnString(); - LOG_DEBUG(log, "Checking startup query condition `{}`", condition); + LOG_DEBUG(log, "Checking startup query condition `{}`", formatQueryForLogging(condition, startup_context->getSettingsRef())); startup_context->setQueryKind(ClientInfo::QueryKind::INITIAL_QUERY); startup_context->setCurrentQueryId(""); @@ -1011,7 +1011,7 @@ void loadStartupScripts(const Poco::Util::AbstractConfiguration & config, const auto read_buffer = ReadBufferFromString(query); auto write_buffer = WriteBufferFromOwnString(); - LOG_DEBUG(log, "Executing query `{}`", query); + LOG_DEBUG(log, "Executing query `{}`", formatQueryForLogging(query, startup_context->getSettingsRef())); startup_context->setQueryKind(ClientInfo::QueryKind::INITIAL_QUERY); startup_context->setCurrentQueryId(""); @@ -2308,8 +2308,9 @@ try global_context->getProcessList().setMaxWaitingQueriesAmount(new_server_settings[ServerSetting::max_waiting_queries]); if (config().has("keeper_server")) + { global_context->updateKeeperConfiguration(config()); - + } /// Reload the number of threads for global pools. /// Note: If you specified it in the top level config (not it config of default profile) /// then ClickHouse will use it exactly. diff --git a/programs/server/config.xml b/programs/server/config.xml index 8c16f317b3209..9f546c7560691 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -870,6 +870,10 @@ however you can change this behaviour by setting this to true --> false + + false + 600 @@ -1025,6 +1029,16 @@ Regexps are not aligned: don't forget to add ^ and $. Also don't forget to escape dot (.) metacharacter (forgetting to do so is a common source of error). --> + + + 3600 + diff --git a/tests/integration/test_backward_compatibility/configs/remote_url_allow_hosts_without_minio.xml b/tests/integration/test_backward_compatibility/configs/remote_url_allow_hosts_without_minio.xml new file mode 100644 index 0000000000000..b62bb5c5623e8 --- /dev/null +++ b/tests/integration/test_backward_compatibility/configs/remote_url_allow_hosts_without_minio.xml @@ -0,0 +1,5 @@ + + + allowed.example.com + + diff --git a/tests/integration/test_backward_compatibility/test_custom_s3_disk_remote_url_allow_hosts.py b/tests/integration/test_backward_compatibility/test_custom_s3_disk_remote_url_allow_hosts.py new file mode 100644 index 0000000000000..03b0fa8596db2 --- /dev/null +++ b/tests/integration/test_backward_compatibility/test_custom_s3_disk_remote_url_allow_hosts.py @@ -0,0 +1,138 @@ +import uuid + +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.config_cluster import minio_secret_key + +cluster = ClickHouseCluster(__file__) + +# 26.3.50.11 does not check SQL disks against `remote_url_allow_hosts`. The objects are created there with a list +# that does not name MinIO, then the server restarts on the current binary. +# Two instances: a failed check on the database metadata disk would stop the server. +node = cluster.add_instance( + "node", + main_configs=[ + "configs/remote_url_allow_hosts_without_minio.xml", + "configs/drop_table_delay.xml", + ], + image="clickhouse/clickhouse-server", + tag="26.3.50.11", + with_installed_binary=True, + with_minio=True, + stay_alive=True, +) +node_db = cluster.add_instance( + "node_db", + main_configs=[ + "configs/remote_url_allow_hosts_without_minio.xml", + "configs/drop_table_delay.xml", + ], + image="clickhouse/clickhouse-server", + tag="26.3.50.11", + with_installed_binary=True, + with_minio=True, + stay_alive=True, +) + +WARNING_IN_LOG = "is not allowed by `remote_url_allow_hosts`" + + +def s3_disk(path, disk_type="s3"): + return ( + f"disk(type = '{disk_type}', endpoint = 'http://minio1:9001/root/data/{path}/', " + f"access_key_id = 'minio', secret_access_key = '{minio_secret_key}')" + ) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def test_table_on_custom_s3_disk_loads_after_upgrade(started_cluster): + node.query( + f"CREATE TABLE t (id UInt64) ENGINE = MergeTree ORDER BY id SETTINGS disk = {s3_disk('upgrade_disk')}" + ) + node.query("INSERT INTO t VALUES (1), (2), (3)") + assert node.query("SELECT id FROM t ORDER BY id") == "1\n2\n3\n" + # Detached permanently before the upgrade, so its disk is first built by the short ATTACH on the new binary. + node.query( + f"CREATE TABLE t_detached (id UInt64) ENGINE = MergeTree ORDER BY id SETTINGS disk = {s3_disk('detached_disk')}" + ) + node.query("INSERT INTO t_detached VALUES (4), (5)") + node.query("DETACH TABLE t_detached PERMANENTLY") + # Dropped before the upgrade and still in the drop queue: the new binary rebuilds it at start from + # metadata_dropped under FORCE_RESTORE, which is the path UNDROP relies on. + node.query( + f"CREATE TABLE t_dropped (id UInt64) ENGINE = MergeTree ORDER BY id SETTINGS disk = {s3_disk('dropped_disk')}" + ) + node.query("INSERT INTO t_dropped VALUES (6), (7)") + node.query("DROP TABLE t_dropped") + + node.restart_with_latest_version() + + # Loaded from existing metadata: reported, not rejected. + assert node.query("SELECT id FROM t ORDER BY id") == "1\n2\n3\n" + assert node.contains_in_log(WARNING_IN_LOG) + assert ( + node.query( + "SELECT count() FROM system.warnings WHERE message LIKE '%remote_url_allow_hosts%' AND message LIKE '%minio1:9001/root%'" + ) + == "1\n" + ) + # Two disks were built at start and reported: the live table's and the dropped table's. + assert ( + node.query( + "SELECT countSubstrings(message, '->') FROM system.warnings WHERE message LIKE '%remote_url_allow_hosts%'" + ) + == "2\n" + ) + node.query("UNDROP TABLE t_dropped") + assert node.query("SELECT id FROM t_dropped ORDER BY id") == "6\n7\n" + + # Only loading existing metadata is exempt. A user ATTACH is checked like a CREATE, in the short form too: + # the first build of the parked table's disk is rejected, like every other disk check on a short ATTACH. + error = node.query_and_get_error("ATTACH TABLE t_detached") + assert "UNACCEPTABLE_URL" in error, error + error = node.query_and_get_error( + f"CREATE TABLE t_new (id UInt64) ENGINE = MergeTree ORDER BY id SETTINGS disk = {s3_disk('new_disk')}" + ) + assert "UNACCEPTABLE_URL" in error, error + error = node.query_and_get_error( + f"ATTACH TABLE t_attached UUID '{uuid.uuid4()}' (id UInt64) ENGINE = MergeTree ORDER BY id SETTINGS disk = {s3_disk('attached_disk')}" + ) + assert "UNACCEPTABLE_URL" in error, error + + # The operator adds the entry that the warning names; the parked table attaches without a restart. + node.replace_config( + "/etc/clickhouse-server/config.d/remote_url_allow_hosts_without_minio.xml", + "allowed.example.comminio1:9001/root", + ) + node.query("SYSTEM RELOAD CONFIG") + node.query("ATTACH TABLE t_detached") + assert node.query("SELECT id FROM t_detached ORDER BY id") == "4\n5\n" + + +def test_database_on_custom_s3_disk_loads_after_upgrade(started_cluster): + node_db.query( + f"CREATE DATABASE db ENGINE = Atomic SETTINGS disk = {s3_disk('upgrade_db_disk', 's3_plain_rewritable')}" + ) + node_db.query("CREATE TABLE db.t (id UInt64) ENGINE = MergeTree ORDER BY id") + node_db.query("INSERT INTO db.t VALUES (1), (2), (3)") + assert node_db.query("SELECT id FROM db.t ORDER BY id") == "1\n2\n3\n" + + node_db.restart_with_latest_version() + + # The server starts and the database loads; the disk is reported. + assert node_db.query("SELECT id FROM db.t ORDER BY id") == "1\n2\n3\n" + assert node_db.contains_in_log(WARNING_IN_LOG) + + # DETACH and ATTACH reuse the metadata disk built at server start, so no check runs. + node_db.query("DETACH DATABASE db") + node_db.query("ATTACH DATABASE db") + assert node_db.query("SELECT id FROM db.t ORDER BY id") == "1\n2\n3\n" diff --git a/tests/integration/test_cache_bypass_on_disk_failure/__init__.py b/tests/integration/test_cache_bypass_on_disk_failure/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_cache_bypass_on_disk_failure/config.d/storage.xml b/tests/integration/test_cache_bypass_on_disk_failure/config.d/storage.xml new file mode 100644 index 0000000000000..8e1fd570e3a0d --- /dev/null +++ b/tests/integration/test_cache_bypass_on_disk_failure/config.d/storage.xml @@ -0,0 +1,44 @@ + + + + + object_storage + s3 + plain_rewritable + http://minio1:9001/root/data/ + minio + ClickHouse_Minio_P@ssw0rd + + + cache + s3_disk + /var/lib/clickhouse/cache_skip/ + 10Gi + 1 + + + cache + s3_disk + /var/lib/clickhouse/cache_noskip/ + 10Gi + 0 + + + + + +
+ cache_disk_skip +
+ + + + +
+ cache_disk_noskip +
+ + + + + diff --git a/tests/integration/test_cache_bypass_on_disk_failure/test.py b/tests/integration/test_cache_bypass_on_disk_failure/test.py new file mode 100644 index 0000000000000..884d4aeef2e6a --- /dev/null +++ b/tests/integration/test_cache_bypass_on_disk_failure/test.py @@ -0,0 +1,169 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +node = cluster.add_instance( + "node", + main_configs=["config.d/storage.xml"], + with_minio=True, + stay_alive=True, +) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def _drop_cache(node): + """Drop all filesystem cache to ensure next read goes via S3 → cache write path.""" + node.query("SYSTEM DROP FILESYSTEM CACHE") + + +def test_skip_cache_on_disk_failure_select(started_cluster): + """skip=true: failpoint triggers during SELECT, query should succeed (fall back to S3), no broken parts.""" + node.query( + """ + CREATE TABLE t_skip_sel (x UInt64) + ENGINE = MergeTree ORDER BY x + SETTINGS storage_policy = 's3_cache_skip' + """ + ) + try: + node.query("INSERT INTO t_skip_sel SELECT number FROM numbers(100)") + + # Drop cache so next read goes via S3 → writeCache() path, triggering the failpoint + _drop_cache(node) + node.query("SYSTEM ENABLE FAILPOINT cache_filesystem_failure") + try: + # Use sum() to force real column data read, bypassing optimize_trivial_count_query + result = node.query("SELECT sum(x) FROM t_skip_sel") + assert result.strip() == str(sum(range(100))), f"Unexpected result: {result}" + + # No parts should be moved to detached + broken = node.query( + "SELECT count() FROM system.detached_parts " + "WHERE table = 't_skip_sel' AND database = currentDatabase()" + ) + assert broken.strip() == "0", f"Unexpected detached parts: {broken}" + finally: + node.query("SYSTEM DISABLE FAILPOINT cache_filesystem_failure") + finally: + node.query("DROP TABLE IF EXISTS t_skip_sel") + + +def test_noskip_cache_on_disk_failure_select(started_cluster): + """skip=false (default): failpoint triggers during SELECT, should raise CACHE_CANNOT_WRITE_TO_CACHE_DISK.""" + node.query( + """ + CREATE TABLE t_noskip_sel (x UInt64) + ENGINE = MergeTree ORDER BY x + SETTINGS storage_policy = 's3_cache_noskip' + """ + ) + try: + node.query("INSERT INTO t_noskip_sel SELECT number FROM numbers(100)") + + # Drop cache so next read goes via S3 → writeCache() path, triggering the failpoint + _drop_cache(node) + node.query("SYSTEM ENABLE FAILPOINT cache_filesystem_failure") + try: + with pytest.raises(Exception) as exc_info: + node.query("SELECT sum(x) FROM t_noskip_sel") + assert "CACHE_CANNOT_WRITE_TO_CACHE_DISK" in str(exc_info.value) + finally: + node.query("SYSTEM DISABLE FAILPOINT cache_filesystem_failure") + + # After disabling the failpoint, SELECT should succeed again + result = node.query("SELECT sum(x) FROM t_noskip_sel") + assert result.strip() == str(sum(range(100))) + finally: + node.query("DROP TABLE IF EXISTS t_noskip_sel") + + +def test_skip_cache_on_disk_failure_attach(started_cluster): + """ + skip=true: failpoint active during ATTACH TABLE (loadDataPart path). + Part loading should succeed (bypass cache write, read from S3 directly). + system.detached_parts must remain empty. + """ + node.query( + """ + CREATE TABLE t_skip_att (x UInt64) + ENGINE = MergeTree ORDER BY x + SETTINGS storage_policy = 's3_cache_skip' + """ + ) + try: + node.query("INSERT INTO t_skip_att SELECT number FROM numbers(100)") + node.query("DETACH TABLE t_skip_att") + _drop_cache(node) + + node.query("SYSTEM ENABLE FAILPOINT cache_filesystem_failure") + try: + node.query("ATTACH TABLE t_skip_att") + node.query("SYSTEM WAIT LOADING PARTS t_skip_att") + + result = node.query("SELECT sum(x) FROM t_skip_att") + assert result.strip() == str(sum(range(100))) + + broken = node.query( + "SELECT count() FROM system.detached_parts " + "WHERE table = 't_skip_att' AND database = currentDatabase()" + ) + assert broken.strip() == "0", f"Unexpected detached parts: {broken}" + finally: + node.query("SYSTEM DISABLE FAILPOINT cache_filesystem_failure") + finally: + node.query("DROP TABLE IF EXISTS t_skip_att") + + +def test_noskip_cache_on_disk_failure_attach(started_cluster): + """ + skip=false (default): failpoint active during ATTACH TABLE (loadDataPart path). + - ATTACH should fail with CACHE_CANNOT_WRITE_TO_CACHE_DISK. + - Parts must NOT be moved to detached (core assertion: not marked broken). + - After disabling the failpoint, ATTACH should succeed. + """ + node.query( + """ + CREATE TABLE t_noskip_att (x UInt64) + ENGINE = MergeTree ORDER BY x + SETTINGS storage_policy = 's3_cache_noskip' + """ + ) + try: + node.query("INSERT INTO t_noskip_att SELECT number FROM numbers(100)") + node.query("DETACH TABLE t_noskip_att") + _drop_cache(node) + + node.query("SYSTEM ENABLE FAILPOINT cache_filesystem_failure") + try: + with pytest.raises(Exception) as exc_info: + node.query("ATTACH TABLE t_noskip_att") + assert "CACHE_CANNOT_WRITE_TO_CACHE_DISK" in str(exc_info.value) + + # Core assertion: parts must not be marked broken (moved to detached) + broken = node.query( + "SELECT count() FROM system.detached_parts " + "WHERE table = 't_noskip_att' AND database = currentDatabase()" + ) + assert broken.strip() == "0", ( + f"Bug: parts moved to detached despite cache-only failure: {broken}" + ) + finally: + node.query("SYSTEM DISABLE FAILPOINT cache_filesystem_failure") + + # After disabling the failpoint, ATTACH should succeed + node.query("ATTACH TABLE t_noskip_att") + node.query("SYSTEM WAIT LOADING PARTS t_noskip_att") + result = node.query("SELECT sum(x) FROM t_noskip_att") + assert result.strip() == str(sum(range(100))) + finally: + node.query("DROP TABLE IF EXISTS t_noskip_att") diff --git a/tests/integration/test_database_backup/test.py b/tests/integration/test_database_backup/test.py index 9dc7392d77972..b5d20d7d64109 100644 --- a/tests/integration/test_database_backup/test.py +++ b/tests/integration/test_database_backup/test.py @@ -12,6 +12,17 @@ with_minio=True, ) +# `test_database_backup_metadata_with_quoted_locator_loads_on_restart` rewrites a database metadata file +# in place, and such a file only exists when the metadata lives on the local disk, so that test runs on an +# instance which keeps the local database disk. +instance_local_metadata = cluster.add_instance( + "instance_local_metadata", + main_configs=["configs/backups.xml"], + stay_alive=True, + with_minio=True, + with_remote_database_disk=False, +) + @pytest.fixture(scope="module", autouse=True) def start_cluster(): @@ -144,3 +155,153 @@ def test_database_backup_table(backup_destination): instance.query("DROP DATABASE test_table_backup") instance.query("DROP DATABASE test_database") cleanup_backup_files(instance) + + +@pytest.mark.parametrize( + "backup_destination", + [ + "Disk('backup_disk_s3_plain', 'test_database_backup')", + ], +) +def test_multiple_databases_from_same_backup(backup_destination): + # Written by @orloffv in https://github.com/ClickHouse/ClickHouse/pull/83220 + # Regression test for https://github.com/ClickHouse/ClickHouse/issues/83219 + cleanup_backup_files(instance) + + instance.query( + f""" + DROP DATABASE IF EXISTS test_database SYNC; + DROP DATABASE IF EXISTS test_database_backup_1 SYNC; + DROP DATABASE IF EXISTS test_database_backup_2 SYNC; + + CREATE DATABASE test_database; + + CREATE TABLE test_database.test_table (id UInt64, value String) ENGINE=MergeTree ORDER BY id; + INSERT INTO test_database.test_table VALUES (1, 'from_backup'); + + BACKUP DATABASE test_database TO {backup_destination}; + + CREATE DATABASE test_database_backup_1 ENGINE=Backup('test_database', {backup_destination}); + CREATE DATABASE test_database_backup_2 ENGINE=Backup('test_database', {backup_destination}); + """ + ) + + assert ( + instance.query("SELECT id, value FROM test_database_backup_1.test_table") + == "1\tfrom_backup\n" + ) + + assert ( + instance.query("SELECT id, value FROM test_database_backup_2.test_table") + == "1\tfrom_backup\n" + ) + + # Both databases must still read after a restart: the storage policy name is derived + # again on every open, so it has to come out identical. + instance.restart_clickhouse() + + assert ( + instance.query("SELECT id, value FROM test_database_backup_1.test_table") + == "1\tfrom_backup\n" + ) + + assert ( + instance.query("SELECT id, value FROM test_database_backup_2.test_table") + == "1\tfrom_backup\n" + ) + + instance.query("DROP DATABASE IF EXISTS test_database_backup_1 SYNC") + instance.query("DROP DATABASE IF EXISTS test_database_backup_2 SYNC") + instance.query("DROP DATABASE IF EXISTS test_database SYNC") + cleanup_backup_files(instance) + + +def test_database_backup_metadata_with_quoted_locator_loads_on_restart(): + # Regression test for https://github.com/ClickHouse/ClickHouse/issues/118349 + # An older server regenerated the definition of a `Backup` database with the locator quoted into a + # string literal, and `ALTER DATABASE ... MODIFY COMMENT` wrote that back into `metadata/.sql`. + # The next start replays the stored full `ATTACH DATABASE ... ENGINE = Backup(...)` statement, which + # is neither the short `ATTACH` nor a force-restore load, so it has to accept that form on its own. + # + # The metadata file is rewritten in place here, so this runs on the instance whose metadata is a file + # on the local disk rather than an object on a remote database disk. + instance = instance_local_metadata + + cleanup_backup_files(instance) + + instance.query( + """ + DROP DATABASE IF EXISTS test_database SYNC; + DROP DATABASE IF EXISTS test_database_backup SYNC; + + CREATE DATABASE test_database; + + CREATE TABLE test_database.test_table (id UInt64, value String) ENGINE=MergeTree ORDER BY id; + INSERT INTO test_database.test_table VALUES (0, 'test_database.test_table'); + + BACKUP DATABASE test_database TO File('test_database_backup_file'); + CREATE DATABASE test_database_backup ENGINE = Backup('test_database', File('test_database_backup_file')); + """ + ) + assert ( + instance.query("SELECT id, value FROM test_database_backup.test_table") + == "0\ttest_database.test_table\n" + ) + + # The metadata file exactly as a pre-fix server left it after a comment change. + instance.stop_clickhouse() + metadata = ( + "ATTACH DATABASE test_database_backup\n" + "ENGINE = Backup('test_database', 'File(\\'test_database_backup_file\\')')\n" + "COMMENT 'written by an older server'\n" + ) + instance.exec_in_container( + [ + "bash", + "-c", + "cat > /var/lib/clickhouse/metadata/test_database_backup.sql <<'SQL'\n" + + metadata + + "SQL\n", + ], + user="root", + ) + assert "'File(\\'test_database_backup_file\\')'" in instance.exec_in_container( + ["cat", "/var/lib/clickhouse/metadata/test_database_backup.sql"] + ) + instance.start_clickhouse() + + # The server started and the database loaded with its tables and comment. + assert ( + instance.query("SELECT id, value FROM test_database_backup.test_table") + == "0\ttest_database.test_table\n" + ) + assert ( + instance.query( + "SELECT comment FROM system.databases WHERE name = 'test_database_backup'" + ) + == "written by an older server\n" + ) + # The definition is regenerated with the locator as the function it is. + assert ( + "ENGINE = Backup('test_database', File('test_database_backup_file'))" + in instance.query("SHOW CREATE DATABASE test_database_backup FORMAT TSVRaw") + ) + + # A comment change on this server writes the function form, and that survives a restart too. + instance.query( + "ALTER DATABASE test_database_backup MODIFY COMMENT 'written by this server'" + ) + instance.restart_clickhouse() + assert ( + instance.query("SELECT id, value FROM test_database_backup.test_table") + == "0\ttest_database.test_table\n" + ) + assert ( + instance.query( + "SELECT comment FROM system.databases WHERE name = 'test_database_backup'" + ) + == "written by this server\n" + ) + + instance.query("DROP DATABASE test_database_backup SYNC") + instance.query("DROP DATABASE test_database SYNC") diff --git a/tests/integration/test_dictionaries_ddl/configs/user_admin.xml b/tests/integration/test_dictionaries_ddl/configs/user_admin.xml index b5e94aed63a54..6d28d28ac68e4 100644 --- a/tests/integration/test_dictionaries_ddl/configs/user_admin.xml +++ b/tests/integration/test_dictionaries_ddl/configs/user_admin.xml @@ -7,6 +7,8 @@ default default + 1 + 1 diff --git a/tests/integration/test_dictionaries_ddl/test.py b/tests/integration/test_dictionaries_ddl/test.py index 6d455b8c07f3f..dd8031348d749 100644 --- a/tests/integration/test_dictionaries_ddl/test.py +++ b/tests/integration/test_dictionaries_ddl/test.py @@ -623,8 +623,7 @@ def test_secure(started_cluster): # Secure is set to 0 in named collection and in 1 in DDL node1.query("DROP DICTIONARY IF EXISTS test.clickhouse_secure") - node1.query( - """ + create_with_override = """ CREATE DICTIONARY test.clickhouse_secure( id UInt64, value String @@ -638,7 +637,9 @@ def test_secure(started_cluster): )) LIFETIME(MIN 1 MAX 10) """ - ) + # Overriding a stored key requires `SHOW NAMED COLLECTIONS SECRETS`, which `default` does not have + assert "ACCESS_DENIED" in node1.query_and_get_error(create_with_override) + node1.query(create_with_override, user="admin") value = node1.query( "SELECT dictGet('test.clickhouse_secure', 'value', toUInt64(1))" ) diff --git a/tests/integration/test_dictionaries_mysql/configs/dictionaries/mysql_dict_local_infile.xml b/tests/integration/test_dictionaries_mysql/configs/dictionaries/mysql_dict_local_infile.xml new file mode 100644 index 0000000000000..9d5a3fe658f37 --- /dev/null +++ b/tests/integration/test_dictionaries_mysql/configs/dictionaries/mysql_dict_local_infile.xml @@ -0,0 +1,30 @@ + + + dict_local_infile + + + test + mysql80 + 3306 + root + ClickHouse_MySQL_P@ssw0rd +
dict_local_infile_table
+ 1 + + + + + + + + id + + + value + String + + + + 0 + + diff --git a/tests/integration/test_dictionaries_mysql/configs/users.xml b/tests/integration/test_dictionaries_mysql/configs/users.xml index 70c7d3bc2c1b0..725dfacd7de77 100644 --- a/tests/integration/test_dictionaries_mysql/configs/users.xml +++ b/tests/integration/test_dictionaries_mysql/configs/users.xml @@ -13,6 +13,7 @@ default default 1 + 1 diff --git a/tests/integration/test_dictionaries_mysql/test.py b/tests/integration/test_dictionaries_mysql/test.py index 8381773d0adde..47214de2a7477 100644 --- a/tests/integration/test_dictionaries_mysql/test.py +++ b/tests/integration/test_dictionaries_mysql/test.py @@ -12,7 +12,11 @@ from helpers.port_forward import PortForward from helpers.config_cluster import mysql_pass -DICTS = ["configs/dictionaries/mysql_dict1.xml", "configs/dictionaries/mysql_dict2.xml"] +DICTS = [ + "configs/dictionaries/mysql_dict1.xml", + "configs/dictionaries/mysql_dict2.xml", + "configs/dictionaries/mysql_dict_local_infile.xml", +] CONFIG_FILES = [ "configs/remote_servers.xml", "configs/named_collections.xml", @@ -612,8 +616,79 @@ class MySQL_Instance: assert ( int( query( - "SELECT count() FROM system.text_log WHERE message like '%Reestablishing connection to % has failed: mysqlxx::ConnectionFailed: Can\\'t connect to MySQL server on %'" + "SELECT count() FROM system.text_log WHERE message like '%Reestablishing connection to % has failed: mysqlxx::ConnectionFailed: Can\\'t connect to server on %'" ) ) > 0 ) + + +def test_enable_local_infile_xml_dict(started_cluster): + """`enable_local_infile` is rejected in a dictionary created with a DDL query, but a dictionary + defined in a server configuration file is written by an operator and keeps working.""" + mysql_connection = get_mysql_conn(started_cluster) + + try: + execute_mysql_query( + mysql_connection, "DROP TABLE IF EXISTS test.dict_local_infile_table;" + ) + execute_mysql_query( + mysql_connection, + "CREATE TABLE test.dict_local_infile_table (id INT NOT NULL, value TEXT, PRIMARY KEY(id));", + ) + execute_mysql_query( + mysql_connection, + "INSERT INTO test.dict_local_infile_table VALUES (1, 'local_infile');", + ) + + # The dictionary is lazily loaded, so nothing has touched the source yet; reload it now that + # its table exists. + for _ in range(10): + try: + instance.query("SYSTEM RELOAD DICTIONARY dict_local_infile") + break + except Exception: + time.sleep(0.5) + + # Loading the dictionary means the source was instantiated and connected with the option on, + # which is what the guard must not prevent for this route. + value = instance.query( + "SELECT dictGet('dict_local_infile', 'value', toUInt64(1))" + ).strip() + last_exception = instance.query( + "SELECT last_exception FROM system.dictionaries WHERE name = 'dict_local_infile'" + ).strip() + assert value == "local_infile", ( + " was rejected in the XML dict config: " + f"{last_exception!r}" + ) + + # The same option from a DDL query is rejected, at the source and at a replica alike. The + # endpoint here is the working MySQL server, so a rejection cannot be a connection failure + # in disguise. + credentials = f"USER 'root' PASSWORD '{mysql_pass}' DB 'test' TABLE 'dict_local_infile_table'" + for source in ( + f"HOST 'mysql80' PORT 3306 {credentials} ENABLE_LOCAL_INFILE 1", + f"{credentials} REPLICA(PRIORITY 1 HOST 'mysql80' PORT 3306 ENABLE_LOCAL_INFILE 1)", + ): + instance.query("DROP DICTIONARY IF EXISTS dict_local_infile_ddl") + instance.query( + f""" + CREATE DICTIONARY dict_local_infile_ddl (id UInt64, value String) + PRIMARY KEY id + SOURCE(MYSQL({source})) + LAYOUT(FLAT()) + LIFETIME(0) + """ + ) + with pytest.raises(Exception) as exc: + instance.query("SYSTEM RELOAD DICTIONARY dict_local_infile_ddl") + assert "cannot be enabled in a dictionary created with a DDL query" in str( + exc.value + ), f"Unexpected error for {source!r}: {exc.value}" + finally: + instance.query("DROP DICTIONARY IF EXISTS dict_local_infile_ddl") + execute_mysql_query( + mysql_connection, "DROP TABLE IF EXISTS test.dict_local_infile_table;" + ) + mysql_connection.close() diff --git a/tests/integration/test_dictionaries_postgresql/configs/users.xml b/tests/integration/test_dictionaries_postgresql/configs/users.xml index beb08eb6ed4d2..1c5f4d5a21fc8 100644 --- a/tests/integration/test_dictionaries_postgresql/configs/users.xml +++ b/tests/integration/test_dictionaries_postgresql/configs/users.xml @@ -4,7 +4,8 @@ default default - 1 + 1 + 1 diff --git a/tests/integration/test_dictionaries_postgresql/test.py b/tests/integration/test_dictionaries_postgresql/test.py index 88de097bbc1cc..8a80d291545b2 100644 --- a/tests/integration/test_dictionaries_postgresql/test.py +++ b/tests/integration/test_dictionaries_postgresql/test.py @@ -23,6 +23,7 @@ "configs/named_collections.xml", "configs/bg_reconnect.xml", ], + user_configs=["configs/users.xml"], with_postgres=True, with_postgres_cluster=True, ) diff --git a/tests/integration/test_distributed_frozen_replica/test.py b/tests/integration/test_distributed_frozen_replica/test.py index 3e7ba0fb940eb..53b1f9b6e5201 100644 --- a/tests/integration/test_distributed_frozen_replica/test.py +++ b/tests/integration/test_distributed_frozen_replica/test.py @@ -106,7 +106,15 @@ def test_cluster_all_replicas_query( except QueryRuntimeException as e: error = str(e) assert "ALL_CONNECTION_TRIES_FAILED" in error - assert "receive timeout 857 ms" in error + # The configured handshake_timeout_ms=857 may surface through the async + # `HedgedConnections` / `AsyncTaskExecutor` path (which formats as + # "(..., receive timeout {} ms)") OR through the synchronous + # `ReadBufferFromPocoSocket` path (which formats as + # "(peer: ..., local: ..., {} ms)" without the "receive timeout" prefix). + # Both formats include the timeout value in milliseconds, so check for + # "857 ms" alone — this verifies the configured timeout was applied + # without locking the test to one specific socket-error code path. + assert "857 ms" in error @pytest.mark.parametrize( diff --git a/tests/integration/test_drop_database_replica/test.py b/tests/integration/test_drop_database_replica/test.py index 95fa68ad06fd9..ee0efec063ccb 100644 --- a/tests/integration/test_drop_database_replica/test.py +++ b/tests/integration/test_drop_database_replica/test.py @@ -193,7 +193,7 @@ def test_drop_database_replica(started_cluster, with_tables: bool): f"CREATE DATABASE db ENGINE = Replicated('{zk_path}'" + r", '{shard}', '{replica}')" ) - node4.query("DETACH DATABASE db") + node4.query("DETACH DATABASE db SYNC") node4.query( f"INSERT INTO system.zookeeper(name, path, value) VALUES ('active', '{zk_path}/replicas/s2|r2', '{node4_uuid}')", ) diff --git a/tests/integration/test_drop_replica/test.py b/tests/integration/test_drop_replica/test.py index eae997340e533..b69c98b5da120 100644 --- a/tests/integration/test_drop_replica/test.py +++ b/tests/integration/test_drop_replica/test.py @@ -156,9 +156,9 @@ def test_drop_replica(start_cluster): "SYSTEM DROP REPLICA 'node_1_1' FROM ZKPATH '/clickhouse/tables/test'" ) - node_1_1.query("DETACH DATABASE test") + node_1_1.query("DETACH DATABASE test SYNC") for i in range(1, 5): - node_1_1.query("DETACH DATABASE test{}".format(i)) + node_1_1.query("DETACH DATABASE test{} SYNC".format(i)) assert "does not exist" in node_1_3.query_and_get_error( "SYSTEM DROP REPLICA 'node_1_1' FROM TABLE test.test_table" diff --git a/tests/integration/test_filesystem_cache/test.py b/tests/integration/test_filesystem_cache/test.py index 3a3161a737839..4ce1e49829fcf 100644 --- a/tests/integration/test_filesystem_cache/test.py +++ b/tests/integration/test_filesystem_cache/test.py @@ -189,6 +189,83 @@ def test_parallel_cache_loading_on_startup(cluster, node_name): assert count == int(node.query("SELECT count() FROM test")) +@pytest.mark.parametrize("node_name", ["node"]) +def test_bypass_cache_does_not_overread_non_last_segment(cluster, node_name): + """ + Regression test for an over-read on the `REMOTE_FS_READ_BYPASS_CACHE` path. + + A non-last file segment read in bypass mode relies on the buffer being + right-bounded (the read size is clamped to the range only for a single held + segment). For readers without right-bounded support (local object storage) + the bypass buffer must be wrapped into `BoundedReadBuffer`, otherwise it + reads past the segment and trips a logical error in + `CachedOnDiskReadBufferFromFile`. + + The `cache_filesystem_failure` failpoint with `skip_cache_on_disk_failure` + leaves segments in `PARTIALLY_DOWNLOADED_NO_CONTINUATION`, so concurrent + readers read the front segment in bypass mode while holding the next ones. + """ + node = cluster.instances[node_name] + cache_name = f"bypass_overread_{uuid.uuid4().hex[:8]}" + table_name = f"bypass_overread_{uuid.uuid4().hex[:8]}" + try: + node.query( + f""" + DROP TABLE IF EXISTS {table_name} SYNC; + CREATE TABLE {table_name} (key UInt32, value String) + ENGINE = MergeTree() ORDER BY key + SETTINGS disk = disk( + type = cache, + name = '{cache_name}', + path = '{cache_name}/', + max_size = '1Gi', + max_file_segment_size = 32768, + boundary_alignment = 32768, + skip_cache_on_disk_failure = true, + disk = 'hdd_blob' + ); + INSERT INTO {table_name} SELECT number, randomString(100) FROM numbers(100000); + SYSTEM DROP FILESYSTEM CACHE; + """ + ) + + test_start = node.query("SELECT now()").strip() + + # Force every download write to fail so segments stay in + # PARTIALLY_DOWNLOADED_NO_CONTINUATION and reads fall back to bypass. + node.query("SYSTEM ENABLE FAILPOINT cache_filesystem_failure") + try: + # Concurrent readers: while one query leaves the front segment in a + # bypass state, others read it together with the following segments. + node.exec_in_container( + [ + "/usr/bin/clickhouse", + "benchmark", + "--iterations", + "200", + "--concurrency", + "50", + "--query", + f"SELECT * FROM {table_name} FORMAT Null", + ] + ) + finally: + node.query("SYSTEM DISABLE FAILPOINT cache_filesystem_failure") + + # If the over-read aborted the server, the queries above raise a + # connection error (and the cluster teardown reports the crash). + # Otherwise make sure no logical error was recorded. + node.query("SELECT 1") + errors = int( + node.query( + f"SELECT count() FROM system.errors WHERE name = 'LOGICAL_ERROR' AND last_error_time >= '{test_start}'" + ).strip() + ) + assert errors == 0, f"LOGICAL_ERROR occurred on {node.name}" + finally: + node.query(f"DROP TABLE IF EXISTS {table_name} SYNC") + + @pytest.mark.parametrize("node_name", ["node"]) def test_caches_with_the_same_configuration(cluster, node_name): node = cluster.instances[node_name] diff --git a/tests/integration/test_filesystem_split_cache/test.py b/tests/integration/test_filesystem_split_cache/test.py index c611dd42f81e8..3bff16aff2a67 100644 --- a/tests/integration/test_filesystem_split_cache/test.py +++ b/tests/integration/test_filesystem_split_cache/test.py @@ -167,31 +167,71 @@ def test_split_cache_system_files_no_eviction(started_cluster, storage_policy): node.restart_clickhouse() wait_for_cache_initialized(node, storage_policy) - count = int( - node.query( - f"SELECT count(*) FROM system.filesystem_cache WHERE cache_name = '{filesystem_cache_name}' AND segment_type='System'" - ) - ) - assert count > 0 - - def assert_cache_state(): + def system_segments(): """ - The state of cache can change with background processes, so make sure that the state does not change dramatically. + Identify each cached System segment, not just how many there are: a count can stay the + same (or grow) while the original segments are evicted and replaced by different ones. """ - current_count = int( + return set( node.query( - f"SELECT count(*) FROM system.filesystem_cache WHERE cache_name = '{filesystem_cache_name}' AND segment_type='System'" + f"SELECT key, file_segment_range_begin, size FROM system.filesystem_cache " + f"WHERE cache_name = '{filesystem_cache_name}' AND segment_type = 'System' AND size > 0" ) + .strip() + .splitlines() + ) + + def wait_for_stable_system_segments(required_stable_seconds=3.0, timeout_seconds=30.0): + """ + `wait_for_cache_initialized` only reports that the cache became usable; background + loading keeps adding System segments afterwards, sometimes in batches with a gap + between them, so a single repeated sample can land in such a gap and look settled + right before the next batch arrives. Require the set to stay unchanged for a + continuous window, not just across one poll, before accepting it as the baseline; + otherwise the assertions below would compare against a still-partial snapshot, + defeating this wait entirely. Failing to reach that window within the timeout is + itself a test failure, not a signal to fall back to whatever was last observed. + """ + deadline = time.monotonic() + timeout_seconds + current = system_segments() + stable_since = time.monotonic() + while time.monotonic() < deadline: + time.sleep(0.5) + sample = system_segments() + if sample != current: + current = sample + stable_since = time.monotonic() + continue + if current and time.monotonic() - stable_since >= required_stable_seconds: + return current + raise Exception( + f"System segment set for cache '{filesystem_cache_name}' did not stay " + f"unchanged for {required_stable_seconds}s within {timeout_seconds}s" + ) + + baseline = wait_for_stable_system_segments() + assert len(baseline) > 0 + + def assert_no_eviction(current): + """ + System files live in their own cache partition, so the full scan (17 MiB of data through + a separate 4 MiB data partition) must not push them out, and they must survive a restart. + Startup and the scan may cache more of them, hence a subset check rather than equality: + what must not happen is one of the baseline segments disappearing. + """ + evicted = baseline - current + assert not evicted, ( + f"{len(evicted)} of {len(baseline)} System segments were evicted, e.g. {sorted(evicted)[:5]}" ) - fraction = abs(current_count - count) / count - assert fraction <= 0.5, f"System cache count changed too much: {count} -> {current_count}" node.query("SELECT * FROM t0 FORMAT NULL") - assert_cache_state() + assert_no_eviction(system_segments()) node.restart_clickhouse() wait_for_cache_initialized(node, storage_policy) - assert_cache_state() + # Same race as the baseline: a cache that is still reloading looks like eviction, so only + # compare once the set has settled. + assert_no_eviction(wait_for_stable_system_segments()) node.query("DROP TABLE t0 SYNC") diff --git a/tests/integration/test_grpc_protocol/test.py b/tests/integration/test_grpc_protocol/test.py index 135ec000f75af..687cbdbc419f5 100644 --- a/tests/integration/test_grpc_protocol/test.py +++ b/tests/integration/test_grpc_protocol/test.py @@ -506,6 +506,53 @@ def test_external_table(): ) +def test_settings_profile_constraints(): + profile = "grpc_profile_constraints" + query(f"DROP SETTINGS PROFILE IF EXISTS {profile}") + query( + f"CREATE SETTINGS PROFILE {profile} SETTINGS max_execution_time = 10 CONST, max_result_rows = 12345, format_csv_delimiter = '|'" + ) + try: + # `settings` is a protobuf map with no order: the profile goes first, so its constraints bind the + # other settings and they override the values it sets. + e = query_and_get_error( + "SELECT 1", settings={"profile": profile, "max_execution_time": "999"} + ) + assert "Setting max_execution_time should not be changed" in e.display_text + assert ( + query( + "SELECT getSetting('max_result_rows')", + settings={"profile": profile, "max_result_rows": "7"}, + ) + == "7\n" + ) + + columns = [ + clickhouse_grpc_pb2.NameAndType(name="UserID", type="UInt64"), + clickhouse_grpc_pb2.NameAndType(name="UserName", type="String"), + ] + + def ext(settings): + return clickhouse_grpc_pb2.ExternalTable( + name="ext1", columns=columns, data=b"1;Alex\n", format="CSV", settings=settings + ) + + e = query_and_get_error( + "SELECT * FROM ext1", + external_tables=[ext({"profile": profile, "max_execution_time": "999"})], + ) + assert "Setting max_execution_time should not be changed" in e.display_text + assert ( + query( + "SELECT * FROM ext1", + external_tables=[ext({"profile": profile, "format_csv_delimiter": ";"})], + ) + == "1\tAlex\n" + ) + finally: + query(f"DROP SETTINGS PROFILE {profile}") + + def test_external_table_streaming(): columns = [ clickhouse_grpc_pb2.NameAndType(name="UserID", type="UInt64"), diff --git a/tests/integration/test_inserts_with_keeper_retries/configs/backups_disk.xml b/tests/integration/test_inserts_with_keeper_retries/configs/backups_disk.xml new file mode 100644 index 0000000000000..f7d666c65426e --- /dev/null +++ b/tests/integration/test_inserts_with_keeper_retries/configs/backups_disk.xml @@ -0,0 +1,14 @@ + + + + + local + /backups/ + + + + + backups + /backups/ + + diff --git a/tests/integration/test_inserts_with_keeper_retries/test.py b/tests/integration/test_inserts_with_keeper_retries/test.py index 2ab991452a937..e020794533351 100644 --- a/tests/integration/test_inserts_with_keeper_retries/test.py +++ b/tests/integration/test_inserts_with_keeper_retries/test.py @@ -15,7 +15,8 @@ cluster = ClickHouseCluster(__file__) node1 = cluster.add_instance( "node1", - main_configs=["configs/storage_conf.xml"], + main_configs=["configs/storage_conf.xml", "configs/backups_disk.xml"], + external_dirs=["/backups/"], with_zookeeper=True, with_minio=True, with_remote_database_disk=False, # The tests stop Keeper connections, some queries will not work with the remote disk. @@ -188,3 +189,272 @@ def test_retries_should_not_wait_for_global_connection( pm.heal_all() cluster.start_zookeeper_nodes(["zoo1", "zoo2", "zoo3"]) node1.query("DROP TABLE IF EXISTS zk_down_retries SYNC") + + +def test_ambiguous_zk_commit_query_timeout_preserves_data(started_cluster): + query_id = str(uuid.uuid4()) + pool = Pool(1) + job = None + + try: + node1.query( + "CREATE TABLE amb_timeout (a UInt64, b String) ENGINE=ReplicatedMergeTree('/test/amb_timeout', '0') ORDER BY tuple()" + ) + node1.query( + "SYSTEM ENABLE FAILPOINT replicated_merge_tree_commit_zk_fail_after_op" + ) + node1.query( + "SYSTEM ENABLE FAILPOINT replicated_merge_tree_commit_zk_fail_when_recovering_from_hw_fault" + ) + node1.query( + "SYSTEM ENABLE FAILPOINT replicated_merge_tree_insert_retry_pause" + ) + + job = pool.apply_async( + node1.query_and_get_error, + ( + "INSERT INTO amb_timeout SELECT number, toString(number) FROM numbers(10) ORDER BY ALL " + "SETTINGS insert_keeper_max_retries=10000, insert_keeper_retry_initial_backoff_ms=50, " + "insert_keeper_retry_max_backoff_ms=100, max_execution_time=2", + ), + {"query_id": query_id}, + ) + + # Ensure that the Keeper `multi` was committed and its response was lost before + # allowing `max_execution_time` to expire, so a slow CI host cannot time out the + # query before it enters ambiguous-commit recovery. + node1.query( + "SYSTEM WAIT FAILPOINT replicated_merge_tree_insert_retry_pause PAUSE", + timeout=60, + ) + time.sleep(2.1) + node1.query( + "SYSTEM DISABLE FAILPOINT replicated_merge_tree_insert_retry_pause" + ) + + error = job.get(timeout=60) + assert "TIMEOUT_EXCEEDED" in error + + node1.query( + "SYSTEM DISABLE FAILPOINT replicated_merge_tree_commit_zk_fail_when_recovering_from_hw_fault" + ) + node1.query( + "SYSTEM DISABLE FAILPOINT replicated_merge_tree_commit_zk_fail_after_op" + ) + + # The `multi` was committed in Keeper before its response was lost. The local + # part must remain readable and retrying the insert must be deduplicated. + assert node1.query("SELECT count() FROM amb_timeout") == "10\n" + node1.query( + "INSERT INTO amb_timeout SELECT number, toString(number) FROM numbers(10) ORDER BY ALL" + ) + assert node1.query("SELECT count() FROM amb_timeout") == "10\n" + assert ( + node1.query( + "SELECT lost_part_count FROM system.replicas WHERE table = 'amb_timeout'" + ) + == "0\n" + ) + finally: + if job is not None and not job.ready(): + node1.query(f"KILL QUERY WHERE query_id = '{query_id}' ASYNC") + node1.query( + "SYSTEM DISABLE FAILPOINT replicated_merge_tree_insert_retry_pause" + ) + node1.query( + "SYSTEM DISABLE FAILPOINT replicated_merge_tree_commit_zk_fail_when_recovering_from_hw_fault" + ) + node1.query( + "SYSTEM DISABLE FAILPOINT replicated_merge_tree_commit_zk_fail_after_op" + ) + pool.close() + pool.join() + node1.query("DROP TABLE IF EXISTS amb_timeout SYNC") + + +def test_ambiguous_zk_commit_kill_preserves_data(started_cluster): + query_id = str(uuid.uuid4()) + pool = Pool(1) + job = None + + try: + node1.query( + "CREATE TABLE amb_kill (a UInt64, b String) ENGINE=ReplicatedMergeTree('/test/amb_kill', '0') ORDER BY tuple()" + ) + node1.query( + "SYSTEM ENABLE FAILPOINT replicated_merge_tree_commit_zk_fail_after_op" + ) + node1.query( + "SYSTEM ENABLE FAILPOINT replicated_merge_tree_commit_zk_fail_when_recovering_from_hw_fault" + ) + node1.query( + "SYSTEM ENABLE FAILPOINT replicated_merge_tree_insert_retry_pause" + ) + + job = pool.apply_async( + node1.query_and_get_error, + ( + "INSERT INTO amb_kill SELECT number, toString(number) FROM numbers(10) ORDER BY ALL " + "SETTINGS insert_keeper_max_retries=10000, insert_keeper_retry_initial_backoff_ms=100, " + "insert_keeper_retry_max_backoff_ms=200", + ), + {"query_id": query_id}, + ) + + # Pause inside the recovery iteration, after the Keeper commit response was + # lost, so `KILL QUERY` cannot race with the initial `multi`. + node1.query( + "SYSTEM WAIT FAILPOINT replicated_merge_tree_insert_retry_pause PAUSE", + timeout=60, + ) + node1.query(f"KILL QUERY WHERE query_id = '{query_id}' ASYNC") + node1.query( + "SYSTEM DISABLE FAILPOINT replicated_merge_tree_insert_retry_pause" + ) + + error = job.get(timeout=60) + assert "QUERY_WAS_CANCELLED" in error + + node1.query( + "SYSTEM DISABLE FAILPOINT replicated_merge_tree_commit_zk_fail_when_recovering_from_hw_fault" + ) + node1.query( + "SYSTEM DISABLE FAILPOINT replicated_merge_tree_commit_zk_fail_after_op" + ) + + assert node1.query("SELECT count() FROM amb_kill") == "10\n" + node1.query( + "INSERT INTO amb_kill SELECT number, toString(number) FROM numbers(10) ORDER BY ALL" + ) + assert node1.query("SELECT count() FROM amb_kill") == "10\n" + assert ( + node1.query( + "SELECT lost_part_count FROM system.replicas WHERE table = 'amb_kill'" + ) + == "0\n" + ) + finally: + if job is not None and not job.ready(): + node1.query(f"KILL QUERY WHERE query_id = '{query_id}' ASYNC") + node1.query( + "SYSTEM DISABLE FAILPOINT replicated_merge_tree_insert_retry_pause" + ) + node1.query( + "SYSTEM DISABLE FAILPOINT replicated_merge_tree_commit_zk_fail_when_recovering_from_hw_fault" + ) + node1.query( + "SYSTEM DISABLE FAILPOINT replicated_merge_tree_commit_zk_fail_after_op" + ) + pool.close() + pool.join() + node1.query("DROP TABLE IF EXISTS amb_kill SYNC") + + +def test_ambiguous_zk_commit_attach_keeps_part_in_table_path(started_cluster): + try: + node1.query( + "CREATE TABLE amb_attach (a UInt64, b String) ENGINE=ReplicatedMergeTree('/test/amb_attach', '0') ORDER BY tuple()" + ) + node1.query("INSERT INTO amb_attach VALUES (1, '1'), (2, '2'), (3, '3')") + part_name = node1.query( + "SELECT name FROM system.parts WHERE table = 'amb_attach' AND active" + ).strip() + node1.query(f"ALTER TABLE amb_attach DETACH PART '{part_name}'") + + node1.query( + "SYSTEM ENABLE FAILPOINT replicated_merge_tree_commit_zk_fail_after_op" + ) + node1.query( + "SYSTEM ENABLE FAILPOINT replicated_merge_tree_commit_zk_fail_when_recovering_from_hw_fault" + ) + + error = node1.query_and_get_error( + f"ALTER TABLE amb_attach ATTACH PART '{part_name}'", + settings={"insert_keeper_max_retries": 0}, + ) + assert "UNKNOWN_STATUS_OF_INSERT" in error + + node1.query( + "SYSTEM DISABLE FAILPOINT replicated_merge_tree_commit_zk_fail_when_recovering_from_hw_fault" + ) + node1.query( + "SYSTEM DISABLE FAILPOINT replicated_merge_tree_commit_zk_fail_after_op" + ) + + assert node1.query("SELECT count() FROM amb_attach") == "3\n" + active_part_path = node1.query( + "SELECT path FROM system.parts WHERE table = 'amb_attach' AND active" + ) + assert "/detached/" not in active_part_path + assert ( + node1.query( + "SELECT count() FROM system.detached_parts WHERE table = 'amb_attach'" + ) + == "0\n" + ) + assert ( + node1.query( + "SELECT lost_part_count FROM system.replicas WHERE table = 'amb_attach'" + ) + == "0\n" + ) + finally: + node1.query( + "SYSTEM DISABLE FAILPOINT replicated_merge_tree_commit_zk_fail_when_recovering_from_hw_fault" + ) + node1.query( + "SYSTEM DISABLE FAILPOINT replicated_merge_tree_commit_zk_fail_after_op" + ) + node1.query("DROP TABLE IF EXISTS amb_attach SYNC") + + +def test_ambiguous_zk_commit_restore_keeps_part_in_table_path(started_cluster): + backup_name = f"Disk('backups', 'amb_restore_{uuid.uuid4().hex}')" + + try: + node1.query( + "CREATE TABLE amb_restore (a UInt64, b String) ENGINE=ReplicatedMergeTree('/test/amb_restore', '0') ORDER BY tuple()" + ) + node1.query("INSERT INTO amb_restore VALUES (1, '1'), (2, '2'), (3, '3')") + node1.query(f"BACKUP TABLE amb_restore TO {backup_name}") + node1.query("TRUNCATE TABLE amb_restore") + + node1.query( + "SYSTEM ENABLE FAILPOINT replicated_merge_tree_commit_zk_fail_after_op" + ) + node1.query( + "SYSTEM ENABLE FAILPOINT replicated_merge_tree_commit_zk_fail_when_recovering_from_hw_fault" + ) + + error = node1.query_and_get_error( + f"RESTORE TABLE amb_restore FROM {backup_name}", + settings={"backup_restore_keeper_max_retries": 0}, + ) + assert "UNKNOWN_STATUS_OF_INSERT" in error + + node1.query( + "SYSTEM DISABLE FAILPOINT replicated_merge_tree_commit_zk_fail_when_recovering_from_hw_fault" + ) + node1.query( + "SYSTEM DISABLE FAILPOINT replicated_merge_tree_commit_zk_fail_after_op" + ) + + assert node1.query("SELECT count() FROM amb_restore") == "3\n" + active_part_path = node1.query( + "SELECT path FROM system.parts WHERE table = 'amb_restore' AND active" + ) + assert "tmp_restore_" not in active_part_path + assert ( + node1.query( + "SELECT lost_part_count FROM system.replicas WHERE table = 'amb_restore'" + ) + == "0\n" + ) + finally: + node1.query( + "SYSTEM DISABLE FAILPOINT replicated_merge_tree_commit_zk_fail_when_recovering_from_hw_fault" + ) + node1.query( + "SYSTEM DISABLE FAILPOINT replicated_merge_tree_commit_zk_fail_after_op" + ) + node1.query("DROP TABLE IF EXISTS amb_restore SYNC") diff --git a/tests/integration/test_keeper_empty_multi/__init__.py b/tests/integration/test_keeper_empty_multi/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_keeper_empty_multi/configs/keeper.xml b/tests/integration/test_keeper_empty_multi/configs/keeper.xml new file mode 100644 index 0000000000000..43cb16bae3cab --- /dev/null +++ b/tests/integration/test_keeper_empty_multi/configs/keeper.xml @@ -0,0 +1,32 @@ + + + + node + 9181 + + + + + /clickhouse/task_queue/ddl + + + + 9181 + 1 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 5000 + 10000 + + + + + 1 + node + 9234 + + + + diff --git a/tests/integration/test_keeper_empty_multi/test.py b/tests/integration/test_keeper_empty_multi/test.py new file mode 100644 index 0000000000000..fe73c9f635cfa --- /dev/null +++ b/tests/integration/test_keeper_empty_multi/test.py @@ -0,0 +1,55 @@ +import pytest + +import helpers.keeper_utils as keeper_utils +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance( + "node", + main_configs=["configs/keeper.xml"], + stay_alive=True, + with_zookeeper=False, +) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + keeper_utils.wait_nodes(cluster, [node]) + yield cluster + finally: + cluster.shutdown() + + +def test_empty_multi_request(started_cluster): + # A `Multi` request whose subrequest list holds nothing but the terminator record - what a client + # sends when it builds a transaction from a list that turns out to be empty; kazoo serializes it + # as exactly that for a transaction without operations. It parses, and it is preprocessed into + # no deltas at all, so it used to be written to the changelog and then read `deltas.front()` on + # an empty range on the raft commit thread, terminating the process - and terminating it again + # on every restart that replayed the entry. + zk = keeper_utils.get_fake_zk(cluster, node.name) + try: + # ZooKeeper answers with an empty successful multi response, which kazoo turns into an + # empty list of results. + assert zk.transaction().commit() == [] + + assert keeper_utils.send_4lw_cmd(cluster, node, "ruok") == "imok" + + # The session and the server keep working. + zk.create("/after_empty_multi", b"1") + assert zk.get("/after_empty_multi")[0] == b"1" + # A multi transaction with subrequests keeps working. + transaction = zk.transaction() + transaction.create("/after_empty_multi/child", b"2") + assert transaction.commit() == ["/after_empty_multi/child"] + assert zk.get("/after_empty_multi/child")[0] == b"2" + finally: + zk.stop() + zk.close() + + # The entry of the empty multi is in the changelog, and the restart replays it. + node.restart_clickhouse() + keeper_utils.wait_nodes(cluster, [node]) + assert keeper_utils.send_4lw_cmd(cluster, node, "ruok") == "imok" diff --git a/tests/integration/test_keeper_memory_soft_limit/test.py b/tests/integration/test_keeper_memory_soft_limit/test.py index 76101778bb060..ebc8dd518d981 100644 --- a/tests/integration/test_keeper_memory_soft_limit/test.py +++ b/tests/integration/test_keeper_memory_soft_limit/test.py @@ -103,4 +103,4 @@ def test_soft_limit_create(started_cluster): return - raise Exception("all records are inserted but no error occurs") + raise Exception("all records are inserted but no error occurs") \ No newline at end of file diff --git a/tests/integration/test_keeper_memory_soft_limit_ratio/__init__.py b/tests/integration/test_keeper_memory_soft_limit_ratio/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_keeper_memory_soft_limit_ratio/configs/keeper_config1.xml b/tests/integration/test_keeper_memory_soft_limit_ratio/configs/keeper_config1.xml new file mode 100644 index 0000000000000..f11cbfca4381f --- /dev/null +++ b/tests/integration/test_keeper_memory_soft_limit_ratio/configs/keeper_config1.xml @@ -0,0 +1,45 @@ + + true + :: + 0.0.0.0 + + + trace + /var/log/clickhouse-keeper/clickhouse-keeper.log + /var/log/clickhouse-keeper/clickhouse-keeper.err.log + + + + 2181 + 1 + 0.5 + + + 10000 + 15000 + trace + false + 2000 + 4000 + 1 + + + + + 1 + zoo1 + 9444 + + + 2 + zoo2 + 9444 + + + 3 + zoo3 + 9444 + + + + diff --git a/tests/integration/test_keeper_memory_soft_limit_ratio/configs/keeper_config1_reload.xml b/tests/integration/test_keeper_memory_soft_limit_ratio/configs/keeper_config1_reload.xml new file mode 100644 index 0000000000000..2294e70a99978 --- /dev/null +++ b/tests/integration/test_keeper_memory_soft_limit_ratio/configs/keeper_config1_reload.xml @@ -0,0 +1,45 @@ + + true + :: + 0.0.0.0 + + + trace + /var/log/clickhouse-keeper/clickhouse-keeper.log + /var/log/clickhouse-keeper/clickhouse-keeper.err.log + + + + 2181 + 1 + 0.9 + + + 10000 + 15000 + trace + false + 2000 + 4000 + 1 + + + + + 1 + zoo1 + 9444 + + + 2 + zoo2 + 9444 + + + 3 + zoo3 + 9444 + + + + diff --git a/tests/integration/test_keeper_memory_soft_limit_ratio/configs/keeper_config2.xml b/tests/integration/test_keeper_memory_soft_limit_ratio/configs/keeper_config2.xml new file mode 100644 index 0000000000000..cc1930f6f2b31 --- /dev/null +++ b/tests/integration/test_keeper_memory_soft_limit_ratio/configs/keeper_config2.xml @@ -0,0 +1,50 @@ + + true + :: + 0.0.0.0 + + + trace + /var/log/clickhouse-keeper/clickhouse-keeper.log + /var/log/clickhouse-keeper/clickhouse-keeper.err.log + + + + 2181 + 2 + + az-zoo2 + 1 + + 300000000 + + + 10000 + 15000 + trace + false + 2000 + 4000 + + 1 + + + + + 1 + zoo1 + 9444 + + + 2 + zoo2 + 9444 + + + 3 + zoo3 + 9444 + + + + diff --git a/tests/integration/test_keeper_memory_soft_limit_ratio/configs/keeper_config3.xml b/tests/integration/test_keeper_memory_soft_limit_ratio/configs/keeper_config3.xml new file mode 100644 index 0000000000000..9dcb69fb052af --- /dev/null +++ b/tests/integration/test_keeper_memory_soft_limit_ratio/configs/keeper_config3.xml @@ -0,0 +1,47 @@ + + true + :: + 0.0.0.0 + + + trace + /var/log/clickhouse-keeper/clickhouse-keeper.log + /var/log/clickhouse-keeper/clickhouse-keeper.err.log + + + + 2181 + 3 + + 300000000 + + + 10000 + 15000 + trace + false + 2000 + 4000 + + 1 + + + + + 1 + zoo1 + 9444 + + + 2 + zoo2 + 9444 + + + 3 + zoo3 + 9444 + + + + diff --git a/tests/integration/test_keeper_memory_soft_limit_ratio/test.py b/tests/integration/test_keeper_memory_soft_limit_ratio/test.py new file mode 100644 index 0000000000000..330db3a57ab52 --- /dev/null +++ b/tests/integration/test_keeper_memory_soft_limit_ratio/test.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +import os +import time + +import pytest +from kazoo.client import KazooClient + +from helpers.cluster import ClickHouseCluster + +CURRENT_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) + +cluster = ClickHouseCluster(__file__, keeper_config_dir="configs/") + +# clickhouse itself will use external zookeeper +node = cluster.add_instance( + "node", + stay_alive=True, + with_zookeeper=True, + with_remote_database_disk=False, +) + + +def get_connection_zk(nodename, timeout=30.0): + _fake_zk_instance = KazooClient( + hosts=f"{cluster.get_instance_ip(nodename)}:2181", timeout=timeout + ) + _fake_zk_instance.start() + return _fake_zk_instance + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +# Verify that `max_memory_usage_soft_limit` is correctly updated +# on config reload without requiring keeper restart. +def test_soft_limit_hot_reload(started_cluster): + started_cluster.wait_zookeeper_to_start() + + zoo1_container = started_cluster.get_container_id("zoo1") + keeper_log = "/var/log/clickhouse-keeper/clickhouse-keeper.log" + + # zoo1 starts with max_memory_usage_soft_limit_ratio=0.5 only (no explicit value). + startup_lines = started_cluster.exec_in_container( + zoo1_container, + ["bash", "-c", f'grep "max_memory_usage_soft_limit is set to" {keeper_log} || true'], + ).strip().splitlines() + assert len(startup_lines) >= 1, f"Expected at least 1 startup log line, got: {startup_lines}" + startup_value = startup_lines[0].split("is set to ")[-1].strip() + startup_count = len(startup_lines) + + # Reload with ratio=0.9 — the new limit must differ from the ratio=0.5 startup value. + started_cluster.copy_file_to_container( + zoo1_container, + os.path.join(CURRENT_TEST_DIR, "configs", "keeper_config1_reload.xml"), + "/etc/clickhouse-keeper/keeper_config1.xml", + ) + + started_cluster.exec_in_container( + zoo1_container, + ["bash", "-c", "pkill -HUP clickhouse-keeper 2>/dev/null || pkill -HUP clickhouse 2>/dev/null || true"], + user="root", + ) + + # Poll for up to 30 s until a new log entry appears beyond the startup lines with a different value. + deadline = time.time() + 30 + reloaded = False + while time.time() < deadline: + lines = started_cluster.exec_in_container( + zoo1_container, + ["bash", "-c", f'grep "max_memory_usage_soft_limit is set to" {keeper_log} || true'], + ).strip().splitlines() + if len(lines) > startup_count: + reload_value = lines[-1].split("is set to ")[-1].strip() + if reload_value != startup_value: + reloaded = True + break + time.sleep(1) + + assert reloaded, ( + f"Timeout: reload value did not change from startup value '{startup_value}'. " + f"Log lines seen: {lines}" + ) + + # Verify keeper is still accepting connections after reload. + node_zk = get_connection_zk("zoo1") + try: + assert node_zk.exists("/") is not None, "Keeper root node not accessible after reload" + finally: + node_zk.stop() + node_zk.close() diff --git a/tests/integration/test_keeper_remove_rejoin_leader/test.py b/tests/integration/test_keeper_remove_rejoin_leader/test.py index 4a95403558aef..8cb03c599d26c 100644 --- a/tests/integration/test_keeper_remove_rejoin_leader/test.py +++ b/tests/integration/test_keeper_remove_rejoin_leader/test.py @@ -242,6 +242,9 @@ def test_leader_election_after_rolling_membership_change(started_cluster): for n in remaining: keeper_utils.wait_until_connected(cluster, n, timeout=60.0) + # Re-discover the leader after the membership change. + leader = keeper_utils.get_leader(cluster, remaining) + # Step 3: Add node5 to the cluster. node1/3/4 are active at this point, # so node5 can connect to them during startup (async Keeper init). cfg5 = _get_generated_cfg(node_names, 5) @@ -281,9 +284,13 @@ def test_leader_election_after_rolling_membership_change(started_cluster): time.sleep(2) - for n in [leader, node4, node5]: + active_nodes = [leader, node4, node5] + for n in active_nodes: keeper_utils.wait_until_connected(cluster, n, timeout=60.0) + # Re-discover the leader after the membership change. + leader = keeper_utils.get_leader(cluster, active_nodes) + # Step 5: Add node6 to the cluster. node1/4/5 are active at this point, # so node6 can connect to them during startup (async Keeper init). cfg6 = _get_generated_cfg(node_names, 6) diff --git a/tests/integration/test_keeper_secure_client/configs/generate_certs.sh b/tests/integration/test_keeper_secure_client/configs/generate_certs.sh new file mode 100755 index 0000000000000..e398289b8cc9b --- /dev/null +++ b/tests/integration/test_keeper_secure_client/configs/generate_certs.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# Self-signed certificate that Keeper presents on its secure port. It is also its own trust +# anchor (CA:TRUE), so a client can point caConfig at it directly. +# +# The subjectAltName has to list every name a client may connect to: `node1` is the cluster +# member name used in use_secure_keeper.xml, and `localhost` is kept because the raft +# configuration and the existing arms use it. Once a dNSName SAN is present OpenSSL ignores +# the common name, so localhost must be listed explicitly rather than left to CN. +openssl req -newkey rsa:2048 -x509 -days 36500 -nodes -batch \ + -keyout server.key -out server.crt \ + -subj "/CN=localhost" \ + -addext "subjectAltName=DNS:localhost,DNS:node1" \ + -addext "basicConstraints=critical,CA:TRUE" diff --git a/tests/integration/test_keeper_secure_client/configs/server.crt b/tests/integration/test_keeper_secure_client/configs/server.crt index 7ade2d9627338..d9ad8941d6158 100644 --- a/tests/integration/test_keeper_secure_client/configs/server.crt +++ b/tests/integration/test_keeper_secure_client/configs/server.crt @@ -1,19 +1,19 @@ -----BEGIN CERTIFICATE----- -MIIC/TCCAeWgAwIBAgIJANjx1QSR77HBMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV -BAMMCWxvY2FsaG9zdDAgFw0xODA3MzAxODE2MDhaGA8yMjkyMDUxNDE4MTYwOFow -FDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB -CgKCAQEAs9uSo6lJG8o8pw0fbVGVu0tPOljSWcVSXH9uiJBwlZLQnhN4SFSFohfI -4K8U1tBDTnxPLUo/V1K9yzoLiRDGMkwVj6+4+hE2udS2ePTQv5oaMeJ9wrs+5c9T -4pOtlq3pLAdm04ZMB1nbrEysceVudHRkQbGHzHp6VG29Fw7Ga6YpqyHQihRmEkTU -7UCYNA+Vk7aDPdMS/khweyTpXYZimaK9f0ECU3/VOeG3fH6Sp2X6FN4tUj/aFXEj -sRmU5G2TlYiSIUMF2JPdhSihfk1hJVALrHPTU38SOL+GyyBRWdNcrIwVwbpvsvPg -pryMSNxnpr0AK0dFhjwnupIv5hJIOQIDAQABo1AwTjAdBgNVHQ4EFgQUjPLb3uYC -kcamyZHK4/EV8jAP0wQwHwYDVR0jBBgwFoAUjPLb3uYCkcamyZHK4/EV8jAP0wQw -DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAM/ocuDvfPus/KpMVD51j -4IdlU8R0vmnYLQ+ygzOAo7+hUWP5j0yvq4ILWNmQX6HNvUggCgFv9bjwDFhb/5Vr -85ieWfTd9+LTjrOzTw4avdGwpX9G+6jJJSSq15tw5ElOIFb/qNA9O4dBiu8vn03C -L/zRSXrARhSqTW5w/tZkUcSTT+M5h28+Lgn9ysx4Ff5vi44LJ1NnrbJbEAIYsAAD -+UA+4MBFKx1r6hHINULev8+lCfkpwIaeS8RL+op4fr6kQPxnULw8wT8gkuc8I4+L -P9gg/xDHB44T3ADGZ5Ib6O0DJaNiToO6rnoaaxs0KkotbvDWvRoxEytSbXKoYjYp -0g== +MIIDKDCCAhCgAwIBAgIUBTZCvmMS+9yaX8IRKGkGqbcMBOIwDQYJKoZIhvcNAQEL +BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MCAXDTI2MDgyNzExMzMzOFoYDzIxMjYw +ODAzMTEzMzM4WjAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQCyOozYThV2lJBPteVCm7YMVJ/w4hralGidnbkpGZxQ +OnILomL9mym+e2+xdtztN3fbqZhiX3Cpgns6EBM0RYv7SQtcoUG2YErKHuASHSSc +DdCZv18EW1BuJSwA9Qq1fRiySjW+wkyOobxhUFROGgguoIdVOKUtDwib/mq27048 ++uz8i8bYcvlJqMRmiJy2LcwsQMdgYopGtwUGle4rMlAR8oTDBmHlrowoEOjN/mwd +WnLni5I7/4TIrsaARaGSivKLgcMlgs5JTUItvXreLuvdt1wQoCC1bH6h6K3ymsu8 +1xiC0zL9QTbGvhZItybzez/RzXcq3nsGEKCFXtunioRbAgMBAAGjcDBuMB0GA1Ud +DgQWBBQP0fRsGOqD4ZxQ1oTe7v9HIzc3RTAfBgNVHSMEGDAWgBQP0fRsGOqD4ZxQ +1oTe7v9HIzc3RTAbBgNVHREEFDASgglsb2NhbGhvc3SCBW5vZGUxMA8GA1UdEwEB +/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAGw73J/dlsYphCnXZzVfOWiwFtRX +ij9M3/GjtCfmfh2MriYcNlCGqw+7GvzUUJ44rKjRxWyF+FfVM7DC4O0a6oimvtwE +yhb9mFshGDUpMX6WtpjhFGDtNjbgXrz6FXkt8U5SQayVoVLQjIT6XZSmXEccMJpM +U3lGAqPaeMVYVZqQPcvdO+t+Jfrjjznpa66biD87BvJ4CCWNynWi/ftpK76bk1fj +SbL0RgG1SUBAJikJirL2OW+ADgCyPxP86IgTKdOV92e+00spcfvSlms7UzrChXA7 +Kv0Z/9WUgL3tL87kiP0eG3XaZp83w+WS0OOcaU8uQ0AhofzWXMnQT0Bgq/8= -----END CERTIFICATE----- diff --git a/tests/integration/test_keeper_secure_client/configs/server.key b/tests/integration/test_keeper_secure_client/configs/server.key index f0fb61ac443fa..390ae5e9995f6 100644 --- a/tests/integration/test_keeper_secure_client/configs/server.key +++ b/tests/integration/test_keeper_secure_client/configs/server.key @@ -1,28 +1,28 @@ -----BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCz25KjqUkbyjyn -DR9tUZW7S086WNJZxVJcf26IkHCVktCeE3hIVIWiF8jgrxTW0ENOfE8tSj9XUr3L -OguJEMYyTBWPr7j6ETa51LZ49NC/mhox4n3Cuz7lz1Pik62WreksB2bThkwHWdus -TKxx5W50dGRBsYfMenpUbb0XDsZrpimrIdCKFGYSRNTtQJg0D5WTtoM90xL+SHB7 -JOldhmKZor1/QQJTf9U54bd8fpKnZfoU3i1SP9oVcSOxGZTkbZOViJIhQwXYk92F -KKF+TWElUAusc9NTfxI4v4bLIFFZ01ysjBXBum+y8+CmvIxI3GemvQArR0WGPCe6 -ki/mEkg5AgMBAAECggEATrbIBIxwDJOD2/BoUqWkDCY3dGevF8697vFuZKIiQ7PP -TX9j4vPq0DfsmDjHvAPFkTHiTQXzlroFik3LAp+uvhCCVzImmHq0IrwvZ9xtB43f -7Pkc5P6h1l3Ybo8HJ6zRIY3TuLtLxuPSuiOMTQSGRL0zq3SQ5DKuGwkz+kVjHXUN -MR2TECFwMHKQ5VLrC+7PMpsJYyOMlDAWhRfUalxC55xOXTpaN8TxNnwQ8K2ISVY5 -212Jz/a4hn4LdwxSz3Tiu95PN072K87HLWx3EdT6vW4Ge5P/A3y+smIuNAlanMnu -plHBRtpATLiTxZt/n6npyrfQVbYjSH7KWhB8hBHtaQKBgQDh9Cq1c/KtqDtE0Ccr -/r9tZNTUwBE6VP+3OJeKdEdtsfuxjOCkS1oAjgBJiSDOiWPh1DdoDeVZjPKq6pIu -Mq12OE3Doa8znfCXGbkSzEKOb2unKZMJxzrz99kXt40W5DtrqKPNb24CNqTiY8Aa -CjtcX+3weat82VRXvph6U8ltMwKBgQDLxjiQQzNoY7qvg7CwJCjf9qq8jmLK766g -1FHXopqS+dTxDLM8eJSRrpmxGWJvNeNc1uPhsKsKgotqAMdBUQTf7rSTbt4MyoH5 -bUcRLtr+0QTK9hDWMOOvleqNXha68vATkohWYfCueNsC60qD44o8RZAS6UNy3ENq -cM1cxqe84wKBgQDKkHutWnooJtajlTxY27O/nZKT/HA1bDgniMuKaz4R4Gr1PIez -on3YW3V0d0P7BP6PWRIm7bY79vkiMtLEKdiKUGWeyZdo3eHvhDb/3DCawtau8L2K -GZsHVp2//mS1Lfz7Qh8/L/NedqCQ+L4iWiPnZ3THjjwn3CoZ05ucpvrAMwKBgB54 -nay039MUVq44Owub3KDg+dcIU62U+cAC/9oG7qZbxYPmKkc4oL7IJSNecGHA5SbU -2268RFdl/gLz6tfRjbEOuOHzCjFPdvAdbysanpTMHLNc6FefJ+zxtgk9sJh0C4Jh -vxFrw9nTKKzfEl12gQ1SOaEaUIO0fEBGbe8ZpauRAoGAMAlGV+2/K4ebvAJKOVTa -dKAzQ+TD2SJmeR1HZmKDYddNqwtZlzg3v4ZhCk4eaUmGeC1Bdh8MDuB3QQvXz4Dr -vOIP4UVaOr+uM+7TgAgVnP4/K6IeJGzUDhX93pmpWhODfdu/oojEKVcpCojmEmS1 -KCBtmIrQLqzMpnBpLNuSY+Q= +MIIEugIBADANBgkqhkiG9w0BAQEFAASCBKQwggSgAgEAAoIBAQCyOozYThV2lJBP +teVCm7YMVJ/w4hralGidnbkpGZxQOnILomL9mym+e2+xdtztN3fbqZhiX3Cpgns6 +EBM0RYv7SQtcoUG2YErKHuASHSScDdCZv18EW1BuJSwA9Qq1fRiySjW+wkyOobxh +UFROGgguoIdVOKUtDwib/mq27048+uz8i8bYcvlJqMRmiJy2LcwsQMdgYopGtwUG +le4rMlAR8oTDBmHlrowoEOjN/mwdWnLni5I7/4TIrsaARaGSivKLgcMlgs5JTUIt +vXreLuvdt1wQoCC1bH6h6K3ymsu81xiC0zL9QTbGvhZItybzez/RzXcq3nsGEKCF +XtunioRbAgMBAAECgf9xetYy44TG+9Dm7+oJfxtqvncX6N8wTU5PDR3YDXZ5nrz+ +q2ZcE2+A2RdE52nDoI2wT0u3Vw1k4K+VSKbxoVZ/qQKS+BnfBXCZQQ9qeSPWOPWY +70CQhZOwjfp2oY8v/3a6DoYp74zPRCutynfjQq/zAEMq/+Yrymaw7E0GLVijehE7 +eLhAFL/1IAjzvr5ikaN2MgSGHwcZ7XNpTN0Lvpz1PH2LkpDmh7PWUKcvu7nV3cQc +x8BkEFFaqvlXq0NfgPNCZ9qL9/Y+VrJjv0Bta7KCO/6nbmJubUGVZbzv0qwOJBVB +/3iki0BoEG2YJzJLuKnJJ0Ugf6usNai1NY9PKtkCgYEA3V+QJzcL0fWrvw6DBP0m +WWv1afZGr2ONjHmz65IvegFpt9qmm2CnRVzFPXet1VJkxK+MNFiPs+h29s1d2G7/ +VdwU5+b2KdQkgG9QKWsRv9bkNUqK48LWvLYkUCNhYW7oxCH4SxBNbSdVDUpXnZJ/ +u6T71YRgY9yYRI6HnHrJr40CgYEAzhtZ7phK4inHokcXWLJBhYy6HwiENzNUlEm9 +dLRTeExzrdb/zA8WCPc+Vb1dSoNa0l/5TFfyoRpZ2RsfX9iGZIR2GiQhC8ftK2XH +ZynQBN+rGNBoarWhPfpyMU8g5pHCIxChOJaMTA5sb52oHE5D03oQXa0jUQDS7ie8 +IIUh9YcCgYB2I8zLkM3nFAO/J6B2Zh80M7A6B2t7qFZfNIf3XQGnS9++dZraav95 +veOpRRhpMtqCOOlK+kRej94FIl5F5h6wFI63oAOZGRUN0dnm8LP9PFJ3ujtbB50Q +AuhoOCl9FPJ10WcvsBiR+x6hxB30Ar3WR1r6fPXv9Ubxt1raAJFn9QKBgCCNg1vC +4/jqnmRMFCznIqAkRPVH1BIf4lj1eJ3pKVHTyWWIkqg5LcLGwiqqmQR+0KZvkCF1 +tpWpYs1+kisCON/BaCHi2PFSZ2w2TBNIMFnEtfJaYFOSUmBGoSiWldy04tPE+aHF +bW1JzeinHvkxh1bsbY9OHEjb93S0A+ui+2UbAoGADWmYVq4ksRSrHMa/9gwqGC5k +bWx791nrTf2UKlpsBA2b7kQ76DLMzgPGzCAGo5condd65y/4oP/Mr2+mEZkDu+Dh +W8jCsFN9RW2XP4ekgJj41hKAyKedP7JkbBb7pznvtqq1oLndIKN00jjrlfIbsKD0 +Reoz9LQnDowqs/fGhh8= -----END PRIVATE KEY----- diff --git a/tests/integration/test_keeper_secure_client/configs/ssl_conf_verify.xml b/tests/integration/test_keeper_secure_client/configs/ssl_conf_verify.xml new file mode 100644 index 0000000000000..55a744c632f5a --- /dev/null +++ b/tests/integration/test_keeper_secure_client/configs/ssl_conf_verify.xml @@ -0,0 +1,23 @@ + + + + /etc/clickhouse-server/config.d/server.crt + /etc/clickhouse-server/config.d/server.key + none + true + sslv2,sslv3 + + + + /etc/clickhouse-server/config.d/server.crt + false + relaxed + true + sslv2,sslv3 + + RejectCertificateHandler + + + + diff --git a/tests/integration/test_keeper_secure_client/test.py b/tests/integration/test_keeper_secure_client/test.py index 00bc72658c60b..5cab15411686a 100644 --- a/tests/integration/test_keeper_secure_client/test.py +++ b/tests/integration/test_keeper_secure_client/test.py @@ -27,6 +27,16 @@ "configs/server.key", ], ) +# Same secure Keeper, but this client actually verifies the certificate it is presented. +node3 = cluster.add_instance( + "node3", + main_configs=[ + "configs/use_secure_keeper.xml", + "configs/ssl_conf_verify.xml", + "configs/server.crt", + "configs/server.key", + ], +) @pytest.fixture(scope="module") @@ -43,3 +53,9 @@ def started_cluster(): def test_connection(started_cluster): # just nothrow node2.query_with_retry("SELECT * FROM system.zookeeper WHERE path = '/'") + + +def test_connection_verifying_certificate(started_cluster): + # The socket connects to the address node1 resolves to, so this only succeeds if + # the certificate is matched against `node1` rather than against that address. + node3.query_with_retry("SELECT * FROM system.zookeeper WHERE path = '/'") diff --git a/tests/integration/test_mask_sensitive_info/configs/users.xml b/tests/integration/test_mask_sensitive_info/configs/users.xml index f767216e90786..962c4d01437e4 100644 --- a/tests/integration/test_mask_sensitive_info/configs/users.xml +++ b/tests/integration/test_mask_sensitive_info/configs/users.xml @@ -10,6 +10,7 @@ default 1 + 1 diff --git a/tests/integration/test_mask_sensitive_info/test.py b/tests/integration/test_mask_sensitive_info/test.py index 6613e907ab29f..83dd82d92c8b6 100644 --- a/tests/integration/test_mask_sensitive_info/test.py +++ b/tests/integration/test_mask_sensitive_info/test.py @@ -1031,6 +1031,89 @@ def test_backup_table_s3_named_collection(): node.query("DROP NAMED COLLECTION IF EXISTS s3_backup_nc") +def test_database_backup_engine_s3(): + """Secrets in the nested S3(...) destination of the `Backup` database engine must be masked in logs, + both for the explicit-key form and for the named-collection form (including session_token and the + Google ADC secrets). The CREATE fails because the backup does not exist, but it is still logged.""" + password = new_password() + + queries = [ + f"CREATE DATABASE backup_db_s3_1 ENGINE = Backup('', S3('http://minio1:9001/root/data/db_backup_nonexistent', 'minio', '{password}'))", + f"CREATE DATABASE backup_db_s3_2 ENGINE = Backup('', S3(named_collection_6, url = 'http://minio1:9001/root/data/db_backup_nonexistent', access_key_id = 'minio', secret_access_key = '{password}', session_token = '{password}', google_adc_client_secret = '{password}'))", + ] + for query in queries: + node.query_and_get_answer_with_error(query) + + check_logs( + must_contain=[ + "ENGINE = Backup('', S3('http://minio1:9001/root/data/db_backup_nonexistent', 'minio', '[HIDDEN]'))", + "session_token = '[HIDDEN]'", + "secret_access_key = '[HIDDEN]'", + "google_adc_client_secret = '[HIDDEN]'", + ], + must_not_contain=[password], + ) + + node.query("DROP DATABASE IF EXISTS backup_db_s3_1") + node.query("DROP DATABASE IF EXISTS backup_db_s3_2") + + +def test_database_backup_engine_azure_display_surfaces(): + """A live `Backup` database over an `AzureBlobStorage` locator: `account_key` must not reach + `SHOW CREATE DATABASE` or `system.databases.engine_full`. Only `S3` locators are reconstructed + argument by argument, so an Azure one keeps its engine name and arity and hides every argument.""" + azure_storage_account_url = cluster.env_variables["AZURITE_STORAGE_ACCOUNT_URL"] + azure_account_name = "devstoreaccount1" + azure_account_key = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" + + # A backup destination is write-once and the container is emptied only when the cluster starts, + # so give every run its own blob path: a repeat inside one session hits BACKUP_ALREADY_EXISTS. + blob = "db_backup_azure_surfaces_" + "".join( + random.choice(string.ascii_lowercase) for _ in range(8) + ) + locator = ( + f"AzureBlobStorage('{azure_storage_account_url}', '{cluster.azurite_container}', " + f"'{blob}', '{azure_account_name}', '{azure_account_key}')" + ) + masked_locator = ( + "AzureBlobStorage('[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]')" + ) + surfaces = [ + "SHOW CREATE DATABASE backup_db_azure_view", + "SELECT engine_full FROM system.databases WHERE name = 'backup_db_azure_view'", + ] + + node.query("DROP DATABASE IF EXISTS backup_db_azure_src SYNC") + node.query("DROP DATABASE IF EXISTS backup_db_azure_view SYNC") + node.query("CREATE DATABASE backup_db_azure_src") + node.query( + "CREATE TABLE backup_db_azure_src.t (x int) ENGINE = MergeTree ORDER BY x" + ) + node.query("INSERT INTO backup_db_azure_src.t SELECT * FROM numbers(10)") + node.query(f"BACKUP DATABASE backup_db_azure_src TO {locator} FORMAT Null") + node.query( + f"CREATE DATABASE backup_db_azure_view ENGINE = Backup('backup_db_azure_src', {locator})" + ) + + # TSVRaw so the locator is compared as written rather than through TSV escaping. + for surface in surfaces: + shown = node.query(f"{surface} FORMAT TSVRaw") + assert masked_locator in shown, shown + assert azure_account_key not in shown, shown + + # The key is in the locator, so its absence above is masking rather than an argument that the + # regenerated definition never carried. + for surface in surfaces: + shown = node.query(f"{surface} {show_secrets}=1 FORMAT TSVRaw") + assert azure_account_key in shown, shown + + # The masked surfaces belong to a working database, not to one that failed to open. + assert node.query("SELECT count() FROM backup_db_azure_view.t") == "10\n" + + node.query("DROP DATABASE IF EXISTS backup_db_azure_view SYNC") + node.query("DROP DATABASE IF EXISTS backup_db_azure_src SYNC") + + def test_backup_table_azure_named_collection(): """Test that secrets in Azure named collection backups are masked in system.backups and logs.""" azure_storage_account_url = cluster.env_variables["AZURITE_STORAGE_ACCOUNT_URL"] diff --git a/tests/integration/test_modify_engine_on_restart/configs/config.d/clusters_name_path.xml b/tests/integration/test_modify_engine_on_restart/configs/config.d/clusters_name_path.xml new file mode 100644 index 0000000000000..7f5c93e20a719 --- /dev/null +++ b/tests/integration/test_modify_engine_on_restart/configs/config.d/clusters_name_path.xml @@ -0,0 +1,23 @@ + + + + + true + + ch1 + 9000 + + + + + + + 01 + + + +/clickhouse/tables/{database}/{table} + + diff --git a/tests/integration/test_modify_engine_on_restart/test_unsafe_name.py b/tests/integration/test_modify_engine_on_restart/test_unsafe_name.py new file mode 100644 index 0000000000000..7d5412da2afdd --- /dev/null +++ b/tests/integration/test_modify_engine_on_restart/test_unsafe_name.py @@ -0,0 +1,220 @@ +import pytest + +from helpers.cluster import ClickHouseCluster +from test_modify_engine_on_restart.common import get_table_path, set_convert_flags + +cluster = ClickHouseCluster(__file__) +ch1 = cluster.add_instance( + "ch1", + main_configs=[ + "configs/config.d/clusters_name_path.xml", + "configs/config.d/distributed_ddl.xml", + ], + with_zookeeper=True, + macros={"replica": "node1"}, + stay_alive=True, +) + +database_name = "modify_engine_unsafe_name" + +# A name-based `default_replica_path` splices the table's own name into the Keeper path, so a name +# carrying '/' resolves inside another table's subtree. `victim` is the co-tenant that gets damaged; +# the ghost's name is exactly the replica path underneath it. +VICTIM = "victim" +GHOST = "victim/replicas/ghost" + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def q(query): + return ch1.query(database=database_name, sql=query) + + +def victim_total_replicas(): + return q( + f"SELECT total_replicas FROM system.replicas WHERE database = '{database_name}' AND table = '{VICTIM}'" + ).strip() + + +def active_part_paths(table): + """Absolute in-container paths of the table's active parts, each with a trailing '/'. + + `system.parts` rather than a directory listing: a MergeTree table's data directory always holds + a top-level `detached` directory too, and Outdated parts keep their own, so enumerating + directories counts more than the parts. + """ + return q( + "SELECT path FROM system.parts" + f" WHERE database = '{database_name}' AND table = '{table}' AND active" + ).split() + + +def count_txn_version_files(table): + path = get_table_path(ch1, table, database_name) + return int( + ch1.exec_in_container( + ["bash", "-c", f"find {path} -name txn_version.txt | wc -l"] + ).strip() + ) + + +def plant_txn_version_files(table): + """Write a valid non-transactional `txn_version.txt` onto every active part of `table`. + + Transactions are off by default, so parts carry no such file and a bare count of them cannot + tell whether the conversion removed anything. The content is exactly what + `VersionMetadata::write` emits for a prehistoric part that was never removed, so a surviving + file still loads: no `removal_*` fields, which that function writes only under a removal lock. + """ + nil = "00000000-0000-0000-0000-000000000000" + content = ( + "version: 1\\n" + f"creation_tid: (1, 1, {nil})\\n" + "creation_csn: 1" + ) + for part_path in active_part_paths(table): + ch1.exec_in_container( + ["bash", "-c", f'printf "{content}" > "{part_path}txn_version.txt"'] + ) + + +def test_attach_as_replicated_rejects_unsafe_name(started_cluster): + ch1.query(f"DROP DATABASE IF EXISTS {database_name} SYNC") + ch1.query(f"CREATE DATABASE {database_name}") + + q(f"CREATE TABLE `{VICTIM}` ( A Int64 ) ENGINE = ReplicatedMergeTree ORDER BY A") + q(f"INSERT INTO `{VICTIM}` VALUES (1)") + # The whole scenario needs the path to be derived from the name, so assert that rather than + # assuming the config took effect. + assert ( + q( + f"SELECT zookeeper_path FROM system.replicas WHERE database = '{database_name}' AND table = '{VICTIM}'" + ).strip() + == f"/clickhouse/tables/{database_name}/{VICTIM}" + ) + assert victim_total_replicas() == "1" + + # Merges are pinned off so the set of active parts, and therefore the planted file count, is the + # same before the conversion and after the re-attach below. + q( + f"CREATE TABLE `{GHOST}` ( A Int64 ) ENGINE = MergeTree ORDER BY A" + " SETTINGS max_bytes_to_merge_at_max_space_in_pool = 1" + ) + q(f"INSERT INTO `{GHOST}` VALUES (7)") + q(f"INSERT INTO `{GHOST}` VALUES (8)") + planted = len(active_part_paths(GHOST)) + # Arming assertion: with no part to plant onto, the file count reads 0 in every arm and the + # transaction-metadata assertion below discriminates nothing. + assert planted > 0 + plant_txn_version_files(GHOST) + assert count_txn_version_files(GHOST) == planted + q(f"DETACH TABLE `{GHOST}`") + + # The conversion must be refused. Without the check it succeeds and the table takes a path + # under the victim's own subtree. + assert "BAD_ARGUMENTS" in ch1.query_and_get_error( + f"ATTACH TABLE `{GHOST}` AS REPLICATED", database=database_name + ) + + # The rejection ran before the metadata rewrite, so a plain ATTACH brings the table back as + # the MergeTree it always was. A rejection sited after the rewrite reports ReplicatedMergeTree. + q(f"ATTACH TABLE `{GHOST}`") + assert ( + q( + f"SELECT engine FROM system.tables WHERE database = '{database_name}' AND name = '{GHOST}'" + ).strip() + == "MergeTree" + ) + + # It also ran before the table's transaction metadata was removed. That removal is + # irreversible, so the file count is what pins the check ahead of `clearTransactionMetadata`; the + # row count below is a plain no-regression line. + assert count_txn_version_files(GHOST) == planted + assert q(f"SELECT count() FROM `{GHOST}`").strip() == "2" + + # The victim is untouched, and still accepts a metadata ALTER. A planted ghost replica + # leaves this failing with a Keeper error over the ghost's missing log_pointer. + assert victim_total_replicas() == "1" + assert ( + q( + f"SELECT groupArray(name) FROM system.zookeeper WHERE path = '/clickhouse/tables/{database_name}/{VICTIM}/replicas'" + ).strip() + == "['node1']" + ) + q(f"ALTER TABLE `{VICTIM}` ADD COLUMN B UInt64 SETTINGS alter_sync = 2") + + # Control: a path-safe name converts fine under the very same configuration, so the + # rejection above is about the name and not about the config or the conversion route. + q("CREATE TABLE safe_name ( A Int64 ) ENGINE = MergeTree ORDER BY A") + q("INSERT INTO safe_name VALUES (1), (2)") + q("DETACH TABLE safe_name") + q("ATTACH TABLE safe_name AS REPLICATED") + assert ( + q( + f"SELECT engine FROM system.tables WHERE database = '{database_name}' AND name = 'safe_name'" + ).strip() + == "ReplicatedMergeTree" + ) + assert q("SELECT count() FROM safe_name").strip() == "2" + + # Control: the reverse direction mints no Keeper path, so an unsafe name must not block it. + # The table has to be replicated already AND unsafely named, which conversion cannot produce, so + # it is created directly with an explicit path outside the victim's subtree. + q( + f"CREATE TABLE `{GHOST}2` ( A Int64 )" + f" ENGINE = ReplicatedMergeTree('/clickhouse/unrelated/{database_name}', 'node1') ORDER BY A" + ) + q(f"INSERT INTO `{GHOST}2` VALUES (1), (2)") + q(f"DETACH TABLE `{GHOST}2`") + q(f"ATTACH TABLE `{GHOST}2` AS NOT REPLICATED") + assert ( + q( + f"SELECT engine FROM system.tables WHERE database = '{database_name}' AND name = '{GHOST}2'" + ).strip() + == "MergeTree" + ) + assert q(f"SELECT count() FROM `{GHOST}2`").strip() == "2" + + ch1.query(f"DROP DATABASE IF EXISTS {database_name} SYNC") + + +def test_convert_flag_rejects_unsafe_name(started_cluster): + ch1.query(f"DROP DATABASE IF EXISTS {database_name} SYNC") + ch1.query(f"CREATE DATABASE {database_name}") + + q(f"CREATE TABLE `{GHOST}` ( A Int64 ) ENGINE = MergeTree ORDER BY A") + q(f"INSERT INTO `{GHOST}` VALUES (1), (2)") + set_convert_flags(ch1, database_name, [GHOST]) + # Read while the server is up: every helper here answers from a query, and between the stop and + # the recovery start below there is provably no server to answer one. + table_data_path = get_table_path(ch1, GHOST, database_name) + + # The flag-file route refuses the same conversion, and because it runs during startup the + # server does not come up. That is the behaviour the sibling `checkReplicaPathExists` already + # has on this route (see test_zk_path_exists.py), and the recovery is the same: delete the flag. + ch1.stop_clickhouse() + ch1.start_clickhouse(start_wait_sec=120, expected_to_fail=True) + + ch1.exec_in_container( + ["bash", "-c", f"rm {table_data_path}convert_to_replicated"] + ) + ch1.start_clickhouse() + + # The table is intact: still a MergeTree, still holding its rows. + assert ( + q( + f"SELECT engine FROM system.tables WHERE database = '{database_name}' AND name = '{GHOST}'" + ).strip() + == "MergeTree" + ) + assert q(f"SELECT count() FROM `{GHOST}`").strip() == "2" + + ch1.query(f"DROP DATABASE IF EXISTS {database_name} SYNC") diff --git a/tests/integration/test_mq_remote_host_filter/__init__.py b/tests/integration/test_mq_remote_host_filter/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_mq_remote_host_filter/configs/allowed_hosts.xml b/tests/integration/test_mq_remote_host_filter/configs/allowed_hosts.xml new file mode 100644 index 0000000000000..054acd185f73a --- /dev/null +++ b/tests/integration/test_mq_remote_host_filter/configs/allowed_hosts.xml @@ -0,0 +1,8 @@ + + + localhost:19092 + localhost:14222 + allowed-any-port + .*\.allowed\.example\.com + + diff --git a/tests/integration/test_mq_remote_host_filter/configs/kafka_bootstrap_override.xml b/tests/integration/test_mq_remote_host_filter/configs/kafka_bootstrap_override.xml new file mode 100644 index 0000000000000..e6b7a5f8f93fd --- /dev/null +++ b/tests/integration/test_mq_remote_host_filter/configs/kafka_bootstrap_override.xml @@ -0,0 +1,5 @@ + + + localhost:9999 + + diff --git a/tests/integration/test_mq_remote_host_filter/test.py b/tests/integration/test_mq_remote_host_filter/test.py new file mode 100644 index 0000000000000..7d3e4b1afebea --- /dev/null +++ b/tests/integration/test_mq_remote_host_filter/test.py @@ -0,0 +1,193 @@ +""" +Tests that the `Kafka` and `NATS` table engines respect `remote_url_allow_hosts`. + +Both engines used to hand the user-supplied broker addresses (`kafka_broker_list`, +`nats_url`, `nats_server_list`) to the client library without consulting +`RemoteHostFilter`, so `CREATE TABLE` opened outbound TCP connections to hosts the +operator had explicitly forbidden. `RabbitMQ`, the sibling engine, rejects the same +addresses at `CREATE` with `UNACCEPTABLE_URL`. + +The addresses are parsed, validated element-wise, and rebuilt before they reach the +client library, so an entry whose re-parse by the library could disagree with the +validated form (a path, an empty host, a NUL, ...) is rejected with `BAD_ARGUMENTS` +instead of being repaired. + +No broker runs in this test: a forbidden or malformed address must be rejected before +any connection attempt, and an allowed one must get past the filter (and then fail to +connect, for the engine which connects at DDL time). + +Each case is a triplet: the engine input, the expected error code (`None` means the +table must be created), and an optional string the error message must contain. +""" + +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance( + "node", + main_configs=["configs/allowed_hosts.xml"], +) +# The section of the server configuration (and named collections, loaded by the same code) +# can override the broker list librdkafka receives, behind the back of the validated +# `kafka_broker_list` setting. This instance carries such an override to a forbidden host. +node_kafka_override = cluster.add_instance( + "node_kafka_override", + main_configs=["configs/allowed_hosts.xml", "configs/kafka_bootstrap_override.xml"], +) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +# (kafka_broker_list value, expected error code or None for success, required error message part) +KAFKA_CASES = [ + pytest.param("localhost:9093", "UNACCEPTABLE_URL", "localhost:9093", id="forbidden_broker"), + # One allowed broker must not smuggle a forbidden one past the filter. + pytest.param("localhost:19092,localhost:9093", "UNACCEPTABLE_URL", "localhost:9093", id="element_wise"), + # librdkafka accepts `scheme://host:port` entries; the filter must see host:port. + pytest.param("PLAINTEXT://localhost:9093", "UNACCEPTABLE_URL", "localhost:9093", id="scheme_prefix"), + # A bare host must be checked with librdkafka's default port 9092. + pytest.param("localhost", "UNACCEPTABLE_URL", "localhost:9092", id="default_port"), + # A bare `` allowlist entry allows the host on any port, as for other engines. + pytest.param("allowed-any-port:9095", None, None, id="host_allowed_for_any_port"), + pytest.param("localhost:19092", None, None, id="allowed_broker"), + # librdkafka cuts an entry at the first `/` after `scheme://`. Without the strict parse, the + # filter would see the host `evil.com/.allowed.example.com`, the `host_regexp` allowlist entry + # would fully match it, and librdkafka would dial `evil.com`. + pytest.param( + "PLAINTEXT://evil.com/.allowed.example.com", "BAD_ARGUMENTS", "Unexpected character '/'", + id="path_does_not_bypass_host_regexp", + ), + # librdkafka splits the broker list on spaces too. Without doing the same, the filter would see + # the single host `evil.com .allowed.example.com`, the `host_regexp` allowlist entry would fully + # match it, and librdkafka would dial `evil.com`. + pytest.param("evil.com .allowed.example.com", "UNACCEPTABLE_URL", "evil.com:9092", id="space_separator"), + # librdkafka substitutes `localhost` for an empty host. + pytest.param(":9092", "BAD_ARGUMENTS", "Empty host", id="empty_host"), + # The broker list reaches librdkafka as a C string, so a NUL must not truncate it after the check. + pytest.param("evil.com\\0.allowed.example.com", "BAD_ARGUMENTS", "NUL", id="nul"), + # The validated entries are rebuilt and rejoined; the original text never reaches librdkafka. + pytest.param("localhost:19092 , localhost:19092", None, None, id="canonicalized"), + # An allowed host by the `host_regexp` allowlist entry. + pytest.param("sub.allowed.example.com", None, None, id="host_regexp_allowed"), +] + +# (NATS settings, expected error code or None for success, required error message part) +NATS_CASES = [ + pytest.param("nats_url = 'localhost:9999'", "UNACCEPTABLE_URL", "localhost:9999", id="forbidden_url"), + # libnats accepts `nats://user:password@host:port`; the filter must see host:port. + pytest.param( + "nats_url = 'nats://user:password@localhost:9999'", "UNACCEPTABLE_URL", "localhost:9999", + id="scheme_and_credentials", + ), + # One allowed server must not smuggle a forbidden one past the filter. + pytest.param( + "nats_server_list = 'localhost:14222,localhost:9999'", "UNACCEPTABLE_URL", "localhost:9999", + id="server_list_element_wise", + ), + # A bare host must be checked with libnats' default port 4222. + pytest.param("nats_url = 'localhost'", "UNACCEPTABLE_URL", "localhost:4222", id="default_port"), + # An allowed address must get past the filter and reach the connection attempt. `NATS` connects + # at `CREATE` time and nothing listens on the allowed port, so the statement fails - but with a + # connection error, not `UNACCEPTABLE_URL`. + pytest.param( + "nats_url = 'localhost:14222', nats_startup_connect_tries = 1", "CANNOT_CONNECT_NATS", None, + id="allowed_url_passes_filter", + ), + # libnats allows a `/path` after the port, so a path must not reach the filter as part of the host. + pytest.param( + "nats_url = 'nats://evil.com:4222/x.allowed.example.com'", "BAD_ARGUMENTS", "Unexpected character '/'", + id="path_does_not_bypass_host_regexp", + ), + # libnats substitutes `localhost` for an empty host. + pytest.param("nats_url = 'nats://:4222'", "BAD_ARGUMENTS", "Empty host", id="empty_host"), + # The URL reaches libnats as a C string, so a NUL must not truncate it after the check. + pytest.param("nats_url = 'evil.com\\0.allowed.example.com'", "BAD_ARGUMENTS", "NUL", id="nul"), + # The credentials must survive the rebuild of the validated address. + pytest.param( + "nats_url = 'nats://user:password@localhost:14222', nats_startup_connect_tries = 1", + "CANNOT_CONNECT_NATS", None, + id="credentials_preserved", + ), +] + + +def check(query, expected_error, message_part): + if expected_error is None: + node.query(query) + return + + error = node.query_and_get_error(query) + assert expected_error in error, error + if message_part is not None: + assert message_part in error, error + + +@pytest.mark.parametrize("broker_list, expected_error, message_part", KAFKA_CASES) +def test_kafka(started_cluster, broker_list, expected_error, message_part): + node.query("DROP TABLE IF EXISTS kafka_filtered SYNC") + check( + f""" + CREATE TABLE kafka_filtered (key UInt64, value UInt64) + ENGINE = Kafka + SETTINGS kafka_broker_list = '{broker_list}', + kafka_topic_list = 'topic', + kafka_group_name = 'group', + kafka_format = 'JSONEachRow' + """, + expected_error, + message_part, + ) + + +def test_kafka_config_override_is_validated(started_cluster): + """A broker list supplied by the server configuration must be validated too. + + `getConsumerConfiguration` seeds `metadata.broker.list` from the validated + `kafka_broker_list`, but the `` section of the server configuration (or a named + collection) is merged afterwards and can override it - here through the + `bootstrap.servers` alias. The merged value is validated again when the consumer is + created, so the `CREATE` (which sees only the allowed setting) succeeds and the read + fails on the forbidden override. + """ + node_kafka_override.query( + """ + CREATE TABLE kafka_override (key UInt64, value UInt64) + ENGINE = Kafka + SETTINGS kafka_broker_list = 'localhost:19092', + kafka_topic_list = 'topic', + kafka_group_name = 'group', + kafka_format = 'JSONEachRow' + """ + ) + error = node_kafka_override.query_and_get_error( + "SELECT * FROM kafka_override LIMIT 1" + " SETTINGS stream_like_engine_allow_direct_select = 1" + ) + assert "UNACCEPTABLE_URL" in error, error + assert "localhost:9999" in error, error + node_kafka_override.query("DROP TABLE kafka_override SYNC") + + +@pytest.mark.parametrize("settings, expected_error, message_part", NATS_CASES) +def test_nats(started_cluster, settings, expected_error, message_part): + node.query("DROP TABLE IF EXISTS nats_filtered SYNC") + check( + f""" + CREATE TABLE nats_filtered (key UInt64, value UInt64) + ENGINE = NATS + SETTINGS {settings}, + nats_subjects = 'subject', + nats_format = 'JSONEachRow' + """, + expected_error, + message_part, + ) diff --git a/tests/integration/test_mysql57_database_engine/configs/users.xml b/tests/integration/test_mysql57_database_engine/configs/users.xml index 4b6ba057ecb16..2f9966e1d71a2 100644 --- a/tests/integration/test_mysql57_database_engine/configs/users.xml +++ b/tests/integration/test_mysql57_database_engine/configs/users.xml @@ -4,6 +4,7 @@ default 1 + 1 diff --git a/tests/integration/test_mysql_database_engine/configs/users.xml b/tests/integration/test_mysql_database_engine/configs/users.xml index 4b6ba057ecb16..2f9966e1d71a2 100644 --- a/tests/integration/test_mysql_database_engine/configs/users.xml +++ b/tests/integration/test_mysql_database_engine/configs/users.xml @@ -4,6 +4,7 @@ default 1 + 1 diff --git a/tests/integration/test_mysql_protocol/test.py b/tests/integration/test_mysql_protocol/test.py index e604ec2f0da49..035227146fd4e 100644 --- a/tests/integration/test_mysql_protocol/test.py +++ b/tests/integration/test_mysql_protocol/test.py @@ -499,6 +499,156 @@ def test_mysql_replacement_query(started_cluster): ] +def test_mysql_replacement_query_injection(started_cluster): + # The MySQL handler rewrites a few client commands (SHOW TABLE STATUS LIKE, SET , + # KILL QUERY ) into ClickHouse SQL. The client-supplied argument must be parsed and re-quoted, + # never concatenated verbatim, otherwise a client can inject arbitrary SQL over the MySQL wire. + client = pymysql.connections.Connection( + host=started_cluster.get_instance_ip("node"), + user="default", + password="123", + database="default", + port=server_port, + ) + + # SHOW TABLE STATUS LIKE: a benign pattern reads only system.tables. + cursor = client.cursor(pymysql.cursors.DictCursor) + cursor.execute("SHOW TABLE STATUS LIKE 'one'") + rows = cursor.fetchall() + assert [r["Name"] for r in rows] == ["one"], rows + + # SHOW TABLE STATUS LIKE: an injected UNION must not leak system.users. The whole argument is + # parsed as one string literal, so the UNION tail becomes part of the LIKE pattern and matches + # nothing (the literal is unterminated -> parse fails -> empty pattern). + cursor.execute( + "SHOW TABLE STATUS LIKE '' " + "UNION ALL SELECT name,'','','',0,0,0,0,0,0,'',now(),now(),now(),'','','','' " + "FROM system.users" + ) + leaked = [r["Name"] for r in cursor.fetchall()] + assert "default" not in leaked, "SHOW TABLE STATUS LIKE injection leaked system.users: %s" % leaked + + # SHOW TABLE STATUS LIKE: a single trailing ';' is a statement terminator that programmatic + # MySQL clients pass through (the mysql CLI strips it). executeQuery treats it as end-of-query, + # so it must be accepted and produce the same result as without it. + cursor.execute("SHOW TABLE STATUS LIKE 'one'") + no_semicolon = [r["Name"] for r in cursor.fetchall()] + cursor.execute("SHOW TABLE STATUS LIKE 'one';") + with_semicolon = [r["Name"] for r in cursor.fetchall()] + assert with_semicolon == no_semicolon and "one" in with_semicolon, with_semicolon + + # A second statement after the ';' must still be rejected (only a single terminator is allowed), + # so the UNION tail cannot leak system.users. + cursor.execute( + "SHOW TABLE STATUS LIKE '' ; " + "UNION ALL SELECT name,'','','',0,0,0,0,0,0,'',now(),now(),now(),'','','','' " + "FROM system.users" + ) + leaked = [r["Name"] for r in cursor.fetchall()] + assert "default" not in leaked, "SHOW TABLE STATUS LIKE injection (trailing statement) leaked: %s" % leaked + + # SET : an injected value tail must be REJECTED, and a malformed input must not + # change session state. Before the fix "SET SQL_SELECT_LIMIT=1, max_threads=42" became + # "SET limit=1, max_threads=42" (injecting max_threads); an intermediate fix instead silently + # reset the mapped `limit` to DEFAULT. Now any tail that is not exactly "= " + # (with an optional single trailing ';') is rejected, leaving both `limit` and `max_threads` + # untouched. Use a fresh connection so the prior valid `limit` value is known to survive. + set_client = pymysql.connections.Connection( + host=started_cluster.get_instance_ip("node"), + user="default", + password="123", + database="default", + port=server_port, + ) + set_cursor = set_client.cursor(pymysql.cursors.DictCursor) + set_cursor.execute("SET SQL_SELECT_LIMIT=4242") + set_cursor.execute("SELECT changed FROM system.settings WHERE name = 'limit'") + assert set_cursor.fetchone()["changed"] == 1, "SET SQL_SELECT_LIMIT did not apply" + for injected in ( + "SET SQL_SELECT_LIMIT=1, max_threads=42", + "SET SQL_SELECT_LIMIT=1, max_threads=42;", + ): + with pytest.raises(pymysql.Error): + set_cursor.execute(injected) + set_cursor.execute("SELECT value FROM system.settings WHERE name = 'max_threads'") + assert set_cursor.fetchone()["value"] != "42", "SET replacement injected max_threads: %s" % injected + set_cursor.execute("SELECT changed FROM system.settings WHERE name = 'limit'") + assert set_cursor.fetchone()["changed"] == 1, "rejected SET silently reset `limit`: %s" % injected + set_client.close() + + # A normal SET still works (value is re-serialized, not dropped). + cursor.execute("SET SQL_SELECT_LIMIT=1234567") + cursor.execute("SET NET_WRITE_TIMEOUT=60") + cursor.execute("SET SQL_SELECT_LIMIT=DEFAULT") + + # SET name boundary: the prefix check is byte-wise, so a longer variable that merely starts + # with a mapped name (SQL_SELECT_LIMITED vs SQL_SELECT_LIMIT) must NOT be translated and must + # not touch the mapped `limit` setting. Without the boundary check the value parser failed on + # the "ED=1" tail and the fallback emitted "SET limit = DEFAULT", silently resetting limit. + # (`limit` reads back as 0 regardless of value, so assert on `changed`, not `value`.) Use a + # fresh connection so the earlier SETs on `client` do not bleed into this check. + boundary_client = pymysql.connections.Connection( + host=started_cluster.get_instance_ip("node"), + user="default", + password="123", + database="default", + port=server_port, + ) + boundary = boundary_client.cursor(pymysql.cursors.DictCursor) + boundary.execute("SET SQL_SELECT_LIMIT=4242") + boundary.execute("SELECT changed FROM system.settings WHERE name = 'limit'") + assert boundary.fetchone()["changed"] == 1, "SET SQL_SELECT_LIMIT did not apply" + try: + boundary.execute("SET SQL_SELECT_LIMITED=1") + except pymysql.Error: + pass # an unrelated variable passes through untranslated and errors as an unknown setting + boundary.execute("SELECT changed FROM system.settings WHERE name = 'limit'") + assert boundary.fetchone()["changed"] == 1, "SET SQL_SELECT_LIMITED reset the unrelated `limit` setting" + boundary_client.close() + + # SET with a trailing ';' must still apply the value. Before accepting the + # terminator, "SET SQL_SELECT_LIMIT=2;" failed the value parser and reset limit to DEFAULT. + # Use a fresh connection so `limit` starts unchanged and we can assert on `changed`. + semicolon_client = pymysql.connections.Connection( + host=started_cluster.get_instance_ip("node"), + user="default", + password="123", + database="default", + port=server_port, + ) + semicolon = semicolon_client.cursor(pymysql.cursors.DictCursor) + semicolon.execute("SET SQL_SELECT_LIMIT=2;") + semicolon.execute("SELECT changed FROM system.settings WHERE name = 'limit'") + assert semicolon.fetchone()["changed"] == 1, "SET SQL_SELECT_LIMIT=2; did not apply" + semicolon_client.close() + + # KILL QUERY: a numeric (possibly multi-digit) connection id is translated; previously the + # "^[0-9]" check silently dropped every multi-digit id. A single trailing ';' is accepted + # (programmatic clients pass it through), but a non-numeric tail or a second statement is not, + # so it stays injection-safe and never falls through to the unsupported raw "KILL QUERY ". + cursor.execute("KILL QUERY 12") + cursor.execute("KILL QUERY 12;") + with pytest.raises(pymysql.Error): + cursor.execute("KILL QUERY 12; SELECT 1") + cursor.execute("KILL QUERY WHERE query_id = 'mysql:0'") + + # KILL QUERY separator: the dispatcher matches the key "KILL QUERY" (no separator) but the + # handler slices at len("KILL QUERY ") (with the space). Without a separator check a malformed + # command would skip the non-space byte and be coerced into a valid cancel: "KILL QUERY;12" and + # "KILL QUERYx12" both reached the digit regex as "12". They must instead be rejected (passed + # through and erroring), so the id must be preceded by a real separator. + for malformed in ("KILL QUERY;12", "KILL QUERYx12", "KILL QUERY12"): + with pytest.raises(pymysql.Error): + cursor.execute(malformed) + + # SHOW TABLE STATUS LIKE separator: same dispatcher-key vs prefix-length mismatch. Without the + # separator check "SHOW TABLE STATUS LIKEx'one'" would skip the stray byte and look up 'one'. + # It must instead match nothing (the malformed command is not coerced into a lookup). + cursor.execute("SHOW TABLE STATUS LIKEx'one'") + assert [r["Name"] for r in cursor.fetchall()] == [], "SHOW TABLE STATUS LIKE accepted a missing separator" + client.close() + + def test_mysql_select_user(started_cluster): code, (stdout, stderr) = started_cluster.mysql_client_container.exec_run( """ diff --git a/tests/integration/test_postgresql_database_engine/configs/users.xml b/tests/integration/test_postgresql_database_engine/configs/users.xml index 4b6ba057ecb16..2f9966e1d71a2 100644 --- a/tests/integration/test_postgresql_database_engine/configs/users.xml +++ b/tests/integration/test_postgresql_database_engine/configs/users.xml @@ -4,6 +4,7 @@ default 1 + 1 diff --git a/tests/integration/test_postgresql_database_engine/test.py b/tests/integration/test_postgresql_database_engine/test.py index 042291c5731fc..a8aadabc7cae2 100644 --- a/tests/integration/test_postgresql_database_engine/test.py +++ b/tests/integration/test_postgresql_database_engine/test.py @@ -582,6 +582,46 @@ def test_backup_database(started_cluster): node1.query("DROP DATABASE backup_database") +def test_postgresql_database_engine_schema_sql_injection(started_cluster): + # `DatabasePostgreSQL::checkPostgresTable` embeds the configured `schema` into the PostgreSQL + # query it uses to check that a table exists. The query runs in a `pqxx::nontransaction`, i.e. + # over the libpq simple-query protocol, which executes `;`-separated statements, so a `schema` + # that terminates the literal early gets its own SQL executed on the PostgreSQL server as the + # role stored in the named collection. `schema` is overridable by default, so a user with only + # CREATE DATABASE plus use of the collection reaches this without CREATE TABLE or INSERT. + # + # The assertion is the absence of the marker table the payload tries to create: with the sink + # unescaped PostgreSQL really creates it, so this fails on an unfixed server rather than merely + # observing a different error message. + + # `database=False` connects to the default `postgres` database, which is the one the `postgres1` + # named collection points ClickHouse at. `pg_tables` lists only the current database's tables, + # so the assertion has to run in the database the payload would execute in. + conn = get_postgres_conn(started_cluster.postgres_ip, started_cluster.postgres_port) + cursor = conn.cursor() + cursor.execute("DROP TABLE IF EXISTS injected_marker") + + # `''` is a single quote in ClickHouse SQL; PostgreSQL therefore receives `public'; CREATE ...`. + payload = "public''; CREATE TABLE injected_marker (x integer); -- " + + node1.query("DROP DATABASE IF EXISTS postgres_injection") + # Both statements are allowed to fail: once the schema is escaped, the name simply does not + # exist, and what matters is only what did or did not happen on the PostgreSQL side. + node1.query_and_get_answer_with_error( + f"CREATE DATABASE postgres_injection ENGINE = PostgreSQL(postgres1, schema = '{payload}')" + ) + node1.query_and_get_answer_with_error("EXISTS TABLE postgres_injection.probe") + + cursor.execute("SELECT count(*) FROM pg_tables WHERE tablename = 'injected_marker'") + assert cursor.fetchall()[0][0] == 0, ( + "the schema value was executed as SQL by PostgreSQL: checkPostgresTable built its query " + "without escaping the schema" + ) + + node1.query("DROP DATABASE IF EXISTS postgres_injection") + cursor.execute("DROP TABLE IF EXISTS injected_marker") + + if __name__ == "__main__": cluster.start() input("Cluster created, press any key to destroy...") diff --git a/tests/integration/test_readonly_restricts_set_role/__init__.py b/tests/integration/test_readonly_restricts_set_role/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_readonly_restricts_set_role/configs/readonly_restricts_set_role.xml b/tests/integration/test_readonly_restricts_set_role/configs/readonly_restricts_set_role.xml new file mode 100644 index 0000000000000..80d7145131cdf --- /dev/null +++ b/tests/integration/test_readonly_restricts_set_role/configs/readonly_restricts_set_role.xml @@ -0,0 +1,5 @@ + + + true + + diff --git a/tests/integration/test_readonly_restricts_set_role/configs/readonly_restricts_set_role_absent.xml b/tests/integration/test_readonly_restricts_set_role/configs/readonly_restricts_set_role_absent.xml new file mode 100644 index 0000000000000..76b2225c37b3e --- /dev/null +++ b/tests/integration/test_readonly_restricts_set_role/configs/readonly_restricts_set_role_absent.xml @@ -0,0 +1,5 @@ + + + + + diff --git a/tests/integration/test_readonly_restricts_set_role/test.py b/tests/integration/test_readonly_restricts_set_role/test.py new file mode 100644 index 0000000000000..c36af86fa7203 --- /dev/null +++ b/tests/integration/test_readonly_restricts_set_role/test.py @@ -0,0 +1,116 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +# Every instance gets a copy of the production `config.xml`, which sets the element explicitly, so +# `node_off` removes it to run at the compatibility default an upgraded deployment has. +node_on = cluster.add_instance( + "node_on", main_configs=["configs/readonly_restricts_set_role.xml"] +) +node_off = cluster.add_instance( + "node_off", main_configs=["configs/readonly_restricts_set_role_absent.xml"] +) + + +@pytest.fixture(scope="module", autouse=True) +def started_cluster(): + try: + cluster.start() + + for node in (node_on, node_off): + node.query( + "CREATE ROLE ro_role, other_role;" + "CREATE USER probe IDENTIFIED WITH plaintext_password BY 'x';" + "GRANT ro_role, other_role TO probe;" + "ALTER USER probe DEFAULT ROLE ro_role;" + ) + + yield cluster + + finally: + cluster.shutdown() + + +def as_probe(node, sql, readonly): + return node.query_and_get_answer_with_error( + sql, user="probe", password="x", settings={"readonly": readonly} + ) + + +@pytest.mark.parametrize( + "readonly, statement", + [ + (1, "SET ROLE other_role"), + # The other `ASTSetRoleQuery::Kind` reaching the check. + (1, "SET ROLE DEFAULT"), + # `readonly = 2` is what the HTTP execution path sets for safe methods. + (2, "SET ROLE other_role"), + ], +) +def test_set_role_refused_when_enabled(readonly, statement): + _, error = as_probe(node_on, statement, readonly) + assert "READONLY" in error + + +def test_set_role_allowed_without_readonly(): + answer, error = as_probe(node_on, "SET ROLE other_role; SELECT currentRoles()", 0) + assert error == "" + assert answer.strip() == "['other_role']" + + +def test_set_role_allowed_at_default_settings(): + answer, error = as_probe(node_off, "SET ROLE other_role; SELECT currentRoles()", 1) + assert error == "" + assert answer.strip() == "['other_role']" + + +def in_session(node, session_id): + # Every request is a POST so the HTTP method contributes no `readonly` of its own + # (`setReadOnlyIfHTTPMethodIdempotent` exempts POST), leaving the session's own value as the + # only source. Reading the roles back needs the same session, which is why this is HTTP. + def post(sql): + return node.http_query_and_get_answer_with_error( + sql, method="POST", params={"session_id": session_id}, user="probe", password="x" + ) + + assert post("SET readonly = 1")[1] is None + _, error = post("SET ROLE other_role") + roles, roles_error = post("SELECT currentRoles()") + assert roles_error is None + return error, roles.strip() + + +def test_set_role_refused_mid_session(): + error, roles = in_session(node_on, "refused_mid_session") + assert error and "READONLY" in error + assert roles == "['ro_role']" + + +def test_set_role_allowed_mid_session_at_default_settings(): + error, roles = in_session(node_off, "allowed_mid_session") + assert error is None + assert roles == "['other_role']" + + +def test_http_role_parameter_not_restricted_when_enabled(): + # The `role` URL parameter reaches `Context::setCurrentRoles` without passing through + # `InterpreterSetRoleQuery`, so the restriction does not cover it. + def post(sql, **extra): + return node_on.http_query_and_get_answer_with_error( + sql, + method="POST", + params={"session_id": "http_role_parameter", **extra}, + user="probe", + password="x", + ) + + assert post("SET readonly = 1")[1] is None + answer, error = post("SELECT currentRoles()", role="other_role") + assert error is None + assert answer.strip() == "['other_role']" + # The parameter applies to one request, so the session keeps its default role. + answer, error = post("SELECT currentRoles()") + assert error is None + assert answer.strip() == "['ro_role']" diff --git a/tests/integration/test_refreshable_mv_no_multi_read/__init__.py b/tests/integration/test_refreshable_mv_no_multi_read/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_refreshable_mv_no_multi_read/configs/enable_keeper_multi_read.xml b/tests/integration/test_refreshable_mv_no_multi_read/configs/enable_keeper_multi_read.xml new file mode 100644 index 0000000000000..f63e3e288065f --- /dev/null +++ b/tests/integration/test_refreshable_mv_no_multi_read/configs/enable_keeper_multi_read.xml @@ -0,0 +1,36 @@ + + + 9181 + 1 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 5000 + 10000 + 0 + 0 + 0 + + + + 1 + 1 + + + + + 1 + localhost + 9234 + + + + + + + localhost + 9181 + + + diff --git a/tests/integration/test_refreshable_mv_no_multi_read/configs/enable_keeper_no_multi_read.xml b/tests/integration/test_refreshable_mv_no_multi_read/configs/enable_keeper_no_multi_read.xml new file mode 100644 index 0000000000000..983b3b065922b --- /dev/null +++ b/tests/integration/test_refreshable_mv_no_multi_read/configs/enable_keeper_no_multi_read.xml @@ -0,0 +1,36 @@ + + + 9181 + 1 + /var/lib/clickhouse/coordination/log + /var/lib/clickhouse/coordination/snapshots + + + 5000 + 10000 + 0 + 0 + 0 + + + + 0 + 1 + + + + + 1 + localhost + 9234 + + + + + + + localhost + 9181 + + + diff --git a/tests/integration/test_refreshable_mv_no_multi_read/configs/settings.xml b/tests/integration/test_refreshable_mv_no_multi_read/configs/settings.xml new file mode 100644 index 0000000000000..d15acfaa303bb --- /dev/null +++ b/tests/integration/test_refreshable_mv_no_multi_read/configs/settings.xml @@ -0,0 +1,8 @@ + + + + 1 + Etc/UTC + + + diff --git a/tests/integration/test_refreshable_mv_no_multi_read/test.py b/tests/integration/test_refreshable_mv_no_multi_read/test.py new file mode 100644 index 0000000000000..4113165c91371 --- /dev/null +++ b/tests/integration/test_refreshable_mv_no_multi_read/test.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 + +# Regression test for https://github.com/ClickHouse/ClickHouse/issues/106737 +# +# A coordinated refreshable materialized view (created in a Replicated database) used to +# crash the server when it was re-attached on a Keeper that does not support the MULTI_READ +# feature flag. The constructor's up-front MULTI_READ check was gated behind the fresh-CREATE +# path only, so on ATTACH/restore the view became coordinated=true and the scheduling thread +# later threw NOT_IMPLEMENTED in readZnodesIfNeeded, which the doScheduling catch-all re-raised +# as a LOGICAL_ERROR -> server abort (and crash-loop on restart). The view must instead stop +# gracefully and leave the server running. + +import os + +import pytest + +from helpers.cluster import ClickHouseCluster + +CURRENT_TEST_DIR = os.path.dirname(os.path.abspath(__file__)) + +cluster = ClickHouseCluster(__file__) + +# The keeper config is swapped at runtime (see use_keeper_config), so it is installed from the +# test body rather than as a main_config that the harness would also try to manage. +node = cluster.add_instance( + "node", + user_configs=["configs/settings.xml"], + stay_alive=True, + macros={"shard": 1, "replica": 1}, +) + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def use_keeper_config(config_name): + """Install one of the keeper configs as the single enable_keeper.xml on the node.""" + node.copy_file_to_container( + os.path.join(CURRENT_TEST_DIR, "configs", config_name), + "/etc/clickhouse-server/config.d/enable_keeper.xml", + ) + + +def test_refreshable_mv_attach_without_multi_read(started_cluster): + use_keeper_config("enable_keeper_multi_read.xml") + node.restart_clickhouse() + + node.query( + "CREATE DATABASE rdb ENGINE = Replicated('/clickhouse/rdb', '{shard}', '{replica}')" + ) + # Non-APPEND refreshable MV in a Replicated database is always coordinated. + node.query( + """ + CREATE MATERIALIZED VIEW rdb.mv + REFRESH EVERY 1 SECOND + ENGINE = ReplicatedMergeTree ORDER BY x + EMPTY + AS SELECT number AS x FROM numbers(3) + """ + ) + + # Refresh once while MULTI_READ is available, so coordination znodes exist and the view's + # persisted state is the normal coordinated one. + node.query("SYSTEM REFRESH VIEW rdb.mv") + node.query("SYSTEM WAIT VIEW rdb.mv") + assert node.query("SELECT count() FROM rdb.mv").strip() == "3" + + # Downgrade Keeper: disable MULTI_READ, then restart. On startup the Replicated database + # re-attaches rdb.mv (attach=true), which is exactly the path that used to crash. + use_keeper_config("enable_keeper_no_multi_read.xml") + node.restart_clickhouse() + + # The server must be up and answering queries (no crash, no crash-loop). + assert node.query("SELECT 1").strip() == "1" + + # Attach schedules the graceful-stop pass asynchronously, so the status read can otherwise + # observe the transient Scheduling state before doScheduling reaches the Disabled state. Wait + # for the scheduled pass to settle (WAIT VIEW returns immediately once the view is Disabled). + node.query("SYSTEM WAIT VIEW rdb.mv") + + # The view must have stopped gracefully, reporting the reason rather than aborting. + status = node.query( + "SELECT status, exception FROM system.view_refreshes WHERE view = 'mv'" + ) + assert "Disabled" in status, status + assert "multi-read" in status.lower() or "multi_read" in status.lower(), status + + # The doScheduling catch-all LOGICAL_ERROR must not have fired. + assert not node.contains_in_log("Unexpected exception in refresh scheduling") + + # The gracefully-stopped state must be non-resumable: a coordinated view must never be turned + # into an uncoordinated local refresh (that would corrupt the replicated target table). So + # SYSTEM START VIEW must NOT resume it while MULTI_READ is still missing - it stays Disabled, + # runs no refresh, and the server stays up. + node.query("SYSTEM START VIEW rdb.mv") + node.query("SYSTEM REFRESH VIEW rdb.mv") + assert node.query("SELECT 1").strip() == "1" + # SYSTEM REFRESH VIEW is async: run() moves the task to Scheduling, and only the background + # scheduler later hits the coordination.unavailable branch and switches it back to Disabled. + # Wait for that scheduled pass before asserting, so the read never observes transient Scheduling. + node.query("SYSTEM WAIT VIEW rdb.mv") + status = node.query( + "SELECT status FROM system.view_refreshes WHERE view = 'mv'" + ).strip() + assert status == "Disabled", status + assert not node.contains_in_log("Unexpected exception in refresh scheduling") + + # Restoring MULTI_READ and restarting must keep the server healthy. + use_keeper_config("enable_keeper_multi_read.xml") + node.restart_clickhouse() + assert node.query("SELECT 1").strip() == "1" + + node.query("DROP DATABASE rdb SYNC") diff --git a/tests/integration/test_rename_column/test.py b/tests/integration/test_rename_column/test.py index 870a2f0f21074..b6034365cd51b 100644 --- a/tests/integration/test_rename_column/test.py +++ b/tests/integration/test_rename_column/test.py @@ -130,6 +130,14 @@ def drop_distributed_table(node, table_name): time.sleep(1) +def sync_distributed_table(node, nodes_to_sync, table_name): + # The spool must drain before a replica has anything to fetch, so the order matters. + # Any replica can serve a shard leg under load_balancing=RANDOM, so all are synced. + node.query("SYSTEM FLUSH DISTRIBUTED {}".format(table_name)) + for node_to_sync in nodes_to_sync: + node_to_sync.query("SYSTEM SYNC REPLICA {}_replicated".format(table_name)) + + def insert( node, table_name, @@ -269,6 +277,37 @@ def rename_column_on_cluster( break +def _num2_converged(nodes, table_name): + return all( + node.query( + "SELECT count() FROM system.columns " + "WHERE database = 'default' AND table IN ('{t}', '{t}_replicated') " + "AND name = 'num2'".format(t=table_name) + ).strip() + == "2" + for node in nodes + ) + + +def wait_for_rename_to_num2(nodes, table_name, attempts=12, poll_seconds=5): + # Best-effort cleanup renames + async ON CLUSTER replication can leave the + # column as foo2/foo3 on some replicas. Re-drive the idempotent rename-back + # until num2 is present on every node's Distributed and _replicated tables, + # so the strict queries below do not race schema convergence (UNKNOWN_IDENTIFIER). + tables = [table_name, "%s_replicated" % table_name] + for _ in range(attempts): + if _num2_converged(nodes, table_name): + return + for old_name in ("foo2", "foo3"): + for table in tables: + rename_column_on_cluster(nodes[0], table, old_name, "num2", 1, True) + time.sleep(poll_seconds) + if not _num2_converged(nodes, table_name): + raise Exception( + "columns did not converge to num2 for {} on all nodes".format(table_name) + ) + + def alter_move(node, table_name, iterations=1, ignore_exception=False): i = 0 while True: @@ -732,11 +771,13 @@ def test_rename_distributed(started_cluster): try: create_distributed_table(node1, table_name) insert(node1, table_name, 1000) + sync_distributed_table(node1, nodes, table_name) rename_column_on_cluster(node1, table_name, "num2", "foo2") rename_column_on_cluster(node1, "%s_replicated" % table_name, "num2", "foo2") insert(node1, table_name, 1000, col_names=["num", "foo2"]) + sync_distributed_table(node1, nodes, table_name) select(node1, table_name, "foo2", "1998\n", poll=30) finally: @@ -809,14 +850,7 @@ def test_rename_distributed_parallel_insert_and_select(started_cluster): for task in tasks: task.get(timeout=240) - rename_column_on_cluster(node1, table_name, "foo2", "num2", 1, True) - rename_column_on_cluster( - node1, "%s_replicated" % table_name, "foo2", "num2", 1, True - ) - rename_column_on_cluster(node1, table_name, "foo3", "num2", 1, True) - rename_column_on_cluster( - node1, "%s_replicated" % table_name, "foo3", "num2", 1, True - ) + wait_for_rename_to_num2(nodes, table_name) insert(node1, table_name, 1000, col_names=["num", "num2"]) select(node1, table_name, "num2") diff --git a/tests/integration/test_replicated_database/test.py b/tests/integration/test_replicated_database/test.py index 1315309b55d38..783121399dc06 100644 --- a/tests/integration/test_replicated_database/test.py +++ b/tests/integration/test_replicated_database/test.py @@ -1369,7 +1369,7 @@ def test_replicated_table_structure_alter(started_cluster): ) competing_node.query("CREATE TABLE table_structure.mem (n int) ENGINE=Memory") - dummy_node.query("DETACH DATABASE table_structure") + dummy_node.query("DETACH DATABASE table_structure SYNC") settings = {"distributed_ddl_task_timeout": 0} main_node.query( @@ -1378,7 +1378,7 @@ def test_replicated_table_structure_alter(started_cluster): ) competing_node.query("SYSTEM SYNC DATABASE REPLICA table_structure") - competing_node.query("DETACH DATABASE table_structure") + competing_node.query("DETACH DATABASE table_structure SYNC") main_node.query( "ALTER TABLE table_structure.rmt ADD COLUMN m int", settings=settings diff --git a/tests/integration/test_s3_redirect_remote_host_filter/__init__.py b/tests/integration/test_s3_redirect_remote_host_filter/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_s3_redirect_remote_host_filter/configs/config.xml b/tests/integration/test_s3_redirect_remote_host_filter/configs/config.xml new file mode 100644 index 0000000000000..53b2680c703cc --- /dev/null +++ b/tests/integration/test_s3_redirect_remote_host_filter/configs/config.xml @@ -0,0 +1,10 @@ + + + + resolver:8080 + + + 1 + diff --git a/tests/integration/test_s3_redirect_remote_host_filter/s3_endpoint/endpoint.py b/tests/integration/test_s3_redirect_remote_host_filter/s3_endpoint/endpoint.py new file mode 100644 index 0000000000000..19485a527b5a3 --- /dev/null +++ b/tests/integration/test_s3_redirect_remote_host_filter/s3_endpoint/endpoint.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 +# Malicious S3 endpoint used to test that the AWS-SDK 301 redirect path in +# Client::getURIFromError validates the redirect target against RemoteHostFilter +# (SSRF protection), mirroring the Poco 307 path. +# +# Every S3 request is answered with 301 Moved Permanently whose Location points at +# this same container's IP address, reached under a name (the raw IP) that is NOT in +# . A correctly-behaving server rejects that target with +# UNACCEPTABLE_URL before connecting. If the redirect were followed, the rewritten +# request would land on /forbidden_hit and flip the "followed" flag -- which the test +# asserts never happens. +import socket + +from bottle import response, route, run + +# Resolve our own container IP. It is reachable (same container) but is a different +# "host" from the allow-listed name "resolver", so the host filter must deny it. +OWN_IP = socket.gethostbyname(socket.gethostname()) +REDIRECT_TARGET = OWN_IP + ":8080" + +followed_redirect = {"hit": False} + + +@route("/forbidden_hit/<_path:path>", ["GET", "POST", "PUT", "HEAD", "DELETE"]) +def forbidden(_path): + # Reached only if ClickHouse followed the 301 to the disallowed target (the bug). + followed_redirect["hit"] = True + response.status = 200 + return "SHOULD_NOT_BE_REACHED" + + +@route("/followed") +def followed(): + return "YES" if followed_redirect["hit"] else "NO" + + +@route("/<_bucket>", ["GET", "POST", "PUT", "HEAD", "DELETE"]) +@route("/<_bucket>/<_path:path>", ["GET", "POST", "PUT", "HEAD", "DELETE"]) +def server(_bucket, _path=""): + suffix = _bucket if not _path else _bucket + "/" + _path + response.set_header("Location", "http://" + REDIRECT_TARGET + "/forbidden_hit/" + suffix) + response.status = 301 + return "Redirected" + + +@route("/") +def ping(): + return "OK" + + +run(host="0.0.0.0", port=8080) diff --git a/tests/integration/test_s3_redirect_remote_host_filter/test.py b/tests/integration/test_s3_redirect_remote_host_filter/test.py new file mode 100644 index 0000000000000..799040e85616c --- /dev/null +++ b/tests/integration/test_s3_redirect_remote_host_filter/test.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +# Regression test for the AWS-SDK 301 redirect SSRF: a malicious/compromised +# S3-compatible endpoint returns 301 Moved Permanently with a Location header (or +# XML) pointing at an arbitrary host. Client::getURIFromError must validate +# that target against RemoteHostFilter (like the Poco 307 path already does) instead of +# blindly following it. Without the fix the s3() query reaches the disallowed host +# (SSRF to internal services / cloud metadata); with the fix it fails with UNACCEPTABLE_URL. +import logging +import os +import time + +import pytest + +from helpers.cluster import ClickHouseCluster + + +def run_endpoint(cluster): + logging.info("Starting custom S3 endpoint") + container_id = cluster.get_container_id("resolver") + current_dir = os.path.dirname(__file__) + cluster.copy_file_to_container( + container_id, + os.path.join(current_dir, "s3_endpoint", "endpoint.py"), + "endpoint.py", + ) + cluster.exec_in_container(container_id, ["python", "endpoint.py"], detach=True) + + # Wait for S3 endpoint start + num_attempts = 100 + for attempt in range(num_attempts): + ping_response = cluster.exec_in_container( + cluster.get_container_id("resolver"), + ["curl", "-s", "http://resolver:8080/"], + nothrow=True, + ) + if ping_response != "OK": + if attempt == num_attempts - 1: + assert ping_response == "OK", 'Expected "OK", but got "{}"'.format( + ping_response + ) + else: + time.sleep(1) + else: + break + + logging.info("S3 endpoint started") + + +@pytest.fixture(scope="module") +def cluster(): + try: + cluster = ClickHouseCluster(__file__) + cluster.add_instance( + "node", + main_configs=["configs/config.xml"], + with_minio=True, + ) + logging.info("Starting cluster...") + cluster.start() + logging.info("Cluster started") + + run_endpoint(cluster) + + yield cluster + finally: + cluster.shutdown() + + +def _followed(cluster): + return cluster.exec_in_container( + cluster.get_container_id("resolver"), + ["curl", "-s", "http://resolver:8080/followed"], + nothrow=True, + ) + + +def test_301_redirect_target_is_host_filtered(cluster): + node = cluster.instances["node"] + + # The s3() endpoint (resolver:8080) is allow-listed, but it 301-redirects to its own + # raw IP, which is NOT allow-listed. The redirect target must be rejected. + error = node.query_and_get_error( + "SELECT * FROM s3('http://resolver:8080/bucket/key', 'TSV', 'x String') " + "SETTINGS s3_max_redirects=5" + ) + assert "not allowed in configuration file" in error, ( + "expected RemoteHostFilter to reject the 301 redirect target, got: " + error + ) + + # And the server must not have sent a single request to the disallowed target. + assert _followed(cluster) == "NO", "ClickHouse followed the 301 to a disallowed host (SSRF)" diff --git a/tests/integration/test_ssh/test.py b/tests/integration/test_ssh/test.py index 8f200a53a8450..3b109bedef8cd 100644 --- a/tests/integration/test_ssh/test.py +++ b/tests/integration/test_ssh/test.py @@ -1,6 +1,7 @@ import os import re import socket +import struct import subprocess import time @@ -311,3 +312,138 @@ def test_ssh_interactive_pty_with_high_fds(started_cluster): s.close() except OSError: pass + + +# Matches SGR / cursor-control sequences, OSC sequences, and NUL bytes that +# replxx interleaves throughout the PTY output stream. +ANSI_ESCAPE_RE = re.compile(rb"\x1b\[[0-9;?]*[A-Za-z]|\x1b\][^\x07]*\x07|\x00") + + +def _read_channel_until(channel, timeout, marker=None): + """Read PTY output for up to `timeout` seconds and return it with ANSI + escapes stripped. When `marker` is given, return as soon as it appears + in the cleaned output.""" + buf = b"" + deadline = time.time() + timeout + while time.time() < deadline: + if channel.recv_ready(): + buf += channel.recv(65536) + if marker is not None and marker.encode() in ANSI_ESCAPE_RE.sub(b"", buf): + break + else: + time.sleep(0.05) + return ANSI_ESCAPE_RE.sub(b"", buf).decode(errors="replace") + + +def test_interactive_session_torn_down_with_a_dead_pty(started_cluster): + """Losing the pty while the embedded client is shutting down must not kill the server. + + `ReplxxLineReader::~ReplxxLineReader` writes an escape sequence to the + terminal to reset cursor blinking when overwrite mode was ever enabled. + `Replxx::print` throws `std::runtime_error("write failed")` when that write + does not go through, and a destructor is implicitly `noexcept`, so the + exception used to `std::terminate` the whole server process. + + Reproduce it the way a real disconnect does: turn overwrite mode on with + the `Insert` key, then drop the TCP connection with a RST so that the + server's side of the pty is gone by the time the line reader is destroyed. + """ + # The daemon watchdog restarts the server after `std::terminate`, so "the + # server answers queries again" is not evidence of anything, and neither is + # a growing `uptime()`: on a fresh cluster the uptime before the disconnect + # is only a few seconds, so a restarted server reaches a larger value well + # within the sampling window below. Pin the process identity instead: the + # watchdog restarts the server by forking a new child, so the set of + # `clickhouse-server` pids in the container changes and cannot recover. + # The pattern is anchored at argv0 so that the shell running `pgrep` (whose + # own command line contains the pattern) does not match itself. + def server_pids(): + return instance.exec_in_container( + ["bash", "-c", "pgrep -f '^[^ ]*clickhouse(-| )server' | sort -n"], + nothrow=True, + ).split() + + pids_before = server_pids() + assert pids_before, "no `clickhouse-server` process in the container" + uptime_before = float(instance.query("SELECT uptime()").strip()) + + pkey = paramiko.Ed25519Key.from_private_key_file(f"{SCRIPT_DIR}/keys/lucy_ed25519") + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + client.connect( + hostname=instance.ip_address, + port=9022, + username="lucy", + pkey=pkey, + timeout=30, + ) + try: + channel = client.invoke_shell(term="xterm", width=80, height=24) + channel.settimeout(20) + output = _read_channel_until(channel, timeout=20, marker=":) ") + assert ":) " in output, f"no prompt from the embedded client: {output!r}" + + # `Insert` toggles overwrite mode, which is what makes the destructor + # print the "reset cursor blinking" sequence in the first place. Wait + # for the raw `\033[5 q` ("blinking cursor") escape the key handler + # prints: it is the only observable proof that the server really + # consumed the key and that `overwrite_mode` became true. Without it + # the destructor writes nothing and the test would pass even unfixed. + channel.sendall("\x1b[2~") + raw = b"" + deadline = time.time() + 10 + while time.time() < deadline and b"\x1b[5 q" not in raw: + if channel.recv_ready(): + raw += channel.recv(65536) + else: + time.sleep(0.05) + assert ( + b"\x1b[5 q" in raw + ), f"overwrite mode was not enabled, the destructor would write nothing: {raw!r}" + + # Abort the connection with a RST instead of a graceful shutdown, so + # writes on the server side fail rather than being silently discarded. + sock = client.get_transport().sock + sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack("ii", 1, 0)) + sock.close() + finally: + client.close() + + # The session teardown is asynchronous, so keep sampling for a while. A + # sample that fails only means the teardown is still in flight, but the + # *last* sample of the window must succeed and must still show the very + # same process: the server that survived the disconnect, not a fresh one. + last_failure = None + last_uptime = None + deadline = time.time() + 30 + while time.time() < deadline: + try: + uptime = float(instance.query("SELECT uptime()").strip()) + except Exception as e: # down or restarting — the next sample decides + last_failure = e + last_uptime = None + time.sleep(0.5) + continue + last_uptime = uptime + pids = server_pids() + assert pids == pids_before, ( + "the server process was replaced after the SSH disconnect, i.e. it " + "died while tearing the session down and was started again " + f"(pids {pids} != {pids_before})" + ) + assert uptime >= uptime_before, ( + "the server restarted after the SSH disconnect, i.e. it died while " + f"tearing the session down (uptime {uptime} < {uptime_before})" + ) + time.sleep(0.5) + + assert last_uptime is not None, ( + "the server did not answer a query at the end of the 30 s window after " + f"the SSH disconnect, i.e. it died while tearing the session down: {last_failure}" + ) + + # `from_host=True` also greps the rotated logs, so a restart cannot hide the + # fatal line by rotating it out of the active `clickhouse-server.log`. + assert not instance.contains_in_log( + "std::terminate", from_host=True + ), "the server called `std::terminate` while tearing down the SSH session" diff --git a/tests/integration/test_ssl_cert_authentication/certs/client13-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/client13-cert.pem new file mode 100644 index 0000000000000..82687debfcdec --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client13-cert.pem @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIIFPzCCAyegAwIBAgIUb8W/doRsWnXgJSqDgDz8Yb+cLVwwDQYJKoZIhvcNAQEL +BQAwUjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjAwMTAx +MDAwMDAwWhcNMjkxMjI5MDAwMDAwWjBhMQswCQYDVQQGEwJSVTETMBEGA1UECAwK +U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMRow +GAYDVQQDDBFjbGllbnQxAC5ldmlsLmNvbTCCAiIwDQYJKoZIhvcNAQEBBQADggIP +ADCCAgoCggIBAOMBJ2yqK72eew557aeGID0RXFzVs6lLTN4/71C4/309gAcIoqsw +Phulwzj3IfH7v2PPntsROxGtRCVS5LWarXUTEviEaiWUp1MwoFlrhn2Dm78hIgYC +7NrdtI48tb9t9y6eTio99WdIiXGY2KNlctfnTyCxKeanZi7JZUmb9VW95gh/5dl3 +sajZPjUDrFxqUQbzsSzbgElFkjvOzN5xEoUM1qbjHcyML7aKweXJKhSswdi4K3oA +v/FbFleJQgoRPB43JDqc5YjpPARKJlt5VZqpkgiKFAm0R+o4TYeCBd0kZFtL+7Y3 +TO27ZdQiOFKR/DINPCdA3/nxtQ7GJ17g/nwXLQRKUDxXain7jo+87OHghMA8WfwQ +lMSkQ+sqJdxA/VunQEHNCljw64BHeuU6CqQDU9eEnFNq9+3Qb0INiMRn6EGj20LK +5+q/fJqGm/N8BcuoYHUymbtojAjnJmCIWwvTzo7OYaG84nRwOPkZZvcSOrzn2CbT +zhmXHiSnP7ZAL/r24ZW5oikjw6Ie40iqQ2MyknTv3Ko1LWIKkjAbMcE6fo3SlzMA +fVsm9wAyVa/UyjO1TcnPNNta1TdqSgdRsEPDvR/IMPQYJ7Be0H0SZLj2/v8ZmHmZ +KVfGKdWnFBQKiTSHVEBHBdXADcXi2grYjnOBKnQ7H2UQK9jfGMu4FPYlAgMBAAEw +DQYJKoZIhvcNAQELBQADggIBAF4piZQxvyxwfhQxmNbyc8EtUo5qBIWE026e5Hsl +4QX3glKhAo+xLEQbKAImsMdXXYNdGaBJDTEZRbGSvyEgVa8e4YF0FP+CkFDcM0oF +k7YUgQHkZvlR/lsd5idMM7JJ1WXnux6jay9hpF9GEi2QMB2QkwuRwBHkgtjEun8C +qdyvApSecY6BXQllAJyoalSVR4ZzaMGVgSGL99RKyn320icevO6WDOVocEiohJTK +KOwf8q9G6riKWsRbNNPhkt9QwZzC0/RuKV6r25QR7H+GSjmllcBnbyqW3+DGGuJL +iMtKrCmx4xHedvDXRxi8ZxsfIrWVI7f5hgcWCXFWFMeqkU9nD3TGJrAuamBBQMZn +aC3q9AyVudroJs/1ojrSD92+H3qSzhqoLZ4bd531ShPasQ/EHIiOeN794j3Rxo0d +ituD2LSboe2Zr4TNhi7ou/dSfDXPEwJSb5V1Xgp9OyhQCKnFa+RkbQVtME5dVBMR +oO4HMTqaCgYSzCKhrxIO/vid2p6NQp7ZI8V2hAhUR4aBwnWFFspmuWaXpZqXxE6+ +a46rTA4HLpQ7TKwbi4fS2zWSh64nCHmv72GOHQ86dlXDNvfqLo8vyz79chKPo+31 +fsTZJks9wBzeSsq0tqAWflN/hQQCZJT1IpIc5VwU45FLSraKBGYRpCP4FnciXUuJ +RiyF +-----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client13-key.pem b/tests/integration/test_ssl_cert_authentication/certs/client13-key.pem new file mode 100644 index 0000000000000..452e632233838 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client13-key.pem @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKAIBAAKCAgEA4wEnbKorvZ57Dnntp4YgPRFcXNWzqUtM3j/vULj/fT2ABwii +qzA+G6XDOPch8fu/Y8+e2xE7Ea1EJVLktZqtdRMS+IRqJZSnUzCgWWuGfYObvyEi +BgLs2t20jjy1v233Lp5OKj31Z0iJcZjYo2Vy1+dPILEp5qdmLsllSZv1Vb3mCH/l +2XexqNk+NQOsXGpRBvOxLNuASUWSO87M3nEShQzWpuMdzIwvtorB5ckqFKzB2Lgr +egC/8VsWV4lCChE8HjckOpzliOk8BEomW3lVmqmSCIoUCbRH6jhNh4IF3SRkW0v7 +tjdM7btl1CI4UpH8Mg08J0Df+fG1DsYnXuD+fBctBEpQPFdqKfuOj7zs4eCEwDxZ +/BCUxKRD6yol3ED9W6dAQc0KWPDrgEd65ToKpANT14ScU2r37dBvQg2IxGfoQaPb +Qsrn6r98moab83wFy6hgdTKZu2iMCOcmYIhbC9POjs5hobzidHA4+Rlm9xI6vOfY +JtPOGZceJKc/tkAv+vbhlbmiKSPDoh7jSKpDYzKSdO/cqjUtYgqSMBsxwTp+jdKX +MwB9Wyb3ADJVr9TKM7VNyc8021rVN2pKB1GwQ8O9H8gw9BgnsF7QfRJkuPb+/xmY +eZkpV8Yp1acUFAqJNIdUQEcF1cANxeLaCtiOc4EqdDsfZRAr2N8Yy7gU9iUCAwEA +AQKCAgAhsUz9M2q866tUaY4iXxAk5ZFBvtQ6NztzAfv5YrlFCPOaLP1xOX6rvYLd +eczCLP0/Xr8IwSTKdF7uHUC7Svhzgn++fsTUdse5Bg7JXDdregphUr/JMvZIa+n9 +xzlV7t07wEerZw59Txuwo8uFFlchKtX97QhiPDa4kLAlve4AKmiHRW3DmnY2MKcA +2Qk5nuwPg0Go/fFWBBX20N8WevjmjOfFKxDZXctCeujZkW3bVg2K0ZcJC4McWx/U +oUI52qIDfWgKrQ6NL3T20VyibL64bx0GLccDQ7dS0RrXbISF9Kin5lNiEbuAzjTd +yLB3DO+TgY8L58n2lUSsKzFdGI7yFjJTr6yzmKIayS11c00TCBQ0w/l+oCWOhrvM +zn/k3mrORcRA1aqX5mcqAdzs6hYFIyFbKXNH8hlE7JnUUbcvR76WbzMl6IxPxMT+ +ikbTIQnG+T2PMgmaxVbe9YhyHT2YbMHF7igocz5mI0PHsdNaB1IBSV1G2Knq8QJi +rbYUdCShTJksrNIXghyjbCPx5VIo05fx8NleCuNND18WPs32tI2rceLlF3KSk8GF +moCnPJFTtEgk5IvzWavOGO+abkU1A9xGOhNHCMZVxOca/ZGlhHSRBCuhagpLPx9e +QprxfFVIFG8jNrN0/Mhgv3ivKmbYZuv03wdcWDtKCdMrxkah9wKCAQEA8Xp+V4zk +Ta4nqkjL4TTv+uczszGt/Kb5OYZyb1EzUseabRyOFkb2iMUf+Ol/92UMBlh/FzTa +RFHE1lRed45l8Jhgsbaq9r8gmJZ1WZM2IUByUI++z+USlEtG8O+8wlniyTn9Fc+n +y+wkZwSeJSLWZAWIzeygJs/ceKG+4TpnLZnC2rQqOoKGc+0Dimc1E5HV3mOkhwe1 +CtiYOUvgNf0rEV7E5D7smjpAeiJ8xPhYRg0XXLztN+9rQA+6AvwpbI3VnyAKbPUQ +/J+ZVg0cVMH7pTDXMj0jB2wevrIa7xj2Pn4w/jR8i/tNjFauMMvnofr6RW6KqYk/ +opXhfSE1Pk9w6wKCAQEA8KfWOQEmKlC0KJrN1/D1ZchNZf57pNFtjyWdjLUah05k +a0QLUe1yme7lTw59tsFIfrd/ozeiWKSvhum9/mUNvwBgPjnqp73pmrEEu1bgQ8Ib +cCm55GAmBWlNMO52iD+EPdsYvj+KgAWxu6W8B6XsyjoVLBYqHwNzitZlW9D1PAbY +KWBppmntVAhj7oNvk40uWEFIuTU0o0RIrvcYmX795sr2PY69ozOdiY6UziR4oRAs +DhJ+0a8Sjct0/RMq+sOpBONrpOBHqJzoCX+/alOjbJ8FYIMuc6OU2cNtcxjL3tcZ +cJKInsMl7bMYhi0SIEnttuAgh6mzD4Iox1WX3hbxLwKCAQEAySWPHmCnQCnEsqzG +38aX7DkwsKC+XEm+KnPa2O0bwmWwNhlmJhpgfBcwBciDJtHODW8kFnGZKvWt8BcB +RbehJKPZT89oY/dbMJ+MCtx8Z4BmeML0X/ph2pNF+abJZl83cQVT0xpRnKUMwZ9w +GGEHkvOlPFtSIGJfNUEOXlCm565ASKtwzaIyW1hf7acA6Fc/fmsj/rKl1O4NBxU0 +I+TKPkLh4Xqk3eeOE+6sXeq8pUV8Y2ygcUqG3Sr8eEYSP3F6M32eEZql1rkfIjXz +loqrkrO3XgrdJe6jQZfcpbP2UqINL4MLCBOCSldd7Bm9zgjg9nsZGBXSox0UYoYJ +8uh7lwKCAQA+I3gi+/L41iHOojooWeVjRJcHkPAoHJNndNT7cf/JlCpFsCokG2WN +7at0AE/hkoK/hW4FnOXkcZGJCm2udDVabiRrrNS0P0tEUBTisonxtPsUuRFwsIrg +ttHhopEkmRHyTtJSvWFrsQy1YRPt/Z/oj5rL6WUy4NdCsB032fqYZ0QFWwmsmIlZ +O1liSrnSpY+j5id6+wv+ZDFITDEj4TB6GUn/lw3MSBWTYSd3Gt+y5tQZRhlM1yG5 +TyGD/yEH4uGPi5FN30NhfDJF0aCBOdtmvqDKzNR/s2tJ1zY5k9uATJYbBRsVs1nl +yGq6qoSVpcEliTWdEepURM12utkd1VqtAoIBAFLriLD2AF0+ayheiDFgsbRo78V3 +dygUPBZi7b6nPWv3gugnwvWrTFaCAndACxNZPWNwEcING2Utjp8PTvPTzad9yibl +3xqNSKR9gaqmDYPEY1DwYGCpHRc969PIYcyK6z3dICj1ySEo0RwTa+Ab0OHeRWzS +ROnmmy9a9Vpx7KyMZcmOGhSXL3q7nr2imI+qZ4tGiEJts85d2fXAVhGeBOagzy5n +uD00wYzBc8rZ9u+d7ZOyQetgoJBnhyQNzW7IBCN+Z3EWW5lPlHQJpTXjHCdXgiZU +28pX/4J/mIcaURA0xhexuuLqTEjLlFoAnjNjWCCXv0u5uP8Ba35g4BSAz0o= +-----END RSA PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client14-cert.pem b/tests/integration/test_ssl_cert_authentication/certs/client14-cert.pem new file mode 100644 index 0000000000000..25c2687c0edf6 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client14-cert.pem @@ -0,0 +1,31 @@ +-----BEGIN CERTIFICATE----- +MIIFazCCA1OgAwIBAgIUTgWFBpS4cJ/+Fgg5jAfOubkwokowDQYJKoZIhvcNAQEL +BQAwUjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM +GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjAwMTAx +MDAwMDAwWhcNMjkxMjI5MDAwMDAwWjBmMQswCQYDVQQGEwJSVTETMBEGA1UECAwK +U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMR8w +HQYDVQQDDBZldmlsAC5jb3JwLmV4YW1wbGUuY29tMIICIjANBgkqhkiG9w0BAQEF +AAOCAg8AMIICCgKCAgEAs6uGg+vk4GlmN+ZlJb9gUI6AS6TJVQvLCSw7RPHt0k8x +qMXeoHid3mmVIC7tWK6BDEbAWJB814cWzOviXy5GbuUZB5Yq1MR0bQV968PBa/jp +/X6eDAW3KDfvgsk2eBo+aZqbFh1EJqXzN04d255X2mlAGpwW97T+XhXa/ahxj1Ey +G5smL6jaybcrKj/Xce4qVgSQk0kVlun/35Rfd6kcFTdq+Laap3nO0JKBYwfcoeMU +vVfRZV0iaP7sbOpiZlv0+zFDDGX2Xi18VH2xeclICN4ar8K0h+VNE1uqf27r+3NA +iZbZphA2huN60KYP+IV09KirWhFM3Xcd5LlqbluRUwWxp4xXZNWFIa6AeSbuC+43 +hAa0vBfN9+CE03ifX6stqfUy/eiJsghuPEZOUXqJpBpz4VHpxXVqiLX11hrGw60S +rEQJ30SVahULl4v1/zhFaRgFnZE2yRdQG0h1AejA5OUykKTH5vAkfZRxPmL3hnBB +fg+iti93EFvdC2bXNMB04X+VkvF8CcqRUr2Ka7ZpfleJuO9ps2e0LBtjSJOLxYRA +N13RUMp9m/45F5iGE4QXdUFzOrasrclg8KfP0Zl+6/FQcygV4bLQywqm/zgCX1zC +SAb88JLEElIUmyMFT/cvst1uscy1fvoLseuKcz+9woAF1LA8XrZzMPRA22xSABMC +AwEAAaMlMCMwIQYDVR0RBBowGIIWZXZpbAAuY29ycC5leGFtcGxlLmNvbTANBgkq +hkiG9w0BAQsFAAOCAgEAFuyPgncpYKGYffQPoOWR2yTn6L56XxvQcFNrchyKkwAn +zsBQFLOX4T4z1FuIQTwwKiSRwz+hwuIfcK2++I1yyxgYzw8Cxw7uqpvRi6AgN9pk +LEsXbSOvqaQyFEukVFenbkIejBVexF74gk2LKJYnT7MRVgs/k8C5Lqy5lKpB/2Gk +7xnzVe6mLhdZrMRbtCmt+Pr2m5l+XMGucnoaNCK87zSIGEt8fCYNB//vQ61GeANv +hRYiWgdWcOCnl2ip+EWbq0+ZK2JvJ8Ih2d0S+Qtbhek8XQsuCoa5P3YjLyAfzhpk +/CZb+VbOH41qZ2uWi05xVE3IRbJBlRDnX/bXC4Nx2ojzOnWAqzmrJffsKal/04oP +BcA6dOzm+CvT7AlzImlRI/6ZEC6DlrXSkvm5yRNwdUhTZ5ZjQnSyZBF4lNY3u9Sa +G1ipxyR9wRcA7uac8cFv/MBAGP5godwi3WPvqC/7noGDbZqDxcsZOeVFdP+NFtFg +dilITLxlgaQimN2nxhPvAAz6EcJ2owFgJJk6J3f6ICq8gdbJzFq2uVJrKBBL35St +/aOpflDRhQkMdJwrCuI/+4NBcXG5fQywjH8H+9dvbUONIlttbwE2zZJgXtY0gb+A +kY8fmy5pWBYnYAB8lFdAZxW0Sx6ehkmLA/ExtH/7jz/GwnwS12/of6wSvQHj8so= +-----END CERTIFICATE----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/client14-key.pem b/tests/integration/test_ssl_cert_authentication/certs/client14-key.pem new file mode 100644 index 0000000000000..10d1b89a27011 --- /dev/null +++ b/tests/integration/test_ssl_cert_authentication/certs/client14-key.pem @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKAIBAAKCAgEAs6uGg+vk4GlmN+ZlJb9gUI6AS6TJVQvLCSw7RPHt0k8xqMXe +oHid3mmVIC7tWK6BDEbAWJB814cWzOviXy5GbuUZB5Yq1MR0bQV968PBa/jp/X6e +DAW3KDfvgsk2eBo+aZqbFh1EJqXzN04d255X2mlAGpwW97T+XhXa/ahxj1EyG5sm +L6jaybcrKj/Xce4qVgSQk0kVlun/35Rfd6kcFTdq+Laap3nO0JKBYwfcoeMUvVfR +ZV0iaP7sbOpiZlv0+zFDDGX2Xi18VH2xeclICN4ar8K0h+VNE1uqf27r+3NAiZbZ +phA2huN60KYP+IV09KirWhFM3Xcd5LlqbluRUwWxp4xXZNWFIa6AeSbuC+43hAa0 +vBfN9+CE03ifX6stqfUy/eiJsghuPEZOUXqJpBpz4VHpxXVqiLX11hrGw60SrEQJ +30SVahULl4v1/zhFaRgFnZE2yRdQG0h1AejA5OUykKTH5vAkfZRxPmL3hnBBfg+i +ti93EFvdC2bXNMB04X+VkvF8CcqRUr2Ka7ZpfleJuO9ps2e0LBtjSJOLxYRAN13R +UMp9m/45F5iGE4QXdUFzOrasrclg8KfP0Zl+6/FQcygV4bLQywqm/zgCX1zCSAb8 +8JLEElIUmyMFT/cvst1uscy1fvoLseuKcz+9woAF1LA8XrZzMPRA22xSABMCAwEA +AQKCAgAEaIj8Y6VR/EQNyxFgQ7nRQC3VrU1jUM7CgttRbb4wEtFdGr3DojH9awnF +qGEac+2mp3XAtorZnu7oSEFdpH0F64kZro2OeuOAaUoVps/wHkNffOPT17AOxJCT +3OwBNmOho7F6cW1ipV+6U6hX4yK0sTBpdrr5iO9Uz6R35NIkehGIq93b/YCgwmXE +u5xFp1pSkfoaIwjskwE8Mx/Eh9mwi5OMVq6kvVBdvbp++4pmTnQL0UPKAOb/PIIA +ih+v80GniCXk//tzhBow2ISqQE4MKabt+REE5JNnjjA4wDf6C3Hh7lmYwX0VAi/Z +PrnVlzCvcBQEObhxFqMdIY+C9awzGrf0iJ1UGZms5IrcKwvEL7VsrQqO2COEv42K +KtbVNpRMYG5n+VwtylChmh2g6oX4wMLbYIykDfXZVv/FIjP3vgPxs19t/CRVUEer +x6GztwhAmKvp4WzerVH1xJ7QKcaIuy4cjhNLVUQCcjPbFbtHsHzvuScXHfppCM67 +6rpNQ5DrQ0Wj0a1kCal/57aKQQdD37k4aTTEoCbDJkbD10SoaljJYElHleB26xCV +AwbyXz6gwJ80CxbXS1uco3c86keDekrGV5f73XX53MZouRR90Ii/TJvvdGutXFkX +mpAIaOZlfeDSFMJoFaSpOj3UaY4RBwcXXDB69QpFMDe2ND0gwQKCAQEA6jw1G0dP +HwDYfQSMzTQkhle7hHqxPoyuooBe4Vl6MJ64S22tgvRLE+FuLfubR9tMBYYr/Ub4 +CmSBE/XMbNGpH31yct5I/8rc9OnRg0bcCw2dz/Q7b84LKas6fkBFaf1mMlPvH2zi +nwtS3Bw3Q6DAky1ra6rqjiZDtDCN+vQOh0zXVdOyVjApDBbngX/x8hVhFLs3SuwS +jm3P8YloEQ6aQ0wCjvsemJs+IpDlH7cHwYlgxBREqQ1wOKdo8IZ/MTTvlPOXW04F +pP995LCvtVacjhfOLaWdHTs442yuFvFBoPWFEvx2CGYTQe+dk8zmzLFdYVNpEvbV +dh/m1hMrA3lUdwKCAQEAxF1d+QuDanP39u+16u4+u61p3gzYv2f9m+bjViiYMv6N +kSDzdXrKidD1I+iEgIy1QjRfdHlQMCIpP8D4zuWCrNY57QKdV4bRKs0RSgpOXPv+ +FGeVf9taPIH/lUH8FUJ1gqfoLX+6W2RbZxcNPOSOf13o6jTjBbB6oVetljNjsnDS +us7CJe3+MDk20pO2+E49Q0TuyeALDPRSciSvJrVCVmoPs6rpu22z5zMCxd6+WztL +YBWfFM38QpIMUg1miz7hU7m7lQHlYE9/p9RSI5ePD9mgepPQN+bQg0fGQaD0tzad +MJr/BI0OvHLdwFQNLya66nWK7s85aD/UjDRO/A6kRQKCAQB1zGes21ToM6WsYeBp +xsJjqbWNb6K54UhmQwb0b+pqjzgB/xuW00L6sZGWoIW8QoZd9Ncknk9Z8qeToTb4 +twxF4PHw4Od3dM9ggEK0sasyB9wI3DwUA1xLzWgyXCJMpnqB7wJAHKNv9uLp/Wqx +oSOYIOx4DlG9wXKlKRIOVjUESFm3OSrj+355LP+qeez0oVnccjbhgA3pAULlpwPm +KCDenVhgDdyaROCfw5znMUY+R9eZZNQO7Mo2Q8Mby5gl6AhhMYw6B+gAzdjDbTRA +j1lWgJRZEoQMUl9OyLZYpWYrC66sGLlHigY/T8FAtniQEtbyfl9GgUpjCLIvkR49 +tgQLAoIBAF/WefD9D4y7QQDCifU5hmCvCIaZmogAxyR6EeaRNYdd+dYlUO27mnKd +C6gU6eabxjOjwBrmwp5bbepx0n2YQqj8fZURu51mbVwIbjHGyexUCPQIgky+0FHL +2OQOKmxt3VCBhq3+MwQ7/OhZtdpMasf7G5yDZ3H1akSouE4gkr4alp8aHmPIvlDm ++7zW32xdM0VLtYfN01blP//5q4qm2NO4PCWieyVBK5bhrK7KQfng/K7Onq/WwRH0 +mhLJ+4xmii8E7WqSXFMfOdy9ocFBTU+dFdf9oJhIDOil9Ts+xXFONHXukBy2g8Sy +A0zFORIUQxH/gGmBtjENRj2PoiUfOEkCggEBAMiBLM8L+k7ovGd7KVoeSgk7FjsV +5gq0GDwXGov9HEyeY/mmmuS5mfxZUYvRgzbpQutqy88MvbMsCS7VDtT9m0IFs8zR +qXIUQmOrYJuI37BrA16K5EDTj6wY/7DKRSM4guNqQjYnJ2bYBrXCZLWFUF/OVl3w +VF5reN7BCMOTXGc4aAeJzcmvhGU7M3WnGwL7yGkiX8BCDXo+ddLuf1SyDUttfrmo +Kbb4h4LAe2ATtRO0F9SfEetdmHUwO6iRJGQNQQWO7pWOjUStSry3Z8AM0uVDBRDG +Ol3DmKQlxbFAhLD62KkcWt30eNnNMlYaq8huKY4Y2juyi5ZQ31sDij4worA= +-----END RSA PRIVATE KEY----- diff --git a/tests/integration/test_ssl_cert_authentication/certs/generate_certs.sh b/tests/integration/test_ssl_cert_authentication/certs/generate_certs.sh index 00ea93ae0f861..27d8d646b149e 100755 --- a/tests/integration/test_ssl_cert_authentication/certs/generate_certs.sh +++ b/tests/integration/test_ssl_cert_authentication/certs/generate_certs.sh @@ -55,3 +55,76 @@ openssl x509 -req -days 3650 -in client12-req.pem -CA ca-cert.pem -CAkey ca-key. # 6. Generate one more self-signed certificate and private key for using as wrong certificate (because it's not signed by CA) openssl req -newkey rsa:4096 -x509 -days 3650 -nodes -batch -keyout wrong-key.pem -out wrong-cert.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=client" + +# 7. Generate a CA-signed certificate whose CN carries an embedded NUL byte ("client1\0.evil.com"), +# to test that server-side CN extraction does not truncate at the NUL. User 'john' is configured with +# client1, so a server that truncated the CN at the NUL would extract +# "client1" and wrongly authenticate this certificate as 'john'; the full CN must be preserved so the +# match fails. openssl's CLI cannot place a NUL inside a -subj field, so we build this certificate +# with the 'cryptography' library instead. +python3 - <<'PY' +from cryptography import x509 +from cryptography.x509.oid import NameOID +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa +import datetime + +with open("ca-key.pem", "rb") as f: + ca_key = serialization.load_pem_private_key(f.read(), password=None) +with open("ca-cert.pem", "rb") as f: + ca_cert = x509.load_pem_x509_certificate(f.read()) + +key = rsa.generate_private_key(public_exponent=65537, key_size=4096) +subject = x509.Name([ + x509.NameAttribute(NameOID.COUNTRY_NAME, "RU"), + x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, "Some-State"), + x509.NameAttribute(NameOID.ORGANIZATION_NAME, "Internet Widgits Pty Ltd"), + x509.NameAttribute(NameOID.COMMON_NAME, "client1\x00.evil.com"), +]) +now = datetime.datetime(2020, 1, 1) +cert = (x509.CertificateBuilder() + .subject_name(subject).issuer_name(ca_cert.subject) + .public_key(key.public_key()).serial_number(x509.random_serial_number()) + .not_valid_before(now).not_valid_after(now + datetime.timedelta(days=3650)) + .sign(ca_key, hashes.SHA256())) +with open("client13-key.pem", "wb") as f: + f.write(key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.TraditionalOpenSSL, serialization.NoEncryption())) +with open("client13-cert.pem", "wb") as f: + f.write(cert.public_bytes(serialization.Encoding.PEM)) +PY + +# 8. Generate a CA-signed certificate whose CN and DNS SAN both carry an embedded NUL byte +# ("evil\0.corp.example.com"), to test that wildcard matching rejects it. Users 'wildcard_cn' and +# 'wildcard_dns' are configured with '*.corp.example.com' and 'DNS:*.corp.example.com': without an +# explicit check, '*' would match the single "label" "evil\0" and authenticate this certificate. +python3 - <<'PY' +from cryptography import x509 +from cryptography.x509.oid import NameOID +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa +import datetime + +with open("ca-key.pem", "rb") as f: + ca_key = serialization.load_pem_private_key(f.read(), password=None) +with open("ca-cert.pem", "rb") as f: + ca_cert = x509.load_pem_x509_certificate(f.read()) + +key = rsa.generate_private_key(public_exponent=65537, key_size=4096) +subject = x509.Name([ + x509.NameAttribute(NameOID.COUNTRY_NAME, "RU"), + x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, "Some-State"), + x509.NameAttribute(NameOID.ORGANIZATION_NAME, "Internet Widgits Pty Ltd"), + x509.NameAttribute(NameOID.COMMON_NAME, "evil\x00.corp.example.com"), +]) +now = datetime.datetime(2020, 1, 1) +cert = (x509.CertificateBuilder() + .subject_name(subject).issuer_name(ca_cert.subject) + .public_key(key.public_key()).serial_number(x509.random_serial_number()) + .not_valid_before(now).not_valid_after(now + datetime.timedelta(days=3650)) + .add_extension(x509.SubjectAlternativeName([x509.DNSName("evil\x00.corp.example.com")]), critical=False) + .sign(ca_key, hashes.SHA256())) +with open("client14-key.pem", "wb") as f: + f.write(key.private_bytes(serialization.Encoding.PEM, serialization.PrivateFormat.TraditionalOpenSSL, serialization.NoEncryption())) +with open("client14-cert.pem", "wb") as f: + f.write(cert.public_bytes(serialization.Encoding.PEM)) +PY diff --git a/tests/integration/test_ssl_cert_authentication/test.py b/tests/integration/test_ssl_cert_authentication/test.py index 216dd02f9629b..4639879deeff9 100644 --- a/tests/integration/test_ssl_cert_authentication/test.py +++ b/tests/integration/test_ssl_cert_authentication/test.py @@ -147,6 +147,18 @@ def test_native_fallback_to_password(): assert "AUTHENTICATION_FAILED" in str(err.value) +def test_native_cn_nul_byte_no_bypass(): + # Authentication bypass: client13's CN is "client1\0.evil.com" and user 'john' is configured with + # client1. If server-side CN extraction truncated at the embedded NUL + # byte, the CN would collapse to "client1" and the certificate would authenticate as user 'john'. + # The full CN must be preserved, so the match must fail. + with pytest.raises(Exception) as err: + execute_query_native( + instance, "SELECT currentUser()", user="john", cert_name="client13" + ) + assert "AUTHENTICATION_FAILED" in str(err.value) + + def get_ssl_context(cert_name): context = WrapSSLContextWithSNI(SSL_HOST, ssl.PROTOCOL_TLS_CLIENT) context.load_verify_locations(cafile=f"{SCRIPT_DIR}/certs/ca-cert.pem") @@ -221,6 +233,14 @@ def test_https_wrong_cert(): ) +def test_https_cn_nul_byte_no_bypass(): + # Same bypass as test_native_cn_nul_byte_no_bypass, over the HTTPS interface: client13's CN + # "client1\0.evil.com" must not be truncated to "client1" and authenticate as user 'john'. + with pytest.raises(Exception) as err: + execute_query_https("SELECT currentUser()", user="john", cert_name="client13") + assert "403" in str(err.value) + + def test_https_non_ssl_auth(): # Users with non-SSL authentication are allowed, in this case we can skip sending a client certificate at all (because "verificationMode" is set to "relaxed"). # assert execute_query_https("SELECT currentUser()", user="peter", enable_ssl_auth=False) == "peter\n" @@ -522,6 +542,21 @@ def test_x509_cn_wildcard_single_label(): assert "403" in str(err.value) +def test_x509_wildcard_nul_byte_no_bypass(): + # Authentication bypass: client14's CN and DNS SAN are both "evil\0.corp.example.com". Users + # 'wildcard_cn' and 'wildcard_dns' are configured with '*.corp.example.com' and + # 'DNS:*.corp.example.com'. The '*' must not match the span "evil\0", so both must fail. + for user in ["wildcard_cn", "wildcard_dns"]: + with pytest.raises(Exception) as err: + execute_query_native( + instance, "SELECT currentUser()", user=user, cert_name="client14" + ) + assert "AUTHENTICATION_FAILED" in str(err.value) + with pytest.raises(Exception) as err: + execute_query_https("SELECT currentUser()", user=user, cert_name="client14") + assert "403" in str(err.value) + + def test_x509_uri_san_wildcard_dot_in_segment(): # Non-regression: '.' separates labels for DNS/CN but is NOT a separator for URI SANs, # whose separator is '/'. A wildcard URI path segment may legitimately contain dots, so diff --git a/tests/integration/test_storage_delta/configs/users.d/users.xml b/tests/integration/test_storage_delta/configs/users.d/users.xml index 1cf4851873c51..8ca5fa7963f01 100644 --- a/tests/integration/test_storage_delta/configs/users.d/users.xml +++ b/tests/integration/test_storage_delta/configs/users.d/users.xml @@ -4,6 +4,7 @@ default 1 + 1 1 diff --git a/tests/integration/test_storage_delta/test.py b/tests/integration/test_storage_delta/test.py index 6bf26a3d15e47..3521ddabe30b4 100644 --- a/tests/integration/test_storage_delta/test.py +++ b/tests/integration/test_storage_delta/test.py @@ -1,4 +1,5 @@ import glob +import io import json import logging import os @@ -5185,3 +5186,105 @@ def test_delta_kernel_retry_on_stale_token_via_catalog_callback(started_cluster) f"Expected the catalog-callback retry log line to fire for query {retry_query_id}, " f"found {retry_hits} hits — the stale-token retry path was not exercised." ) + + +@pytest.mark.parametrize( + "use_delta_kernel", + ["0", "1"], +) +def test_azure_url_encoded_partition_path(started_cluster, use_delta_kernel): + instance = get_node(started_cluster, use_delta_kernel) + TABLE_NAME = randomize_table_name("test_azure_url_encoded") + + container_client = started_cluster.blob_service_client.get_container_client( + started_cluster.azure_container_name + ) + + # Upload the parquet file at the on-disk (decoded-once) blob path. + # Partition value "@INTERNAL@" → directory "org=%40INTERNAL%40" (@ → %40). + partition_dir = "org=%40INTERNAL%40" + parquet_blob = f"{TABLE_NAME}/{partition_dir}/part-0.parquet" + + schema = pa.schema([ + pa.field("id", pa.int32()), + pa.field("org", pa.string()), + ]) + table_data = pa.table( + { + "id": pa.array([1, 2, 3], type=pa.int32()), + "org": pa.array(["@INTERNAL@"] * 3, type=pa.string()), + }, + schema=schema, + ) + buf = io.BytesIO() + pq.write_table(table_data, buf, compression=None) + parquet_bytes = buf.getvalue() + container_client.upload_blob(parquet_blob, parquet_bytes, overwrite=True) + + # Build the Delta log manually. add.path is URI-encoded per the Delta protocol: + # '%' in the on-disk name (%40) is encoded to '%25', producing %2540. + add_path = "org=%2540INTERNAL%2540/part-0.parquet" + + protocol = '{"protocol":{"minReaderVersion":1,"minWriterVersion":2}}' + metadata = json.dumps( + { + "metaData": { + "id": str(uuid.uuid4()), + "format": {"provider": "parquet", "options": {}}, + "schemaString": json.dumps( + { + "type": "struct", + "fields": [ + { + "name": "id", + "type": "integer", + "nullable": True, + "metadata": {}, + }, + { + "name": "org", + "type": "string", + "nullable": True, + "metadata": {}, + }, + ], + } + ), + "partitionColumns": ["org"], + "configuration": {}, + "createdTime": 1600000000000, + } + } + ) + add = json.dumps( + { + "add": { + "path": add_path, + "partitionValues": {"org": "@INTERNAL@"}, + "size": len(parquet_bytes), + "modificationTime": 1600000000000, + "dataChange": True, + "stats": json.dumps({"numRecords": 3}), + } + } + ) + + log_content = "\n".join([protocol, metadata, add]) + container_client.upload_blob( + f"{TABLE_NAME}/_delta_log/00000000000000000000.json", + log_content.encode(), + overwrite=True, + ) + + connection_string = started_cluster.env_variables["AZURITE_CONNECTION_STRING"] + instance.query( + f""" + DROP TABLE IF EXISTS {TABLE_NAME}; + CREATE TABLE {TABLE_NAME} + ENGINE = DeltaLakeAzure('{connection_string}', '{started_cluster.azure_container_name}', '{TABLE_NAME}', Parquet) + """ + ) + + assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 3 + + instance.query(f"DROP TABLE IF EXISTS {TABLE_NAME}") diff --git a/tests/integration/test_storage_dict/configs/users.xml b/tests/integration/test_storage_dict/configs/users.xml index 4b6ba057ecb16..2f9966e1d71a2 100644 --- a/tests/integration/test_storage_dict/configs/users.xml +++ b/tests/integration/test_storage_dict/configs/users.xml @@ -4,6 +4,7 @@ default 1 + 1 diff --git a/tests/integration/test_storage_iceberg_concurrent/configs/users.d/users.xml b/tests/integration/test_storage_iceberg_concurrent/configs/users.d/users.xml index 4b6ba057ecb16..2f9966e1d71a2 100644 --- a/tests/integration/test_storage_iceberg_concurrent/configs/users.d/users.xml +++ b/tests/integration/test_storage_iceberg_concurrent/configs/users.d/users.xml @@ -4,6 +4,7 @@ default 1 + 1 diff --git a/tests/integration/test_storage_iceberg_disks/test.py b/tests/integration/test_storage_iceberg_disks/test.py index a9547e4742467..b4de67cc4466b 100644 --- a/tests/integration/test_storage_iceberg_disks/test.py +++ b/tests/integration/test_storage_iceberg_disks/test.py @@ -89,6 +89,9 @@ def generate_cluster_def(common_path, port, azure_container): disk_local_common,disk_s3_common,disk_azure_common + + azurite1 + """ ) diff --git a/tests/integration/test_storage_iceberg_no_spark/configs/users.d/users.xml b/tests/integration/test_storage_iceberg_no_spark/configs/users.d/users.xml index 4b6ba057ecb16..2f9966e1d71a2 100644 --- a/tests/integration/test_storage_iceberg_no_spark/configs/users.d/users.xml +++ b/tests/integration/test_storage_iceberg_no_spark/configs/users.d/users.xml @@ -4,6 +4,7 @@ default 1 + 1 diff --git a/tests/integration/test_storage_iceberg_no_spark/test_local_path_traversal.py b/tests/integration/test_storage_iceberg_no_spark/test_local_path_traversal.py new file mode 100644 index 0000000000000..f78a43db208ea --- /dev/null +++ b/tests/integration/test_storage_iceberg_no_spark/test_local_path_traversal.py @@ -0,0 +1,297 @@ +""" +IcebergLocal path traversal: a manifest entry that uses ../ to escape user_files +must be blocked with PATH_ACCESS_DENIED, not silently followed. +""" +import json +import os +import re +import shutil +import tempfile + +import avro.datafile +import avro.io + +from helpers.iceberg_utils import get_uuid_str + +NOCACHE = { + "use_iceberg_metadata_files_cache": False, + "use_parquet_metadata_cache": False, +} + + +def _prepare_table_then_corrupt_manifest( + instance, table_name, table_path, malicious_path_func, file_format_override=None +): + """Create a valid IcebergLocal table, then rewrite the manifest so that + `data_file.file_path` points outside `user_files`.""" + instance.query( + f"CREATE TABLE {table_name} (c0 Int) ENGINE = IcebergLocal('{table_path}/', 'Parquet');", + settings={"allow_insert_into_iceberg": 1, **NOCACHE}, + ) + instance.query( + f"INSERT INTO {table_name} VALUES (42)", + settings={"allow_insert_into_iceberg": 1, **NOCACHE}, + ) + result = instance.query(f"SELECT c0 FROM {table_name}", settings=NOCACHE) + assert result.strip() == "42" + instance.query(f"DROP TABLE {table_name}") + + temp_dir = tempfile.mkdtemp() + try: + host_path = os.path.join(temp_dir, table_name) + os.makedirs(host_path, exist_ok=True) + _download_table_from_container(instance, table_path, host_path) + + _, manifest_paths = _get_manifest_paths_from_chain(host_path, table_path) + assert manifest_paths, "No manifest_path in manifest-list" + + for rel_path in manifest_paths: + local_manifest = os.path.join(host_path, rel_path) + assert os.path.isfile(local_manifest), f"Manifest not found: {local_manifest}" + + _modify_avro_file(local_manifest, ["data_file", "file_path"], malicious_path_func) + if file_format_override is not None: + _modify_avro_file( + local_manifest, + ["data_file", "file_format"], + lambda _: file_format_override, + ) + + instance.copy_file_to_container(local_manifest, f"{table_path}/{rel_path}") + finally: + shutil.rmtree(temp_dir, ignore_errors=True) + + +def _download_table_from_container(instance, table_path, host_path): + for remote_file in instance.get_files_list_in_container(table_path): + rel = os.path.relpath(remote_file, table_path) + local_path = os.path.join(host_path, rel) + os.makedirs(os.path.dirname(local_path), exist_ok=True) + if instance.file_exists_in_container(remote_file): + instance.copy_file_from_container(remote_file, local_path) + + +def _to_relative_path(path, table_path): + """Strip `table_path` prefix from an absolute path, returning a relative path.""" + if not path or not path.startswith("/"): + return path + base = table_path.rstrip("/") + if path == base or path.startswith(base + "/"): + return path[len(base):].lstrip("/") + return path + + +def _get_manifest_paths_from_chain(host_path, table_path, content_filter=None): + """Follow metadata.json -> manifest-list -> manifest_path(s). + Returns (manifest_list_rel, [manifest_rel, ...]). + + In Iceberg v2 each manifest-list entry carries a ``content`` field + (0 = data, 1 = deletes). Pass *content_filter* to keep only entries + with that value; ``None`` (default) returns all manifests.""" + metadata_dir = os.path.join(host_path, "metadata") + version_hint_path = os.path.join(metadata_dir, "version-hint.text") + if os.path.isfile(version_hint_path): + with open(version_hint_path) as f: + version = f.read().strip() + metadata_file = os.path.join(metadata_dir, f"v{version}.metadata.json") + else: + meta_files = _find_files(metadata_dir, ".metadata.json") + if not meta_files: + raise RuntimeError("No metadata.json found") + + def version_key(p): + m = re.match(r"v(\d+)", os.path.basename(p), re.IGNORECASE) + return int(m.group(1)) if m else 0 + + metadata_file = max(meta_files, key=version_key) + + with open(metadata_file) as f: + data = json.load(f) + snap_id = data.get("current-snapshot-id") + if not snap_id: + raise RuntimeError("No current-snapshot-id in metadata") + snapshot = next((s for s in data.get("snapshots", []) if s.get("snapshot-id") == snap_id), None) + if not snapshot: + raise RuntimeError(f"Snapshot {snap_id} not found") + ml = snapshot.get("manifest-list") + if not ml: + raise RuntimeError("Snapshot has no manifest-list") + + ml_rel = _to_relative_path(ml, table_path) + ml_local = os.path.join(host_path, ml_rel) + if not os.path.isfile(ml_local): + raise RuntimeError(f"Manifest list file not found: {ml_local}") + + manifest_paths = [] + with open(ml_local, "rb") as f: + reader = avro.datafile.DataFileReader(f, avro.io.DatumReader()) + for record in reader: + if "manifest_path" not in record: + continue + if content_filter is not None and record.get("content") != content_filter: + continue + manifest_paths.append(_to_relative_path(record["manifest_path"], table_path)) + reader.close() + + return ml_rel, manifest_paths + + +def _modify_avro_file(avro_path, field_path, modifier_func): + """Read an Avro file, apply `modifier_func` to the field at `field_path` + in every record, and write the file back.""" + with open(avro_path, "rb") as f: + reader = avro.datafile.DataFileReader(f, avro.io.DatumReader()) + schema = reader.datum_reader.writers_schema + metadata = dict(reader.meta) + records = list(reader) + reader.close() + + for record in records: + obj = record + for key in field_path[:-1]: + if obj is None or key not in obj: + break + obj = obj[key] + else: + if obj and field_path[-1] in obj: + obj[field_path[-1]] = modifier_func(obj[field_path[-1]]) + + with open(avro_path, "wb") as f: + writer = avro.datafile.DataFileWriter(f, avro.io.DatumWriter(), schema) + for key, value in metadata.items(): + if not key.startswith("avro."): + writer.set_meta(key, value) + for record in records: + writer.append(record) + writer.close() + + +def _find_files(directory, suffix): + result = [] + for root, _, files in os.walk(directory): + for f in files: + if f.endswith(suffix): + result.append(os.path.join(root, f)) + return result + + +def _assert_delete_path_access_denied(error): + assert "PATH_ACCESS_DENIED" in error, f"Expected PATH_ACCESS_DENIED, got: {error}" + + + +def test_local_iceberg_path_traversal(started_cluster_iceberg_no_spark): + """Manifest entry with ../../../../../../../etc/passwd as data_file.file_path + (format RAWBLOB) must be rejected with PATH_ACCESS_DENIED. + + If the error message instead shows 'No such file or directory' + or any other error, the security check is broken: the engine attempted + to open and read the file instead of blocking the path early.""" + instance = started_cluster_iceberg_no_spark.instances["node1"] + table_name = "test_local_iceberg_path_traversal" + get_uuid_str() + table_path = f"/var/lib/clickhouse/user_files/iceberg_data/default/{table_name}" + + _prepare_table_then_corrupt_manifest( + instance, + table_name, + table_path, + lambda _: f"{table_path}/../../../../../../../etc/passwd", + file_format_override="RAWBLOB", + ) + + path_for_query = f"/var/lib/clickhouse/user_files/iceberg_data/default/{table_name}" + error = instance.query_and_get_error( + f"SELECT * FROM icebergLocal(local, path = '{path_for_query}', format='RawBLOB')", + settings=NOCACHE, + ) + _assert_delete_path_access_denied(error) + + +def _corrupt_delete_manifest_and_query(instance, test_name, modify_manifest_fn): + """Create an IcebergLocal table with position deletes, apply + *modify_manifest_fn(local_manifest_path, table_path)* to every + delete manifest, and return the error string from a SELECT.""" + table_name = test_name + get_uuid_str() + table_path = f"/var/lib/clickhouse/user_files/iceberg_data/default/{table_name}" + + instance.query( + f"CREATE TABLE {table_name} (c0 Int) ENGINE = IcebergLocal('{table_path}/', 'Parquet');", + settings={"allow_insert_into_iceberg": 1, **NOCACHE}, + ) + instance.query( + f"INSERT INTO {table_name} VALUES (1), (2), (3)", + settings={"allow_insert_into_iceberg": 1, **NOCACHE}, + ) + instance.query( + f"DELETE FROM {table_name} WHERE c0 = 1", + settings={"allow_insert_into_iceberg": 1, **NOCACHE}, + ) + result = instance.query( + f"SELECT c0 FROM {table_name} ORDER BY c0", settings=NOCACHE + ) + assert result.strip() == "2\n3" + instance.query(f"DROP TABLE {table_name}") + + temp_dir = tempfile.mkdtemp() + try: + host_path = os.path.join(temp_dir, table_name) + os.makedirs(host_path, exist_ok=True) + _download_table_from_container(instance, table_path, host_path) + + _, delete_manifest_paths = _get_manifest_paths_from_chain( + host_path, table_path, content_filter=1 + ) + assert delete_manifest_paths, "No delete manifest found after DELETE FROM" + + for rel_path in delete_manifest_paths: + local_manifest = os.path.join(host_path, rel_path) + assert os.path.isfile(local_manifest), f"Not found: {local_manifest}" + modify_manifest_fn(local_manifest, table_path) + instance.copy_file_to_container( + local_manifest, f"{table_path}/{rel_path}" + ) + finally: + shutil.rmtree(temp_dir, ignore_errors=True) + + return instance.query_and_get_error( + f"SELECT * FROM icebergLocal(local, path = '{table_path}/')", + settings=NOCACHE, + ) + +def test_local_iceberg_delete_file_path_traversal(started_cluster_iceberg_no_spark): + """Position-delete file_path with ../ must be rejected via + validateDeleteFilePaths (not LocalPathValidatingIterator).""" + instance = started_cluster_iceberg_no_spark.instances["node1"] + + def corrupt(manifest, table_path): + _modify_avro_file( + manifest, + ["data_file", "file_path"], + lambda _: f"{table_path}/../../../../../../../etc/passwd", + ) + + _assert_delete_path_access_denied( + _corrupt_delete_manifest_and_query(instance, "test_local_iceberg_path_traversal", corrupt) + ) + + +def test_local_iceberg_equality_delete_file_path_traversal( + started_cluster_iceberg_no_spark, +): + """Equality-delete file_path with ../ must be rejected via + validateDeleteFilePaths. Patches the delete manifest to look like + an equality delete (content=2, equality_ids=[1]).""" + instance = started_cluster_iceberg_no_spark.instances["node1"] + + def corrupt(manifest, table_path): + _modify_avro_file(manifest, ["data_file", "content"], lambda _: 2) + _modify_avro_file(manifest, ["data_file", "equality_ids"], lambda _: [1]) + _modify_avro_file( + manifest, + ["data_file", "file_path"], + lambda _: f"{table_path}/../../../../../../../etc/passwd", + ) + + _assert_delete_path_access_denied( + _corrupt_delete_manifest_and_query(instance, "test_local_iceberg_path_traversal", corrupt) + ) diff --git a/tests/integration/test_storage_iceberg_no_spark/test_writes_manifest_list_partition_summary.py b/tests/integration/test_storage_iceberg_no_spark/test_writes_manifest_list_partition_summary.py new file mode 100644 index 0000000000000..87e53b388e3bc --- /dev/null +++ b/tests/integration/test_storage_iceberg_no_spark/test_writes_manifest_list_partition_summary.py @@ -0,0 +1,205 @@ +"""Every manifest that an INSERT or a DELETE writes holds exactly one partition tuple, so its +manifest-list entry must carry a field summary pinned to that tuple: `lower_bound == upper_bound` +per partition field. Without the summary a reader cannot skip a manifest by partition and has to +open all of them.""" + +import glob +import os +import struct + +import avro.datafile +import avro.io +import pytest + +from helpers.iceberg_utils import ( + create_iceberg_table, + default_download_directory, + get_last_snapshot, + get_uuid_str, + unescape_path, +) + +TABLE_ROOT = "/var/lib/clickhouse/user_files/iceberg_data/default" + + +def _download_table(started_cluster, storage_type, table_name): + path = f"{TABLE_ROOT}/{table_name}/" + default_download_directory(started_cluster, storage_type, path, path) + return path + + +def _read_avro(path): + with open(path, "rb") as f: + return list(avro.datafile.DataFileReader(f, avro.io.DatumReader())) + + +def _manifest_list_of_last_snapshot(table_path): + snapshot_id = get_last_snapshot(table_path) + matches = glob.glob(f"{table_path}/metadata/snap-{snapshot_id}-*.avro") + assert len(matches) == 1, matches + return matches[0] + + +def _encode_bound(value): + """A single-value bound as Iceberg serializes it: UTF-8 for a string, little-endian two's + complement for an int.""" + if isinstance(value, str): + return value.encode() + return struct.pack(" default 1 + 1 diff --git a/tests/integration/test_storage_iceberg_schema_evolution/test_nested_containers_evolved.py b/tests/integration/test_storage_iceberg_schema_evolution/test_nested_containers_evolved.py new file mode 100644 index 0000000000000..55205e2c74ddd --- /dev/null +++ b/tests/integration/test_storage_iceberg_schema_evolution/test_nested_containers_evolved.py @@ -0,0 +1,252 @@ +import pytest + +from helpers.iceberg_utils import ( + default_upload_directory, + get_uuid_str, + get_creation_expression, + check_schema_and_data +) + + +def make_query_executor(started_cluster, storage_type, table_name): + spark = started_cluster.spark_session + + def execute_spark_query(query: str): + spark.sql(query) + default_upload_directory( + started_cluster, + storage_type, + f"/iceberg_data/default/{table_name}/", + f"/iceberg_data/default/{table_name}/", + ) + + return execute_spark_query + + +@pytest.mark.parametrize("storage_type", ["local"]) +def test_map_of_arrays_evolved(started_cluster_iceberg_schema_evolution, storage_type): + instance = started_cluster_iceberg_schema_evolution.instances["node1"] + TABLE_NAME = "test_map_of_arrays_evolved_" + storage_type + "_" + get_uuid_str() + execute_spark_query = make_query_executor( + started_cluster_iceberg_schema_evolution, storage_type, TABLE_NAME + ) + + execute_spark_query(f"DROP TABLE IF EXISTS {TABLE_NAME}") + execute_spark_query( + f""" + CREATE TABLE {TABLE_NAME} ( + entries MAP>> + ) + USING iceberg + OPTIONS ('format-version'='2') + """ + ) + execute_spark_query( + f"INSERT INTO {TABLE_NAME} VALUES (MAP(1, ARRAY(named_struct('a', 10, 'b', 'hello'))))" + ) + + table_function = get_creation_expression( + storage_type, TABLE_NAME, started_cluster_iceberg_schema_evolution, table_function=True + ) + + check_schema_and_data( + instance, + table_function, + [ + ['entries', 'Map(Int32, Array(Tuple(\\n a Nullable(Int32),\\n b Nullable(String))))'], + ], + [ + ["{1:[(10,'hello')]}"], + ], + ) + + execute_spark_query(f"ALTER TABLE {TABLE_NAME} ADD COLUMN entries.value.element.c INT") + execute_spark_query(f"ALTER TABLE {TABLE_NAME} ALTER COLUMN entries.value.element.c FIRST") + + check_schema_and_data( + instance, + table_function, + [ + ['entries', 'Map(Int32, Array(Tuple(\\n c Nullable(Int32),\\n a Nullable(Int32),\\n b Nullable(String))))'], + ], + [ + ["{1:[(NULL,10,'hello')]}"], + ], + ) + + +@pytest.mark.parametrize("storage_type", ["local"]) +def test_struct_with_doubly_nested_array_evolved( + started_cluster_iceberg_schema_evolution, storage_type +): + instance = started_cluster_iceberg_schema_evolution.instances["node1"] + TABLE_NAME = ( + "test_struct_with_doubly_nested_array_evolved_" + storage_type + "_" + get_uuid_str() + ) + execute_spark_query = make_query_executor( + started_cluster_iceberg_schema_evolution, storage_type, TABLE_NAME + ) + + execute_spark_query(f"DROP TABLE IF EXISTS {TABLE_NAME}") + execute_spark_query( + f""" + CREATE TABLE {TABLE_NAME} ( + top STRUCT< + lvl1: ARRAY> + >> + > + ) + USING iceberg + OPTIONS ('format-version'='2') + """ + ) + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES + (named_struct('lvl1', ARRAY(named_struct('lvl2', ARRAY(named_struct('a', 1, 'b', 'x')))))); + """ + ) + + table_function = get_creation_expression( + storage_type, TABLE_NAME, started_cluster_iceberg_schema_evolution, table_function=True + ) + + check_schema_and_data( + instance, + table_function, + [ + ['top', 'Tuple(\\n lvl1 Array(Tuple(\\n lvl2 Array(Tuple(\\n a Nullable(Int32),\\n b Nullable(String))))))'], + ], + [ + ["([([(1,'x')])])"], + ], + ) + + execute_spark_query(f"ALTER TABLE {TABLE_NAME} ADD COLUMN top.lvl1.element.lvl2.element.c INT") + execute_spark_query(f"ALTER TABLE {TABLE_NAME} ALTER COLUMN top.lvl1.element.lvl2.element.c FIRST") + execute_spark_query(f"ALTER TABLE {TABLE_NAME} DROP COLUMN top.lvl1.element.lvl2.element.b") + + check_schema_and_data( + instance, + table_function, + [ + ['top', 'Tuple(\\n lvl1 Array(Tuple(\\n lvl2 Array(Tuple(\\n c Nullable(Int32),\\n a Nullable(Int32))))))'], + ], + [ + ["([([(NULL,1)])])"], + ], + ) + + +@pytest.mark.parametrize("storage_type", ["local"]) +def test_struct_with_sibling_arrays_evolved( + started_cluster_iceberg_schema_evolution, storage_type +): + instance = started_cluster_iceberg_schema_evolution.instances["node1"] + TABLE_NAME = ( + "test_struct_with_sibling_arrays_evolved_" + storage_type + "_" + get_uuid_str() + ) + execute_spark_query = make_query_executor( + started_cluster_iceberg_schema_evolution, storage_type, TABLE_NAME + ) + + execute_spark_query(f"DROP TABLE IF EXISTS {TABLE_NAME}") + execute_spark_query( + f""" + CREATE TABLE {TABLE_NAME} ( + top STRUCT< + x: ARRAY>, + y: ARRAY> + > + ) + USING iceberg + OPTIONS ('format-version'='2') + """ + ) + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES + (named_struct('x', ARRAY(named_struct('a', 1)), 'y', ARRAY(named_struct('b', 2)))); + """ + ) + + table_function = get_creation_expression( + storage_type, TABLE_NAME, started_cluster_iceberg_schema_evolution, table_function=True + ) + + execute_spark_query(f"ALTER TABLE {TABLE_NAME} ADD COLUMN top.x.element.c INT") + execute_spark_query(f"ALTER TABLE {TABLE_NAME} ADD COLUMN top.y.element.d INT") + execute_spark_query(f"ALTER TABLE {TABLE_NAME} ALTER COLUMN top.x.element.c FIRST") + + check_schema_and_data( + instance, + table_function, + [ + ['top', 'Tuple(\\n x Array(Tuple(\\n c Nullable(Int32),\\n a Nullable(Int32))),\\n y Array(Tuple(\\n b Nullable(Int32),\\n d Nullable(Int32))))'], + ], + [ + ["([(NULL,1)],[(2,NULL)])"], + ], + ) + + +@pytest.mark.parametrize("storage_type", ["local"]) +def test_struct_with_primitive_array_sibling_evolved( + started_cluster_iceberg_schema_evolution, storage_type +): + instance = started_cluster_iceberg_schema_evolution.instances["node1"] + TABLE_NAME = ( + "test_struct_with_primitive_array_sibling_evolved_" + + storage_type + + "_" + + get_uuid_str() + ) + execute_spark_query = make_query_executor( + started_cluster_iceberg_schema_evolution, storage_type, TABLE_NAME + ) + + execute_spark_query(f"DROP TABLE IF EXISTS {TABLE_NAME}") + execute_spark_query( + f""" + CREATE TABLE {TABLE_NAME} ( + top STRUCT< + nums: ARRAY, + items: ARRAY> + > + ) + USING iceberg + OPTIONS ('format-version'='2') + """ + ) + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES + (named_struct('nums', ARRAY(1, 2), 'items', ARRAY(named_struct('a', 1, 'b', 'x')))), + (named_struct('nums', CAST(ARRAY() AS ARRAY), 'items', CAST(ARRAY() AS ARRAY>))), + (named_struct('nums', ARRAY(3), 'items', ARRAY(named_struct('a', 2, 'b', 'y'), named_struct('a', 3, 'b', 'z')))); + """ + ) + + table_function = get_creation_expression( + storage_type, TABLE_NAME, started_cluster_iceberg_schema_evolution, table_function=True + ) + + execute_spark_query(f"ALTER TABLE {TABLE_NAME} ADD COLUMN top.items.element.c INT") + execute_spark_query(f"ALTER TABLE {TABLE_NAME} ALTER COLUMN top.items.element.c FIRST") + + check_schema_and_data( + instance, + table_function, + [ + ['top', 'Tuple(\\n nums Array(Nullable(Int32)),\\n items Array(Tuple(\\n c Nullable(Int32),\\n a Nullable(Int32),\\n b Nullable(String))))'], + ], + [ + ["([],[])"], + ["([1,2],[(NULL,1,'x')])"], + ["([3],[(NULL,2,'y'),(NULL,3,'z')])"], + ], + ) diff --git a/tests/integration/test_storage_iceberg_schema_evolution/test_struct_with_nested_array_evolved.py b/tests/integration/test_storage_iceberg_schema_evolution/test_struct_with_nested_array_evolved.py new file mode 100644 index 0000000000000..5efc93ffbfb95 --- /dev/null +++ b/tests/integration/test_storage_iceberg_schema_evolution/test_struct_with_nested_array_evolved.py @@ -0,0 +1,138 @@ +import pytest + +from helpers.iceberg_utils import ( + default_upload_directory, + get_uuid_str, + get_creation_expression, + check_schema_and_data +) + + +@pytest.mark.parametrize("format_version", ["1", "2"]) +@pytest.mark.parametrize("storage_type", ["s3", "local"]) +def test_struct_with_nested_array_evolved( + started_cluster_iceberg_schema_evolution, format_version, storage_type +): + instance = started_cluster_iceberg_schema_evolution.instances["node1"] + spark = started_cluster_iceberg_schema_evolution.spark_session + TABLE_NAME = ( + "test_struct_with_nested_array_evolved_" + + format_version + + "_" + + storage_type + + "_" + + get_uuid_str() + ) + + def execute_spark_query(query: str): + spark.sql(query) + default_upload_directory( + started_cluster_iceberg_schema_evolution, + storage_type, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + return + + execute_spark_query(f"DROP TABLE IF EXISTS {TABLE_NAME}") + execute_spark_query( + f""" + CREATE TABLE {TABLE_NAME} ( + outer STRUCT< + items: ARRAY> + > + ) + USING iceberg + OPTIONS ('format-version'='{format_version}') + """ + ) + + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES + (named_struct('items', ARRAY(named_struct('a', 1, 'b', 'hello'), named_struct('a', 2, 'b', 'world')))); + """ + ) + + table_function = get_creation_expression( + storage_type, TABLE_NAME, started_cluster_iceberg_schema_evolution, table_function=True + ) + + check_schema_and_data( + instance, + table_function, + [ + ['outer', 'Tuple(\\n items Array(Tuple(\\n a Nullable(Int32),\\n b Nullable(String))))'], + ], + [ + ["([(1,'hello'),(2,'world')])"], + ], + ) + + execute_spark_query(f"ALTER TABLE {TABLE_NAME} ADD COLUMN outer.items.element.c INT") + + check_schema_and_data( + instance, + table_function, + [ + ['outer', 'Tuple(\\n items Array(Tuple(\\n a Nullable(Int32),\\n b Nullable(String),\\n c Nullable(Int32))))'], + ], + [ + ["([(1,'hello',NULL),(2,'world',NULL)])"], + ], + ) + + execute_spark_query(f"ALTER TABLE {TABLE_NAME} ALTER COLUMN outer.items.element.c FIRST") + + check_schema_and_data( + instance, + table_function, + [ + ['outer', 'Tuple(\\n items Array(Tuple(\\n c Nullable(Int32),\\n a Nullable(Int32),\\n b Nullable(String))))'], + ], + [ + ["([(NULL,1,'hello'),(NULL,2,'world')])"], + ], + ) + + execute_spark_query(f"ALTER TABLE {TABLE_NAME} RENAME COLUMN outer.items.element.a TO renamed_a") + + check_schema_and_data( + instance, + table_function, + [ + ['outer', 'Tuple(\\n items Array(Tuple(\\n c Nullable(Int32),\\n renamed_a Nullable(Int32),\\n b Nullable(String))))'], + ], + [ + ["([(NULL,1,'hello'),(NULL,2,'world')])"], + ], + ) + + execute_spark_query(f"ALTER TABLE {TABLE_NAME} DROP COLUMN outer.items.element.b") + + check_schema_and_data( + instance, + table_function, + [ + ['outer', 'Tuple(\\n items Array(Tuple(\\n c Nullable(Int32),\\n renamed_a Nullable(Int32))))'], + ], + [ + ["([(NULL,1),(NULL,2)])"], + ], + ) + + execute_spark_query(f"ALTER TABLE {TABLE_NAME} RENAME COLUMN outer.items TO renamed_items") + + check_schema_and_data( + instance, + table_function, + [ + ['outer', 'Tuple(\\n renamed_items Array(Tuple(\\n c Nullable(Int32),\\n renamed_a Nullable(Int32))))'], + ], + [ + ["([(NULL,1),(NULL,2)])"], + ], + ) diff --git a/tests/integration/test_storage_iceberg_schema_evolution/test_struct_with_nested_map_evolved.py b/tests/integration/test_storage_iceberg_schema_evolution/test_struct_with_nested_map_evolved.py new file mode 100644 index 0000000000000..6870275b1dc2f --- /dev/null +++ b/tests/integration/test_storage_iceberg_schema_evolution/test_struct_with_nested_map_evolved.py @@ -0,0 +1,112 @@ +import pytest + +from helpers.iceberg_utils import ( + default_upload_directory, + get_uuid_str, + get_creation_expression, + check_schema_and_data +) + + +@pytest.mark.parametrize("format_version", ["1", "2"]) +@pytest.mark.parametrize("storage_type", ["s3", "local"]) +def test_struct_with_nested_map_evolved( + started_cluster_iceberg_schema_evolution, format_version, storage_type +): + instance = started_cluster_iceberg_schema_evolution.instances["node1"] + spark = started_cluster_iceberg_schema_evolution.spark_session + TABLE_NAME = ( + "test_struct_with_nested_map_evolved_" + + format_version + + "_" + + storage_type + + "_" + + get_uuid_str() + ) + + def execute_spark_query(query: str): + spark.sql(query) + default_upload_directory( + started_cluster_iceberg_schema_evolution, + storage_type, + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + return + + execute_spark_query(f"DROP TABLE IF EXISTS {TABLE_NAME}") + execute_spark_query( + f""" + CREATE TABLE {TABLE_NAME} ( + outer STRUCT< + entries: MAP> + > + ) + USING iceberg + OPTIONS ('format-version'='{format_version}') + """ + ) + + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES + (named_struct('entries', MAP(1, named_struct('a', 10, 'b', 'hello')))); + """ + ) + + table_function = get_creation_expression( + storage_type, TABLE_NAME, started_cluster_iceberg_schema_evolution, table_function=True + ) + + check_schema_and_data( + instance, + table_function, + [ + ['outer', 'Tuple(\\n entries Map(Int32, Tuple(\\n a Nullable(Int32),\\n b Nullable(String))))'], + ], + [ + ["({1:(10,'hello')})"], + ], + ) + + execute_spark_query(f"ALTER TABLE {TABLE_NAME} ADD COLUMN outer.entries.value.c INT") + + check_schema_and_data( + instance, + table_function, + [ + ['outer', 'Tuple(\\n entries Map(Int32, Tuple(\\n a Nullable(Int32),\\n b Nullable(String),\\n c Nullable(Int32))))'], + ], + [ + ["({1:(10,'hello',NULL)})"], + ], + ) + + execute_spark_query(f"ALTER TABLE {TABLE_NAME} ALTER COLUMN outer.entries.value.c FIRST") + + check_schema_and_data( + instance, + table_function, + [ + ['outer', 'Tuple(\\n entries Map(Int32, Tuple(\\n c Nullable(Int32),\\n a Nullable(Int32),\\n b Nullable(String))))'], + ], + [ + ["({1:(NULL,10,'hello')})"], + ], + ) + + execute_spark_query(f"ALTER TABLE {TABLE_NAME} DROP COLUMN outer.entries.value.b") + + check_schema_and_data( + instance, + table_function, + [ + ['outer', 'Tuple(\\n entries Map(Int32, Tuple(\\n c Nullable(Int32),\\n a Nullable(Int32))))'], + ], + [ + ["({1:(NULL,10)})"], + ], + ) diff --git a/tests/integration/test_storage_iceberg_with_spark/configs/users.d/users.xml b/tests/integration/test_storage_iceberg_with_spark/configs/users.d/users.xml index 4b6ba057ecb16..2f9966e1d71a2 100644 --- a/tests/integration/test_storage_iceberg_with_spark/configs/users.d/users.xml +++ b/tests/integration/test_storage_iceberg_with_spark/configs/users.d/users.xml @@ -4,6 +4,7 @@ default 1 + 1 diff --git a/tests/integration/test_storage_iceberg_with_spark/test_manifest_data_path_security.py b/tests/integration/test_storage_iceberg_with_spark/test_manifest_data_path_security.py new file mode 100644 index 0000000000000..7121441706fe8 --- /dev/null +++ b/tests/integration/test_storage_iceberg_with_spark/test_manifest_data_path_security.py @@ -0,0 +1,199 @@ +import json +import os +import tempfile + +from helpers.iceberg_utils import ( + get_uuid_str, + default_upload_directory, +) + + +def patch_manifest_data_path(instance, table_dir, new_data_path): + """ + Reads Iceberg metadata from the container, finds the manifest file, + does a binary find-and-replace of the data file path inside it, + preserving all Avro metadata (partition-spec, schema, etc.). + + Returns the original data file path so the caller can copy it to new_data_path. + """ + import avro.datafile + import avro.io + + metadata_dir = os.path.join(table_dir, "metadata") + + # 1. Find the latest metadata json (version-hint.text -> vN.metadata.json) + version_hint = instance.exec_in_container( + ["cat", os.path.join(metadata_dir, "version-hint.text")] + ).strip() + metadata_json_path = os.path.join(metadata_dir, f"v{version_hint}.metadata.json") + metadata_json_str = instance.exec_in_container(["cat", metadata_json_path]) + metadata = json.loads(metadata_json_str) + + # 2. Find current snapshot's manifest-list path + current_snapshot_id = metadata["current-snapshot-id"] + snapshot = next(s for s in metadata["snapshots"] if s["snapshot-id"] == current_snapshot_id) + manifest_list_path = snapshot["manifest-list"] + + # 3. Download manifest-list (Avro) from container to read manifest path + manifest_list_local = tempfile.mktemp(suffix=".avro") + instance.copy_file_from_container(manifest_list_path, manifest_list_local) + + with open(manifest_list_local, "rb") as f: + reader = avro.datafile.DataFileReader(f, avro.io.DatumReader()) + manifest_entries = list(reader) + reader.close() + + assert len(manifest_entries) > 0, "Expected at least one manifest entry" + manifest_path = manifest_entries[0]["manifest_path"] + + # 4. Download the manifest file (Avro) — read records to get original path + manifest_local = tempfile.mktemp(suffix=".avro") + instance.copy_file_from_container(manifest_path, manifest_local) + + with open(manifest_local, "rb") as f: + reader = avro.datafile.DataFileReader(f, avro.io.DatumReader()) + records = list(reader) + reader.close() + + assert len(records) > 0, "Expected at least one record in manifest" + original_data_path = records[0]["data_file"]["file_path"] + + # 5. Patch the manifest using the avro library: read with DataFileReader, + # then write back with DataFileWriter, copying all file-level metadata + # (partition-spec, schema, etc.) from the original file. + with open(manifest_local, "rb") as f: + reader = avro.datafile.DataFileReader(f, avro.io.DatumReader()) + writer_schema = reader.datum_reader.writers_schema + file_meta = dict(reader.meta) # preserves partition-spec, schema, etc. + codec = reader.codec + all_records = list(reader) + reader.close() + + for record in all_records: + record["data_file"]["file_path"] = new_data_path + + patched_manifest_local = tempfile.mktemp(suffix=".avro") + with open(patched_manifest_local, "wb") as f: + writer = avro.datafile.DataFileWriter(f, avro.io.DatumWriter(), writer_schema, codec=codec) + # Copy all non-standard metadata from original (partition-spec, schema, etc.) + for key, value in file_meta.items(): + if key not in ("avro.schema", "avro.codec"): + writer.set_meta(key, value) + for record in all_records: + writer.append(record) + writer.flush() + writer.close() + + # 6. Upload patched manifest to the container (overwrite) + instance.copy_file_to_container(patched_manifest_local, manifest_path) + + # Cleanup temp files + os.unlink(manifest_list_local) + os.unlink(manifest_local) + os.unlink(patched_manifest_local) + + return original_data_path + + +def test_manifest_data_path_outside_user_files(started_cluster_iceberg_with_spark): + """ + Test that ClickHouse rejects reading an Iceberg table when the manifest + entry points to a data file outside user_files directory via path traversal. + """ + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_manifest_path_security_" + get_uuid_str() + + spark.sql( + f"CREATE TABLE {TABLE_NAME} (id bigint, data string) USING iceberg " + f"TBLPROPERTIES ('format-version' = '2')" + ) + spark.sql(f"INSERT INTO {TABLE_NAME} VALUES (1, 'hello')") + + table_dir = f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}" + default_upload_directory( + started_cluster_iceberg_with_spark, + "local", + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + + evil_dir = "/tmp/evil_data" + evil_data_path = "/var/lib/clickhouse/user_files/../../../../tmp/evil_data/data.parquet" + + original_data_path = patch_manifest_data_path(instance, table_dir, evil_data_path) + assert original_data_path is not None, "Could not find original data file path in manifest" + + instance.exec_in_container(["bash", "-c", f"mkdir -p {evil_dir}"]) + instance.exec_in_container(["bash", "-c", f"cp {original_data_path} {evil_data_path}"]) + + create_sql = ( + f"DROP TABLE IF EXISTS {TABLE_NAME};\n" + f"CREATE TABLE {TABLE_NAME} " + f"ENGINE=IcebergLocal(local, path = '{table_dir}', format='Parquet')" + ) + instance.query(create_sql) + + error = instance.query_and_get_error(f"SELECT * FROM {TABLE_NAME}") + assert "PATH_ACCESS_DENIED" in error, ( + f"Expected PATH_ACCESS_DENIED but got: {error}" + ) + + +def test_manifest_data_path_symlink_escape(started_cluster_iceberg_with_spark): + """ + Test that ClickHouse rejects reading an Iceberg table when the manifest + entry uses a symlink inside user_files that points outside. + + A path like /allowed_link/file where allowed_link -> /tmp + passes a purely lexical check (lexically_normal / lexically_relative) + but accesses data outside user_files via symlink resolution. + The check must use weakly_canonical (or equivalent) to catch this. + """ + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_symlink_escape_" + get_uuid_str() + + spark.sql( + f"CREATE TABLE {TABLE_NAME} (id bigint, data string) USING iceberg " + f"TBLPROPERTIES ('format-version' = '2')" + ) + spark.sql(f"INSERT INTO {TABLE_NAME} VALUES (1, 'hello')") + + table_dir = f"/var/lib/clickhouse/user_files/iceberg_data/default/{TABLE_NAME}" + default_upload_directory( + started_cluster_iceberg_with_spark, + "local", + f"/iceberg_data/default/{TABLE_NAME}/", + f"/iceberg_data/default/{TABLE_NAME}/", + ) + + evil_dir = "/tmp/symlink_evil_data" + evil_data_path = f"{evil_dir}/data.parquet" + + original_data_path = patch_manifest_data_path( + instance, table_dir, + # This path looks valid lexically — it's inside user_files — + # but allowed_link is a symlink to /tmp/symlink_evil_data + "/var/lib/clickhouse/user_files/allowed_link/data.parquet" + ) + assert original_data_path is not None, "Could not find original data file path in manifest" + + instance.exec_in_container(["bash", "-c", f"mkdir -p {evil_dir}"]) + instance.exec_in_container(["bash", "-c", f"cp {original_data_path} {evil_data_path}"]) + instance.exec_in_container([ + "bash", "-c", + f"ln -sfn {evil_dir} /var/lib/clickhouse/user_files/allowed_link" + ]) + + create_sql = ( + f"DROP TABLE IF EXISTS {TABLE_NAME};\n" + f"CREATE TABLE {TABLE_NAME} " + f"ENGINE=IcebergLocal(local, path = '{table_dir}', format='Parquet')" + ) + instance.query(create_sql) + + error = instance.query_and_get_error(f"SELECT * FROM {TABLE_NAME}") + assert "PATH_ACCESS_DENIED" in error, ( + f"Expected PATH_ACCESS_DENIED but got: {error}" + ) diff --git a/tests/integration/test_storage_iceberg_with_spark/test_partition_pruning_with_functions.py b/tests/integration/test_storage_iceberg_with_spark/test_partition_pruning_with_functions.py new file mode 100644 index 0000000000000..51bb446b5f316 --- /dev/null +++ b/tests/integration/test_storage_iceberg_with_spark/test_partition_pruning_with_functions.py @@ -0,0 +1,268 @@ +import pytest + +from helpers.iceberg_utils import ( + check_validity_and_get_prunned_files_general, + execute_spark_query_general, + get_creation_expression, + get_uuid_str, +) + + +@pytest.mark.parametrize("storage_type", ["s3", "local"]) +def test_partition_pruning_with_functions(started_cluster_iceberg_with_spark, storage_type): + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_partition_pruning_with_functions_" + storage_type + "_" + get_uuid_str() + + def execute_spark_query(query: str): + return execute_spark_query_general( + spark, started_cluster_iceberg_with_spark, storage_type, TABLE_NAME, query + ) + + execute_spark_query( + f""" + CREATE TABLE {TABLE_NAME} ( + ts_day TIMESTAMP, + ts_hour TIMESTAMP, + ts_month TIMESTAMP, + d_year DATE, + tag INT + ) + USING iceberg + PARTITIONED BY (days(ts_day), hours(ts_hour), months(ts_month), years(d_year)) + OPTIONS('format-version'='2') + """ + ) + + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES + (TIMESTAMP '2024-01-20 10:00:00', TIMESTAMP '2024-01-20 10:00:00', TIMESTAMP '2024-01-20 10:00:00', DATE '2024-01-20', 1), + (TIMESTAMP '2024-01-21 11:00:00', TIMESTAMP '2024-01-20 11:00:00', TIMESTAMP '2024-02-20 10:00:00', DATE '2025-01-20', 2), + (TIMESTAMP '2024-02-20 10:00:00', TIMESTAMP '2024-01-21 10:00:00', TIMESTAMP '2024-03-20 10:00:00', DATE '2026-01-20', 3), + (TIMESTAMP '2025-02-20 10:00:00', TIMESTAMP '2024-01-21 11:00:00', TIMESTAMP '2025-01-20 10:00:00', DATE '2027-01-20', 4); + """ + ) + + creation_expression = get_creation_expression( + storage_type, TABLE_NAME, started_cluster_iceberg_with_spark, table_function=True + ) + + def check_validity_and_get_prunned_files(select_expression): + settings1 = {"use_iceberg_partition_pruning": 0, "session_timezone": "UTC"} + settings2 = {"use_iceberg_partition_pruning": 1, "session_timezone": "UTC"} + return check_validity_and_get_prunned_files_general( + instance, TABLE_NAME, settings1, settings2, "IcebergPartitionPrunedFiles", select_expression + ) + + def select(where): + return f"SELECT * FROM {creation_expression} WHERE {where} ORDER BY ALL" + + # A filter that wraps the partition source column in a monotonic function must still be able to + # use the partition value, which is what https://github.com/ClickHouse/ClickHouse/issues/103433 + # reported for `toDate`. + assert check_validity_and_get_prunned_files(select("toDate(ts_day) = toDate('2024-01-20')")) == 3 + assert check_validity_and_get_prunned_files(select("toStartOfDay(ts_day) = toDateTime64('2024-01-20 00:00:00', 6)")) == 3 + assert check_validity_and_get_prunned_files(select("toYYYYMMDD(ts_day) = 20240120")) == 3 + assert check_validity_and_get_prunned_files(select("toDate(ts_day) IN (toDate('2024-01-20'), toDate('2024-02-20'))")) == 2 + assert check_validity_and_get_prunned_files(select("toDate(ts_day) > toDate('2024-02-01')")) == 2 + assert check_validity_and_get_prunned_files(select("toYear(ts_day) = 2025")) == 3 + assert check_validity_and_get_prunned_files(select("toStartOfHour(ts_hour) = toDateTime64('2024-01-20 10:00:00', 6)")) == 3 + assert check_validity_and_get_prunned_files(select("toDate(ts_hour) = toDate('2024-01-21')")) == 2 + assert check_validity_and_get_prunned_files(select("toStartOfMonth(ts_month) = toDate('2024-02-01')")) == 3 + assert check_validity_and_get_prunned_files(select("toYYYYMM(ts_month) = 202503")) == 4 + assert check_validity_and_get_prunned_files(select("toYear(d_year) = 2026")) == 3 + assert check_validity_and_get_prunned_files(select("toStartOfYear(d_year) = toDate('2028-01-01')")) == 4 + + # A constant that is not aligned to the transform's granularity: 10:30 is inside the 10:00 hour, + # so the file of that hour must survive, and the same for a day and a month boundary. + assert check_validity_and_get_prunned_files(select("toStartOfHour(ts_hour) < toDateTime64('2024-01-20 10:30:00', 6)")) == 3 + assert check_validity_and_get_prunned_files(select("toStartOfHour(ts_hour) >= toDateTime64('2024-01-20 10:30:00', 6)")) == 1 + assert check_validity_and_get_prunned_files(select("toDate(ts_day) < toDate('2024-01-20') + INTERVAL 12 HOUR")) == 3 + assert check_validity_and_get_prunned_files(select("toStartOfMonth(ts_month) > toDate('2024-02-10')")) == 2 + + # A single day is one weekday, so the partition value answers this too: only 2024-01-20 is a Saturday. + assert check_validity_and_get_prunned_files(select("toDayOfWeek(ts_day) = 6")) == 3 + + # The partition value of a `day` transform says nothing about the hour of the day, so a filter on + # it must not prune anything. + assert check_validity_and_get_prunned_files(select("toHour(ts_day) = 10")) == 0 + + +@pytest.mark.parametrize("storage_type", ["s3", "local"]) +def test_partition_pruning_with_functions_before_epoch(started_cluster_iceberg_with_spark, storage_type): + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_partition_pruning_before_epoch_" + storage_type + "_" + get_uuid_str() + + def execute_spark_query(query: str): + return execute_spark_query_general( + spark, started_cluster_iceberg_with_spark, storage_type, TABLE_NAME, query + ) + + execute_spark_query( + f""" + CREATE TABLE {TABLE_NAME} (ts TIMESTAMP, d DATE, tag INT) + USING iceberg + PARTITIONED BY (days(ts), years(d)) + OPTIONS('format-version'='2') + """ + ) + + # The transforms count from 1970, so these partition values are negative. + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES + (TIMESTAMP '1969-11-15 07:00:00', DATE '1969-11-15', 1), + (TIMESTAMP '1969-12-31 23:00:00', DATE '1970-01-01', 2), + (TIMESTAMP '1970-01-01 00:30:00', DATE '1971-06-01', 3), + (TIMESTAMP '2024-01-20 10:00:00', DATE '2024-01-20', 4); + """ + ) + + creation_expression = get_creation_expression( + storage_type, TABLE_NAME, started_cluster_iceberg_with_spark, table_function=True + ) + + def check_validity_and_get_prunned_files(select_expression): + settings1 = {"use_iceberg_partition_pruning": 0, "session_timezone": "UTC"} + settings2 = {"use_iceberg_partition_pruning": 1, "session_timezone": "UTC"} + return check_validity_and_get_prunned_files_general( + instance, TABLE_NAME, settings1, settings2, "IcebergPartitionPrunedFiles", select_expression + ) + + def select(where): + return f"SELECT * FROM {creation_expression} WHERE {where} ORDER BY ALL" + + assert check_validity_and_get_prunned_files(select("toDate32(ts) = toDate32('1969-11-15')")) == 3 + assert check_validity_and_get_prunned_files(select("toDate32(ts) >= toDate32('1970-01-01')")) == 2 + assert check_validity_and_get_prunned_files(select("toDate32(ts) < toDate32('1970-01-01')")) == 2 + assert check_validity_and_get_prunned_files(select("toYear(d) = 1969")) == 3 + assert check_validity_and_get_prunned_files(select("toYear(d) >= 1971")) == 2 + + +@pytest.mark.parametrize("storage_type", ["s3", "local"]) +def test_partition_pruning_with_functions_before_epoch_separate_commits(started_cluster_iceberg_with_spark, storage_type): + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_partition_pruning_before_epoch_separate_commits_" + storage_type + "_" + get_uuid_str() + + def execute_spark_query(query: str): + return execute_spark_query_general( + spark, started_cluster_iceberg_with_spark, storage_type, TABLE_NAME, query + ) + + execute_spark_query( + f""" + CREATE TABLE {TABLE_NAME} (ts TIMESTAMP, d DATE, tag INT) + USING iceberg + PARTITIONED BY (days(ts), years(d)) + OPTIONS('format-version'='2') + """ + ) + + # One manifest file per commit, so the partition summaries of the first manifest are negative on + # both of their sides, while the ones of the second manifest are positive on both. + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES + (TIMESTAMP '1969-11-15 07:00:00', DATE '1969-11-15', 1), + (TIMESTAMP '1969-12-31 23:00:00', DATE '1969-06-01', 2); + """ + ) + + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES + (TIMESTAMP '2024-01-20 10:00:00', DATE '2024-01-20', 3); + """ + ) + + creation_expression = get_creation_expression( + storage_type, TABLE_NAME, started_cluster_iceberg_with_spark, table_function=True + ) + + def check_validity_and_get_prunned_files(select_expression): + settings1 = {"use_iceberg_partition_pruning": 0, "session_timezone": "UTC"} + settings2 = {"use_iceberg_partition_pruning": 1, "session_timezone": "UTC"} + return check_validity_and_get_prunned_files_general( + instance, TABLE_NAME, settings1, settings2, "IcebergPartitionPrunedFiles", select_expression + ) + + def select(where): + return f"SELECT * FROM {creation_expression} WHERE {where} ORDER BY ALL" + + assert check_validity_and_get_prunned_files(select("toDate32(ts) = toDate32('1969-11-15')")) == 2 + assert check_validity_and_get_prunned_files(select("toDate32(ts) >= toDate32('1970-01-01')")) == 2 + assert check_validity_and_get_prunned_files(select("toYear(d) = 1969")) == 1 + + +@pytest.mark.parametrize("storage_type", ["s3", "local"]) +def test_partition_pruning_without_functions(started_cluster_iceberg_with_spark, storage_type): + instance = started_cluster_iceberg_with_spark.instances["node1"] + spark = started_cluster_iceberg_with_spark.spark_session + TABLE_NAME = "test_partition_pruning_without_functions_" + storage_type + "_" + get_uuid_str() + + def execute_spark_query(query: str): + return execute_spark_query_general( + spark, started_cluster_iceberg_with_spark, storage_type, TABLE_NAME, query + ) + + execute_spark_query( + f""" + CREATE TABLE {TABLE_NAME} ( + ts_day TIMESTAMP, + ts_hour TIMESTAMP, + ts_month TIMESTAMP, + d_year DATE, + tag INT + ) + USING iceberg + PARTITIONED BY (days(ts_day), hours(ts_hour), months(ts_month), years(d_year)) + OPTIONS('format-version'='2') + """ + ) + + execute_spark_query( + f""" + INSERT INTO {TABLE_NAME} VALUES + (TIMESTAMP '2024-01-20 10:00:00', TIMESTAMP '2024-01-20 10:00:00', TIMESTAMP '2024-01-20 10:00:00', DATE '2024-01-20', 1), + (TIMESTAMP '2024-01-21 11:00:00', TIMESTAMP '2024-01-20 11:00:00', TIMESTAMP '2024-02-20 10:00:00', DATE '2025-01-20', 2), + (TIMESTAMP '2024-02-20 10:00:00', TIMESTAMP '2024-01-21 10:00:00', TIMESTAMP '2024-03-20 10:00:00', DATE '2026-01-20', 3), + (TIMESTAMP '2025-02-20 10:00:00', TIMESTAMP '2024-01-21 11:00:00', TIMESTAMP '2025-01-20 10:00:00', DATE '2027-01-20', 4); + """ + ) + + creation_expression = get_creation_expression( + storage_type, TABLE_NAME, started_cluster_iceberg_with_spark, table_function=True + ) + + def check_validity_and_get_prunned_files(select_expression): + settings1 = {"use_iceberg_partition_pruning": 0, "session_timezone": "UTC"} + settings2 = {"use_iceberg_partition_pruning": 1, "session_timezone": "UTC"} + return check_validity_and_get_prunned_files_general( + instance, TABLE_NAME, settings1, settings2, "IcebergPartitionPrunedFiles", select_expression + ) + + def select(where): + return f"SELECT * FROM {creation_expression} WHERE {where} ORDER BY ALL" + + assert check_validity_and_get_prunned_files(select("ts_day = '2024-01-20 10:00:00'")) == 3 + assert check_validity_and_get_prunned_files(select("ts_day < '2024-01-21 00:00:00'")) == 3 + assert check_validity_and_get_prunned_files(select("ts_day >= '2024-02-01 00:00:00'")) == 2 + assert check_validity_and_get_prunned_files(select("ts_day BETWEEN '2024-01-21 00:00:00' AND '2024-02-21 00:00:00'")) == 2 + assert check_validity_and_get_prunned_files(select("ts_day IN ('2024-01-20 10:00:00', '2024-02-20 10:00:00')")) == 2 + + assert check_validity_and_get_prunned_files(select("ts_hour = '2024-01-20 11:00:00'")) == 3 + assert check_validity_and_get_prunned_files(select("ts_hour > '2024-01-21 09:00:00'")) == 2 + + assert check_validity_and_get_prunned_files(select("ts_month = '2024-02-20 10:00:00'")) == 3 + assert check_validity_and_get_prunned_files(select("ts_month < '2024-03-01 00:00:00'")) == 2 + + assert check_validity_and_get_prunned_files(select("d_year = '2026-01-20'")) == 3 + assert check_validity_and_get_prunned_files(select("d_year < '2025-01-01'")) == 3 + assert check_validity_and_get_prunned_files(select("d_year >= '2026-01-01'")) == 2 + assert check_validity_and_get_prunned_files(select("d_year IN ('2024-01-20', '2026-01-20')")) == 2 + + assert check_validity_and_get_prunned_files(select("ts_day = '2024-01-20 12:00:00'")) == 3 + assert check_validity_and_get_prunned_files(select("d_year = '2026-06-01'")) == 3 diff --git a/tests/integration/test_storage_iceberg_with_spark_cache/configs/users.d/users.xml b/tests/integration/test_storage_iceberg_with_spark_cache/configs/users.d/users.xml index 4b6ba057ecb16..2f9966e1d71a2 100644 --- a/tests/integration/test_storage_iceberg_with_spark_cache/configs/users.d/users.xml +++ b/tests/integration/test_storage_iceberg_with_spark_cache/configs/users.d/users.xml @@ -4,6 +4,7 @@ default 1 + 1 diff --git a/tests/integration/test_storage_kafka/test_batch_fast.py b/tests/integration/test_storage_kafka/test_batch_fast.py index c8296dd2bbd34..98028485ce17f 100644 --- a/tests/integration/test_storage_kafka/test_batch_fast.py +++ b/tests/integration/test_storage_kafka/test_batch_fast.py @@ -1855,6 +1855,57 @@ def test_kafka_producer_consumer_separate_settings( assert property_in_log in kafka_producer_applied_properties +@pytest.mark.parametrize( + "create_query_generator", + [ + k.generate_old_create_table_query, + k.generate_new_create_table_query, + ], +) +def test_kafka_password_not_logged(kafka_cluster, create_query_generator): + suffix = k.random_string(6) + kafka_table = f"kafka_{suffix}" + username = f"kafka_user_{suffix}" + password = f"secret_kafka_password_{suffix}" + + instance.rotate_logs() + instance.query( + create_query_generator( + kafka_table, + "key UInt64", + topic_list="password_not_logged", + consumer_group="test", + settings={ + "kafka_sasl_username": username, + "kafka_sasl_password": password, + }, + ) + ) + + # Create an mv to initialize the librdkafka consumers + instance.query(f"CREATE MATERIALIZED VIEW test.{kafka_table}_view ENGINE=MergeTree ORDER BY tuple() AS SELECT * FROM test.{kafka_table}") + instance.wait_for_log_line(f"{kafka_table}.*Created #0 consumer") + instance.query(f"DROP TABLE test.{kafka_table}_view") + instance.query(f"INSERT INTO test.{kafka_table} VALUES (1)") + + assert instance.contains_in_log(f"{kafka_table}.*Kafka producer created") + + # The property-logging loops ran for both the consumer and the producer, + # but they hid the values of the sensitive properties. `sasl.username` is + # hidden because librdkafka marks it with the _RK_SENSITIVE flag, not + # because of the name, so it validates the generated blacklist. + for client_type in ["Consumer", "Producer"]: + for property_name in ["sasl.username", "sasl.password"]: + assert instance.contains_in_log( + f"{kafka_table}.*{client_type} set property {property_name}:\\[HIDDEN\\]" + ) + # The username still appears in the logged CREATE TABLE text (only + # kafka_sasl_password is masked there), so check only the password value. + assert not instance.contains_in_log(password) + + instance.query(f"DROP TABLE test.{kafka_table}") + + @pytest.mark.parametrize( "create_query_generator, log_line", [ diff --git a/tests/integration/test_storage_kafka/test_batch_slow_0.py b/tests/integration/test_storage_kafka/test_batch_slow_0.py index 03e7297aa3dd6..fee476e50da14 100644 --- a/tests/integration/test_storage_kafka/test_batch_slow_0.py +++ b/tests/integration/test_storage_kafka/test_batch_slow_0.py @@ -301,7 +301,6 @@ def test_kafka_formats_with_broken_message(kafka_cluster, create_query_generator data_prefix = data_prefix + [""] if format_opts.get("printable", False) == False: raw_message = "hex(_raw_message)" - k.kafka_produce(kafka_cluster, topic_name, data_prefix + data_sample) create_query = create_query_generator( f"kafka_{format_name}", "id Int64, blockNo UInt16, val1 String, val2 Float32, val3 UInt8", @@ -313,6 +312,10 @@ def test_kafka_formats_with_broken_message(kafka_cluster, create_query_generator "kafka_flush_interval_ms": 1000, }, ) + # Create both materialized views, then detach/re-attach the Kafka table, + # before producing any message. Creating the first view starts the + # streaming loop, so producing earlier lets the loop consume and commit + # the broken message before the errors view is attached, leaving it empty. instance.query( f""" DROP TABLE IF EXISTS test.kafka_{format_name}; @@ -328,8 +331,12 @@ def test_kafka_formats_with_broken_message(kafka_cluster, create_query_generator CREATE MATERIALIZED VIEW test.kafka_errors_{format_name}_mv ENGINE=MergeTree ORDER BY tuple() AS SELECT {raw_message} as raw_message, _error as error, _topic as topic, _partition as partition, _offset as offset FROM test.kafka_{format_name} WHERE length(_error) > 0; + + DETACH TABLE test.kafka_{format_name}; + ATTACH TABLE test.kafka_{format_name}; """ ) + k.kafka_produce(kafka_cluster, topic_name, data_prefix + data_sample) raw_expected = """\ 0 0 AM 0.5 1 {topic_name} 0 {offset_0} @@ -383,13 +390,27 @@ def test_kafka_formats_with_broken_message(kafka_cluster, create_query_generator assert TSV(result) == TSV(expected), "Proper result for format: {}".format( format_name ) - errors_result = json.loads( - instance.query( - "SELECT raw_message, error FROM test.kafka_errors_{format_name}_mv format JSONEachRow".format( - format_name=format_name - ) - ) + # 26.3 predates the per-format retry master wraps this query in, so bring in + # master's retry + guard directly, with #108252's bumped retry_count and + # empty-result guard already folded in. + errors_query = "SELECT raw_message, error FROM test.kafka_errors_{format_name}_mv FORMAT JSONEachRow".format( + format_name=format_name + ) + errors_text = instance.query_with_retry( + errors_query, + retry_count=60, + sleep_time=1, + check_callback=lambda res: len(res) > 0, + ) + # query_with_retry returns the last result even if check_callback never + # passed, so guard against an empty error MV before json.loads (which + # would otherwise raise an opaque "Expecting value" JSONDecodeError). + assert ( + len(errors_text) > 0 + ), "Error row for format {} did not appear in kafka_errors_{}_mv".format( + format_name, format_name ) + errors_result = json.loads(errors_text) # print(errors_result.strip()) # print(errors_expected.strip()) assert ( diff --git a/tests/integration/test_storage_mysql/configs/users.xml b/tests/integration/test_storage_mysql/configs/users.xml index a11985dd113d2..845ae0e64ed70 100644 --- a/tests/integration/test_storage_mysql/configs/users.xml +++ b/tests/integration/test_storage_mysql/configs/users.xml @@ -13,6 +13,7 @@ default 1 + 1 diff --git a/tests/integration/test_storage_nats/__init__.py b/tests/integration/test_storage_nats/__init__.py new file mode 100644 index 0000000000000..3a903416ac795 --- /dev/null +++ b/tests/integration/test_storage_nats/__init__.py @@ -0,0 +1,4 @@ +import pytest + +# we want to have pytest assert introspection in the helpers +pytest.register_assert_rewrite("test_storage_nats.common") diff --git a/tests/integration/test_storage_nats/common.py b/tests/integration/test_storage_nats/common.py new file mode 100644 index 0000000000000..dafb97aee6f3e --- /dev/null +++ b/tests/integration/test_storage_nats/common.py @@ -0,0 +1,151 @@ +import asyncio +import logging +import os.path as p +import shlex +import subprocess +import time +import nats + +from helpers.cluster import check_nats_is_available +from helpers.cluster import nats_connect_ssl as nats_connect_ssl # re-exported for the test modules +from helpers.test_tools import TSV + +def wait_nats_to_start(cluster, timeout=180): + start = time.time() + while time.time() - start < timeout: + try: + if asyncio.run(check_nats_is_available(cluster)): + logging.debug("NATS is available") + return + time.sleep(0.5) + except Exception as ex: + logging.debug("Can't connect to NATS " + str(ex)) + time.sleep(0.5) + + assert False, "NATS is unavailable" + +# function to check if nats is paused, because in some cases we successfully connected to it after calling pause_container +def wait_nats_paused(cluster, timeout=180): + start = time.time() + while time.time() - start < timeout: + try: + asyncio.run(check_nats_is_available(cluster)) + time.sleep(0.5) + except nats.errors.NoServersError: + logging.debug("NATS is paused") + return + except Exception as ex: + logging.warning("Detect NATS status failed with error \"" + str(ex) + "\" - continue waiting for proper status...") + time.sleep(0.5) + + assert False, "NATS is not paused" + +def check_query_result(instance, query, retry_count=60): + result = instance.query_with_retry(query, retry_count=retry_count, ignore_error=True, check_callback=lambda result: check_result(result)) + check_result(result, True) + +def check_result(query_result, check=False, ref_file="test_nats_json.reference"): + fpath = p.join(p.dirname(__file__), ref_file) + with open(fpath) as reference: + if check: + assert TSV(query_result) == TSV(reference) + else: + return TSV(query_result) == TSV(reference) + + +def kill_nats(cluster): + p = subprocess.Popen(("docker", "stop", cluster.nats_docker_id), stdout=subprocess.PIPE) + p.communicate() + return p.returncode == 0 + +def revive_nats(cluster): + p = subprocess.Popen(("docker", "start", cluster.nats_docker_id), stdout=subprocess.PIPE) + p.communicate() + wait_nats_to_start(cluster) + +def wait_query_result(instance, query, wait_query_result, sleep_timeout = 0.5, retry_count = 60): + query_result = instance.query_with_retry( + query, + retry_count=retry_count, + sleep_time=sleep_timeout, + ignore_error=True, + check_callback=lambda result: int(result) == wait_query_result) + + assert int(query_result) == wait_query_result + +def wait_for_table_is_ready(instance, table_name, sleep_timeout = 0.5, time_limit_sec = 60): + deadline = time.monotonic() + time_limit_sec + while (not check_table_is_ready(instance, table_name)) and time.monotonic() < deadline: + time.sleep(sleep_timeout) + + assert(check_table_is_ready(instance, table_name)) + +# waiting for subscription to nats subjects (after subscription direct selection is not available and completed with an error) +def wait_for_mv_attached_to_table(instance, table_name, sleep_timeout = 0.5, time_limit_sec = 60): + deadline = time.monotonic() + time_limit_sec + while check_table_is_ready(instance, table_name) and time.monotonic() < deadline: + time.sleep(sleep_timeout) + + assert(not check_table_is_ready(instance, table_name)) + + # Refusing a direct `SELECT` only proves that a materialized view is attached; the + # background task subscribes to the subjects slightly later. Core NATS has no backlog, so + # a message published in that window is dropped and never reaches the view. + wait_for_streaming_started(instance, table_name, time_limit_sec) + +STREAMING_STARTED_LOG_LINE = "Started streaming to [0-9]+ attached views" + +def wait_for_streaming_started(instance, table_name, time_limit_sec = 60): + # The background task logs the line on every iteration, right after the consumer has + # subscribed, so a fresh occurrence means the subscription is live now. Tests within a + # module reuse table names, hence the wait is anchored past the occurrences the previous + # tests left in the window `wait_for_log_line` looks at. + log_line = "{}.*{}".format(table_name, STREAMING_STARTED_LOG_LINE) + seen = count_in_recent_log(instance, log_line) + instance.wait_for_log_line(log_line, timeout=time_limit_sec, repetitions=seen + 1) + +def count_in_recent_log(instance, pattern, look_behind_lines = 10000): + result = instance.exec_in_container( + [ + "bash", + "-c", + "tail -n{} /var/log/clickhouse-server/clickhouse-server.log | grep -Ec {} || true".format( + look_behind_lines, shlex.quote(pattern) + ), + ] + ) + return int(result.strip() or 0) + +def log_line_count(instance): + # Absolute line count of the whole log, to be used as an anchor for `count_in_log_after`. + result = instance.exec_in_container( + ["bash", "-c", "wc -l < /var/log/clickhouse-server/clickhouse-server.log"] + ) + return int(result.strip() or 0) + +def count_in_log_after(instance, pattern, after_line): + # Counts matches strictly after an absolute line offset, so a count of zero really means "no + # such line was written since the anchor". Unlike `count_in_recent_log`, which looks at a + # fixed-size tail, this cannot lose an older match as the log grows. + # + # An absolute offset only goes stale if the log rotates between the anchor and the count. The + # integration config rotates at 1000M keeping 10 files + # (`helpers/0_common_instance_config.xml`), which is hours away at these tests' log rate, and + # callers assert the log has not shrunk below the anchor so a rotation fails loudly. + result = instance.exec_in_container( + [ + "bash", + "-c", + "tail -n +{} /var/log/clickhouse-server/clickhouse-server.log | grep -Ec {} || true".format( + after_line + 1, shlex.quote(pattern) + ), + ] + ) + return int(result.strip() or 0) + +def check_table_is_ready(instance, table_name): + try: + instance.query("SELECT * FROM {}".format(table_name)) + return True + except Exception: + return False diff --git a/tests/integration/test_storage_nats/nats_auth_proxy.py b/tests/integration/test_storage_nats/nats_auth_proxy.py new file mode 100644 index 0000000000000..2084c1f7c383a --- /dev/null +++ b/tests/integration/test_storage_nats/nats_auth_proxy.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +"""A NATS proxy which rejects credentials on demand, used to drive a JetStream table through a +credential rotation against a real broker. + +`nats_fake_broker.py` speaks just enough core NATS to reject the credentials of a live table, +but JetStream needs a real broker: the stream, the durable consumer, the ack accounting and the +redelivery all live on the server. A real broker cannot be made to reject the credentials of an +already connected client without a restart, which the integration harness does not allow for +the `nats1` container, so this sits between the table and the broker instead. It relays the +protocol verbatim except for the very first exchange: it answers `INFO` on behalf of the broker +and decides from the state file whether to forward the `CONNECT` line or to answer it with +`-ERR 'Authorization Violation'`, exactly the way a broker does after a rotation. + +The broker requires TLS and the proxy terminates it: the table connects to the proxy in plain +text, the proxy connects to the broker over TLS. + +Usage: nats_auth_proxy.py + +The state file holds either `accept` or `reject`. While it holds `reject`, every `CONNECT` is +answered with `-ERR 'Authorization Violation'` and every live connection is dropped. +""" + +import json +import socket +import ssl +import sys +import threading +import time + +STATE_ACCEPT = "accept" +STATE_REJECT = "reject" + +AUTHORIZATION_ERR = b"-ERR 'Authorization Violation'\r\n" + + +def log(message): + print("{:.3f} {}".format(time.time(), message), flush=True) + + +def read_line(connection): + """Reads one `\\r\\n`-terminated line, returning it together with whatever followed it.""" + buffer = b"" + while b"\r\n" not in buffer: + data = connection.recv(4096) + if not data: + raise OSError("the peer closed the connection before sending a line") + buffer += data + line, rest = buffer.split(b"\r\n", 1) + return line, rest + + +class AuthProxy: + def __init__(self, port, state_path, upstream_host, upstream_port): + self.port = port + self.state_path = state_path + self.upstream_host = upstream_host + self.upstream_port = upstream_port + self.lock = threading.Lock() + # Every accepted client connection, with the broker connection it is relayed to. + self.clients = {} + # The broker's certificate is self-signed by the test CA and the table does not verify + # it either, so neither does the proxy. + self.tls_context = ssl.create_default_context() + self.tls_context.check_hostname = False + self.tls_context.verify_mode = ssl.CERT_NONE + + def state(self): + # A missing file reads as `accept`, so the proxy is usable before a test writes any state. + try: + with open(self.state_path) as state_file: + return state_file.read().strip() or STATE_ACCEPT + except FileNotFoundError: + return STATE_ACCEPT + + def run(self): + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind(("127.0.0.1", self.port)) + listener.listen(16) + log( + "listening on port {}, state file {}, upstream {}:{}".format( + self.port, self.state_path, self.upstream_host, self.upstream_port + ) + ) + + threading.Thread(target=self.enforce_state, daemon=True).start() + + while True: + connection, _ = listener.accept() + threading.Thread(target=self.serve, args=(connection,), daemon=True).start() + + def connect_upstream(self): + # The broker sends `INFO` in plain text and only then expects the TLS handshake. + upstream = socket.create_connection((self.upstream_host, self.upstream_port)) + info_line, rest = read_line(upstream) + if rest: + raise OSError("the broker sent more than an INFO line before the TLS handshake") + upstream = self.tls_context.wrap_socket(upstream, server_hostname=self.upstream_host) + return upstream, info_line + + def rewrite_info(self, info_line): + info = json.loads(info_line[len(b"INFO ") :]) + # The client talks to the proxy in plain text, and must not learn the broker's own + # address, otherwise it would reconnect there directly and bypass the rejection. + info["tls_required"] = False + info["tls_available"] = False + info.pop("connect_urls", None) + return b"INFO " + json.dumps(info).encode() + b"\r\n" + + def serve(self, client): + upstream = None + try: + upstream, info_line = self.connect_upstream() + client.sendall(self.rewrite_info(info_line)) + + connect_line, rest = read_line(client) + if not connect_line.upper().startswith(b"CONNECT"): + raise OSError("the client sent {!r} instead of CONNECT".format(connect_line)) + + if self.state() == STATE_REJECT: + log("rejecting the credentials of a connection") + client.sendall(AUTHORIZATION_ERR) + return + + log("credentials accepted") + with self.lock: + self.clients[client] = upstream + upstream.sendall(connect_line + b"\r\n" + rest) + + relay = threading.Thread(target=self.relay, args=(upstream, client, "broker"), daemon=True) + relay.start() + self.relay(client, upstream, "client") + relay.join() + except OSError as error: + log("connection is gone: {}".format(error)) + finally: + with self.lock: + self.clients.pop(client, None) + for connection in (client, upstream): + if connection is not None: + connection.close() + + def relay(self, source, destination, source_name): + try: + while True: + data = source.recv(65536) + if not data: + break + destination.sendall(data) + except OSError as error: + log("relaying from the {} stopped: {}".format(source_name, error)) + # An `EOF` on one side ends the connection on the other side too, so that the relay in + # the opposite direction stops as well. + for connection in (source, destination): + try: + connection.shutdown(socket.SHUT_RDWR) + except OSError: + pass + + def enforce_state(self): + # A connection accepted just before the state changed has to be dropped too, otherwise + # the client would keep consuming through it and never see the rejection. + while True: + if self.state() == STATE_REJECT: + with self.lock: + connections = list(self.clients.items()) + self.clients.clear() + for client, upstream in connections: + log("dropping a live connection") + for connection in (client, upstream): + try: + connection.shutdown(socket.SHUT_RDWR) + except OSError as error: + log("cannot drop a connection: {}".format(error)) + time.sleep(0.1) + + +if __name__ == "__main__": + proxy = AuthProxy( + port=int(sys.argv[1]), + state_path=sys.argv[2], + upstream_host=sys.argv[3], + upstream_port=int(sys.argv[4]), + ) + proxy.run() diff --git a/tests/integration/test_storage_nats/nats_certs.sh b/tests/integration/test_storage_nats/nats_certs.sh new file mode 100755 index 0000000000000..fccd9d688f917 --- /dev/null +++ b/tests/integration/test_storage_nats/nats_certs.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Generates the TLS material the integration NATS broker needs. +# +# ClickHouseCluster runs this script (from the test directory) when a cluster is started +# with with_nats=True, passing NATS_CERT_DIR in the environment. It must produce: +# $NATS_CERT_DIR/ca/ca-cert.pem - CA the Python test client trusts +# $NATS_CERT_DIR/nats/server-cert.pem - server certificate (bind-mounted into the broker) +# $NATS_CERT_DIR/nats/server-key.pem - server private key +# $NATS_CERT_DIR/nats/ca-cert.pem - the CA again, for a broker that verifies clients +# $NATS_CERT_DIR/client/client-cert.pem - client certificate signed by the same CA +# $NATS_CERT_DIR/client/client-key.pem - client private key +# +# The server certificate is signed by the CA and carries SANs for the broker hostname +# (nats1) and localhost, so the Python client - which verifies the CA and the hostname +# when connecting to tls://localhost - accepts it. +set -e + +CERT_DIR="${NATS_CERT_DIR:?NATS_CERT_DIR must be set}" +CA_DIR="$CERT_DIR/ca" +NATS_DIR="$CERT_DIR/nats" +CLIENT_DIR="$CERT_DIR/client" +mkdir -p "$CA_DIR" "$NATS_DIR" "$CLIENT_DIR" + +# 1. CA private key + self-signed certificate. +# OpenSSL 3.x strict verification (used by the Python `ssl` module) requires a CA +# certificate to carry `basicConstraints=CA:TRUE` together with a `keyUsage` extension +# that includes `keyCertSign`; otherwise the client rejects the chain with +# "CA cert does not include key usage extension". +openssl req -newkey rsa:4096 -x509 -days 3650 -nodes -batch \ + -keyout "$CA_DIR/ca-key.pem" -out "$CA_DIR/ca-cert.pem" \ + -subj "/O=ClickHouse/CN=nats-test-ca" \ + -addext "basicConstraints=critical,CA:TRUE" \ + -addext "keyUsage=critical,keyCertSign,cRLSign" + +# 2. Server private key + certificate signing request. +openssl req -newkey rsa:4096 -nodes -batch \ + -keyout "$NATS_DIR/server-key.pem" -out "$NATS_DIR/server-req.pem" \ + -subj "/O=ClickHouse/CN=nats1" + +# 3. Sign the server CSR with the CA, adding SANs for the broker hostname and localhost +# plus the key usages a TLS server certificate needs for strict verification. +SAN_CNF="$NATS_DIR/server-ext.cnf" +cat > "$SAN_CNF" <<'EOF' +subjectAltName=DNS:nats1,DNS:localhost,IP:127.0.0.1 +basicConstraints=critical,CA:FALSE +keyUsage=critical,digitalSignature,keyEncipherment +extendedKeyUsage=serverAuth +EOF +openssl x509 -req -days 3650 -in "$NATS_DIR/server-req.pem" \ + -CA "$CA_DIR/ca-cert.pem" -CAkey "$CA_DIR/ca-key.pem" -CAcreateserial \ + -extfile "$SAN_CNF" -out "$NATS_DIR/server-cert.pem" + +rm -f "$NATS_DIR/server-req.pem" "$SAN_CNF" + +# 4. Client private key + certificate signed by the same CA. A broker started with +# `--tlsverify` demands it, and one started without ignores it. +openssl req -newkey rsa:4096 -nodes -batch \ + -keyout "$CLIENT_DIR/client-key.pem" -out "$CLIENT_DIR/client-req.pem" \ + -subj "/O=ClickHouse/CN=clickhouse-nats-client" +CLIENT_CNF="$CLIENT_DIR/client-ext.cnf" +cat > "$CLIENT_CNF" <<'EOF' +basicConstraints=critical,CA:FALSE +keyUsage=critical,digitalSignature,keyEncipherment +extendedKeyUsage=clientAuth +EOF +openssl x509 -req -days 3650 -in "$CLIENT_DIR/client-req.pem" \ + -CA "$CA_DIR/ca-cert.pem" -CAkey "$CA_DIR/ca-key.pem" -CAcreateserial \ + -extfile "$CLIENT_CNF" -out "$CLIENT_DIR/client-cert.pem" + +rm -f "$CLIENT_DIR/client-req.pem" "$CLIENT_CNF" + +# 5. Verifying client certificates needs the CA inside the broker's bind-mounted directory. +cp "$CA_DIR/ca-cert.pem" "$NATS_DIR/ca-cert.pem" diff --git a/tests/integration/test_storage_nats/nats_fake_broker.py b/tests/integration/test_storage_nats/nats_fake_broker.py new file mode 100644 index 0000000000000..ae0c5e1198127 --- /dev/null +++ b/tests/integration/test_storage_nats/nats_fake_broker.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +"""A minimal NATS broker, used to drive a `NATS` table through a credential rotation. + +A real broker cannot be made to reject the credentials of an already connected client without +restarting it with a different configuration, which the integration harness does not allow for +the `nats1` container. This speaks just enough of the NATS protocol for the client library the +`NATS` engine uses - `INFO` on accept, `PONG` for every `PING`, and a `MSG` for every +subscription - and decides whether to accept a connection from the state file it polls, so a +test can reject the credentials of a live table exactly the way a broker does after a rotation. + +Usage: nats_fake_broker.py [] + +The state file holds either `accept` or `reject`. While it holds `reject`, every `CONNECT` is +answered with `-ERR 'Authorization Violation'` and every live connection is dropped, which is +what a broker sends to a client whose password no longer matches. +""" + +import json +import socket +import sys +import threading +import time + +STATE_ACCEPT = "accept" +STATE_REJECT = "reject" + +AUTHORIZATION_ERR = b"-ERR 'Authorization Violation'\r\n" +PONG = b"PONG\r\n" + + +def log(message): + print("{:.3f} {}".format(time.time(), message), flush=True) + + +class FakeBroker: + def __init__(self, port, state_path, publish_interval): + self.port = port + self.state_path = state_path + self.publish_interval = publish_interval + self.lock = threading.Lock() + # Every connection whose credentials were accepted, with the subscriptions it has + # declared: {socket: {sid: subject}} + self.clients = {} + self.published = 0 + + def state(self): + # A missing file reads as `accept`, so the broker is usable before a test writes any state. + try: + with open(self.state_path) as state_file: + return state_file.read().strip() or STATE_ACCEPT + except FileNotFoundError: + return STATE_ACCEPT + + def info_line(self): + info = { + "server_id": "NATS_FAKE_BROKER", + "server_name": "nats_fake_broker", + "version": "2.10.0", + "proto": 1, + "host": "127.0.0.1", + "port": self.port, + "headers": True, + "auth_required": True, + "tls_required": False, + "tls_available": False, + "max_payload": 1048576, + "client_id": 1, + } + return b"INFO " + json.dumps(info).encode() + b"\r\n" + + def run(self): + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind(("127.0.0.1", self.port)) + listener.listen(16) + log("listening on port {}, state file {}".format(self.port, self.state_path)) + + threading.Thread(target=self.enforce_state, daemon=True).start() + threading.Thread(target=self.publish_loop, daemon=True).start() + + while True: + connection, _ = listener.accept() + threading.Thread(target=self.serve, args=(connection,), daemon=True).start() + + def serve(self, connection): + # The client reads the whole first line and discards whatever followed it in the same + # read, so an error has to be sent in answer to `CONNECT` rather than right after `INFO`. + connection.sendall(self.info_line()) + + try: + self.read_commands(connection) + except OSError as error: + log("connection is gone: {}".format(error)) + finally: + with self.lock: + self.clients.pop(connection, None) + connection.close() + + def read_commands(self, connection): + buffer = b"" + while True: + data = connection.recv(4096) + if not data: + return + buffer += data + + while b"\r\n" in buffer: + line, rest = buffer.split(b"\r\n", 1) + command = line.split() + name = command[0].upper() if command else b"" + + if name in (b"PUB", b"HPUB"): + # The payload follows the command line and is not a command itself, so it has + # to be consumed here or the loop would try to interpret it as one. + payload_size = int(command[-1]) + len(b"\r\n") + if len(rest) < payload_size: + break + rest = rest[payload_size:] + elif name == b"CONNECT": + if self.state() == STATE_REJECT: + log("rejecting the credentials of a connection") + connection.sendall(AUTHORIZATION_ERR) + return + log("credentials accepted") + with self.lock: + self.clients[connection] = {} + elif name == b"SUB": + # `SUB [queue group] ` + subject = command[1].decode() + sid = command[-1].decode() + with self.lock: + if connection in self.clients: + self.clients[connection][sid] = subject + log("subscribed sid {} to subject {}".format(sid, subject)) + elif name == b"UNSUB": + sid = command[1].decode() + with self.lock: + if connection in self.clients: + self.clients[connection].pop(sid, None) + log("unsubscribed sid {}".format(sid)) + elif name == b"PING": + connection.sendall(PONG) + + buffer = rest + + def enforce_state(self): + # A connection accepted just before the state changed has to be dropped too, otherwise + # the client would keep consuming through it and never see the rejection. + while True: + if self.state() == STATE_REJECT: + with self.lock: + connections = list(self.clients) + self.clients.clear() + for connection in connections: + log("dropping a live connection") + try: + connection.shutdown(socket.SHUT_RDWR) + except OSError as error: + log("cannot drop a connection: {}".format(error)) + time.sleep(0.1) + + def publish_loop(self): + while True: + time.sleep(self.publish_interval) + if self.state() != STATE_ACCEPT: + continue + + with self.lock: + deliveries = [ + (connection, sid, subject) + for connection, subscriptions in self.clients.items() + for sid, subject in subscriptions.items() + ] + + for connection, sid, subject in deliveries: + self.published += 1 + payload = json.dumps( + {"key": self.published, "value": self.published} + ).encode() + message = "MSG {} {} {}\r\n".format(subject, sid, len(payload)).encode() + try: + connection.sendall(message + payload + b"\r\n") + except OSError as error: + log("cannot deliver a message: {}".format(error)) + + +if __name__ == "__main__": + fake_broker = FakeBroker( + port=int(sys.argv[1]), + state_path=sys.argv[2], + publish_interval=float(sys.argv[3]) if len(sys.argv) > 3 else 0.2, + ) + fake_broker.run() diff --git a/tests/integration/test_storage_nats/test_nats_credentials_rotation.py b/tests/integration/test_storage_nats/test_nats_credentials_rotation.py new file mode 100644 index 0000000000000..25975f5783513 --- /dev/null +++ b/tests/integration/test_storage_nats/test_nats_credentials_rotation.py @@ -0,0 +1,165 @@ +import logging +import os.path as p +import time + +import pytest + +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +instance = cluster.add_instance("instance") + +BROKER_SCRIPT = "nats_fake_broker.py" +BROKER_IN_CONTAINER = "/" + BROKER_SCRIPT +BROKER_PORT = 4456 +BROKER_STATE = "/nats_fake_broker_state" +BROKER_LOG = "/var/log/clickhouse-server/nats_fake_broker.log" + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + start_fake_broker() + yield cluster + finally: + cluster.shutdown() + + +def set_broker_state(state): + instance.exec_in_container( + ["bash", "-c", "echo {} > {}".format(state, BROKER_STATE)], user="root" + ) + + +def broker_log(): + return instance.exec_in_container( + ["bash", "-c", "cat {} 2>/dev/null || true".format(BROKER_LOG)], user="root" + ) + + +def broker_log_count(needle): + return broker_log().count(needle) + + +def start_fake_broker(): + set_broker_state("accept") + + instance.copy_file_to_container( + p.join(p.dirname(__file__), BROKER_SCRIPT), BROKER_IN_CONTAINER + ) + instance.exec_in_container( + [ + "bash", + "-c", + "python3 {} {} {} > {} 2>&1".format( + BROKER_IN_CONTAINER, BROKER_PORT, BROKER_STATE, BROKER_LOG + ), + ], + detach=True, + user="root", + ) + + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + if "listening" in broker_log(): + logging.debug("The fake NATS broker is listening") + return + time.sleep(0.2) + + raise Exception("The fake NATS broker did not start") + + +def create_pipeline(subject, extra_settings=""): + instance.query("DROP DATABASE IF EXISTS test SYNC") + instance.query("CREATE DATABASE test") + instance.query( + """ + CREATE TABLE test.nats (key UInt64, value UInt64) + ENGINE = NATS + SETTINGS nats_url = '127.0.0.1:{}', + nats_subjects = '{}', + nats_format = 'JSONEachRow', + nats_username = 'clickhouse', + nats_password = 'the_original_one', + nats_reconnect_wait = 500, + nats_startup_connect_tries = 1{}; + + CREATE TABLE test.destination (key UInt64, value UInt64) + ENGINE = MergeTree ORDER BY key; + + CREATE MATERIALIZED VIEW test.consumer TO test.destination AS + SELECT * FROM test.nats; + """.format( + BROKER_PORT, subject, extra_settings + ) + ) + + +def consumed(): + return int(instance.query("SELECT count() FROM test.destination")) + + +def wait_for_consumed_above(consumed_before, time_limit_sec=120): + deadline = time.monotonic() + time_limit_sec + while time.monotonic() < deadline: + if consumed() > consumed_before: + return + time.sleep(0.5) + + raise Exception( + "The table consumed no message beyond the {} it had".format(consumed_before) + ) + + +def wait_for_consumption_to_stop(time_limit_sec=60): + """Returns the number of consumed messages once it has stopped growing. + + A streaming cycle which was in flight when the table stopped consuming still inserts what it + had, so the count is only a usable baseline once it has settled. + """ + previous = None + deadline = time.monotonic() + time_limit_sec + while time.monotonic() < deadline: + current = consumed() + if current == previous: + return current + previous = current + time.sleep(3) + + raise Exception("The table kept consuming after it was supposed to stop") + + +def test_nats_credentials_rejected_after_rotation(started_cluster): + set_broker_state("accept") + create_pipeline("rotated_subject") + + # The pipeline is live: the broker delivers a message to every subscription it holds. + wait_for_consumed_above(0) + + # The broker starts rejecting the credentials of the table, the way it does after the password + # has been rotated on its side, and drops the live connection. The client library retries once + # and gives up on the second identical authorization error, closing the connection for good. + # The table reports the closed connection under its own name, with the error the broker + # answered, so that an operator with several NATS tables can tell whose credentials to fix. + set_broker_state("reject") + instance.wait_for_log_line( + r"StorageNATS \(test\.nats\): The NATS client library closed the connection to .* " + r"Last error: Authorization Violation\.", + timeout=120, + ) + + # The connection was closed on the thread which serves every NATS table of the server, and + # before the fix the event loop adapter segfaulted there, taking the whole server down. + assert instance.query("SELECT 1") == "1\n" + + consumed_before = wait_for_consumption_to_stop() + + # The rotation is rolled back. Nothing in the client library reopens a closed connection, so + # the table has to notice and build a new one, otherwise it would stay idle until it is + # detached and attached again. + set_broker_state("accept") + wait_for_consumed_above(consumed_before) + + instance.query("DROP DATABASE test SYNC") + diff --git a/tests/integration/test_storage_nats/test_nats_jetstream_credentials_rotation.py b/tests/integration/test_storage_nats/test_nats_jetstream_credentials_rotation.py new file mode 100644 index 0000000000000..9e9c0100ff038 --- /dev/null +++ b/tests/integration/test_storage_nats/test_nats_jetstream_credentials_rotation.py @@ -0,0 +1,280 @@ +"""A JetStream `NATS` table recovers from a connection the client library closed for good. + +`test_nats_credentials_rotation.py` proves the recovery for core NATS against a fake broker. The +recovery path is shared with JetStream, whose half of the contract is different: replacing the +connection tears down the durable pull consumer's subscription with messages possibly still +unacknowledged, and the table has to resume through a fresh subscription to the same durable +consumer, with the broker redelivering whatever was not acknowledged. That needs a real broker, +so these tests put an authentication proxy (`nats_auth_proxy.py`) in front of `nats1`, which +rejects the credentials of the live table the way the broker does after a rotation. +""" + +import asyncio +import json +import logging +import os.path as p +import time + +import pytest + +from helpers.cluster import ClickHouseCluster, nats_connect_ssl +from helpers.config_cluster import nats_user, nats_pass + +from . import common as nats_helpers + +cluster = ClickHouseCluster(__file__) +instance = cluster.add_instance("instance", with_nats=True) + +PROXY_SCRIPT = "nats_auth_proxy.py" +PROXY_IN_CONTAINER = "/" + PROXY_SCRIPT +PROXY_PORT = 4457 +PROXY_STATE = "/nats_auth_proxy_state" +PROXY_LOG = "/var/log/clickhouse-server/nats_auth_proxy.log" + +# The table reports the closed connection under its own name, with the error the proxy injected. +CONNECTION_CLOSED_LOG_LINE = ( + r"StorageNATS \(test\.nats\): The NATS client library closed the connection to .* " + r"Last error: Authorization Violation\." +) + +STREAM = "rotated_stream" +SUBJECT = "rotated_subject" +DURABLE = "rotated_durable" + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + nats_helpers.wait_nats_to_start(cluster) + start_auth_proxy() + yield cluster + finally: + cluster.shutdown() + + +def set_proxy_state(state): + instance.exec_in_container( + ["bash", "-c", "echo {} > {}".format(state, PROXY_STATE)], user="root" + ) + + +def proxy_log(): + return instance.exec_in_container( + ["bash", "-c", "cat {} 2>/dev/null || true".format(PROXY_LOG)], user="root" + ) + + +def proxy_log_count(needle): + return proxy_log().count(needle) + + +def start_auth_proxy(): + set_proxy_state("accept") + + instance.copy_file_to_container( + p.join(p.dirname(__file__), PROXY_SCRIPT), PROXY_IN_CONTAINER + ) + instance.exec_in_container( + [ + "bash", + "-c", + "python3 {} {} {} {} 4444 > {} 2>&1".format( + PROXY_IN_CONTAINER, PROXY_PORT, PROXY_STATE, cluster.nats_host, PROXY_LOG + ), + ], + detach=True, + user="root", + ) + + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + if "listening" in proxy_log(): + logging.debug("The NATS authentication proxy is listening") + return + time.sleep(0.2) + + raise Exception("The NATS authentication proxy did not start") + + +def wait_for_proxy_log_count(needle, at_least, time_limit_sec=120): + deadline = time.monotonic() + time_limit_sec + while time.monotonic() < deadline: + if proxy_log_count(needle) >= at_least: + return + time.sleep(0.5) + + raise Exception( + "The proxy log holds {} occurrences of {!r}, expected {}".format( + proxy_log_count(needle), needle, at_least + ) + ) + + +def wait_for_connection_closed(): + # The tests reuse the table name, so the wait is anchored past the occurrences the previous + # tests left in the window `wait_for_log_line` looks at. + seen = nats_helpers.count_in_recent_log(instance, CONNECTION_CLOSED_LOG_LINE) + instance.wait_for_log_line( + CONNECTION_CLOSED_LOG_LINE, timeout=120, repetitions=seen + 1 + ) + + +def jetstream_setup(ack_wait_seconds): + """A stream and an explicit-ack durable pull consumer, recreated from scratch.""" + from nats.js.api import AckPolicy, ConsumerConfig + + async def run(): + nc = await nats_connect_ssl(cluster) + js = nc.jetstream() + try: + await js.delete_stream(STREAM) + except Exception: + pass # did not exist yet + await js.add_stream(name=STREAM, subjects=[SUBJECT]) + await js.add_consumer( + STREAM, + ConsumerConfig( + durable_name=DURABLE, + filter_subject=SUBJECT, + ack_policy=AckPolicy.EXPLICIT, + ack_wait=ack_wait_seconds, + ), + ) + await nc.close() + + asyncio.run(run()) + + +def jetstream_publish(start, count): + async def run(): + nc = await nats_connect_ssl(cluster) + js = nc.jetstream() + for i in range(start, start + count): + await js.publish(SUBJECT, json.dumps({"key": i, "value": i}).encode()) + await nc.close() + + asyncio.run(run()) + + +def jetstream_ack_pending(): + """Number of messages delivered to the durable consumer but not yet acknowledged.""" + + async def run(): + nc = await nats_connect_ssl(cluster) + info = await nc.jetstream().consumer_info(STREAM, DURABLE) + await nc.close() + return info.num_ack_pending + + return asyncio.run(run()) + + +def create_pipeline(extra_settings=""): + instance.query("DROP DATABASE IF EXISTS test SYNC") + instance.query("CREATE DATABASE test") + instance.query( + """ + CREATE TABLE test.nats (key UInt64, value UInt64) + ENGINE = NATS + SETTINGS nats_url = '127.0.0.1:{}', + nats_subjects = '{}', + nats_stream = '{}', + nats_consumer_name = '{}', + nats_format = 'JSONEachRow', + nats_username = '{}', + nats_password = '{}', + nats_reconnect_wait = 500, + nats_startup_connect_tries = 1{}; + + CREATE TABLE test.destination (key UInt64, value UInt64) + ENGINE = MergeTree ORDER BY key; + + CREATE MATERIALIZED VIEW test.consumer TO test.destination AS + SELECT * FROM test.nats; + """.format( + PROXY_PORT, SUBJECT, STREAM, DURABLE, nats_user, nats_pass, extra_settings + ) + ) + + +def consumed(): + return int(instance.query("SELECT count() FROM test.destination")) + + +def consumed_keys(): + return sorted( + int(key) for key in instance.query("SELECT DISTINCT key FROM test.destination").split() + ) + + +def wait_for_consumed_at_least(expected, time_limit_sec=120): + deadline = time.monotonic() + time_limit_sec + while time.monotonic() < deadline: + if consumed() >= expected: + return + time.sleep(0.5) + + raise Exception("The table consumed {} messages, expected {}".format(consumed(), expected)) + + +def wait_for_ack_pending(expected, time_limit_sec=60): + deadline = time.monotonic() + time_limit_sec + while time.monotonic() < deadline: + if jetstream_ack_pending() == expected: + return + time.sleep(0.5) + + raise Exception( + "The durable consumer has {} messages pending acknowledgement, expected {}".format( + jetstream_ack_pending(), expected + ) + ) + + +def assert_consumed_stays(expected, seconds=10): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + assert consumed() == expected, "The table consumed a message while it was not supposed to" + time.sleep(1) + + +def test_jetstream_credentials_rejected_after_rotation(started_cluster): + set_proxy_state("accept") + # A short ack-wait: a message which was delivered but not acknowledged when the connection + # went away is redelivered soon after the table subscribes again. + jetstream_setup(ack_wait_seconds=3) + create_pipeline() + + jetstream_publish(0, 10) + wait_for_consumed_at_least(10) + wait_for_ack_pending(0) + + # The broker starts rejecting the credentials of the table, the way it does after the password + # has been rotated on its side, and drops the live connection. The client library retries once + # and gives up on the second identical authorization error, closing the connection for good. + set_proxy_state("reject") + wait_for_connection_closed() + assert instance.query("SELECT 1") == "1\n" + + # The stream keeps what is published meanwhile: it is the backlog the table has to drain once + # it is back, and nothing reaches the table while its connection is closed. + jetstream_publish(10, 10) + assert_consumed_stays(10) + + # The rotation is rolled back. The table replaces the closed connection and its consumer, and + # a fresh pull subscription to the same durable consumer drains the backlog: every key exactly + # once, with nothing left pending acknowledgement. + set_proxy_state("accept") + wait_for_consumed_at_least(20) + wait_for_ack_pending(0) + assert consumed_keys() == list(range(20)) + + # Consumption has resumed for good, not just for the backlog. + jetstream_publish(20, 10) + wait_for_consumed_at_least(30) + wait_for_ack_pending(0) + assert consumed_keys() == list(range(30)) + assert consumed() == 30, "A message was consumed twice" + + instance.query("DROP DATABASE test SYNC") + diff --git a/tests/integration/test_storage_postgresql/configs/users.xml b/tests/integration/test_storage_postgresql/configs/users.xml index 4b6ba057ecb16..2f9966e1d71a2 100644 --- a/tests/integration/test_storage_postgresql/configs/users.xml +++ b/tests/integration/test_storage_postgresql/configs/users.xml @@ -4,6 +4,7 @@ default 1 + 1 diff --git a/tests/integration/test_storage_redis/fake_redis.py b/tests/integration/test_storage_redis/fake_redis.py new file mode 100644 index 0000000000000..2407443329828 --- /dev/null +++ b/tests/integration/test_storage_redis/fake_redis.py @@ -0,0 +1,128 @@ +""" +A minimal RESP server that answers MGET with a reply of the test's choosing. + +`StorageRedis` pairs the elements of an MGET reply with the requested keys by position, so both +the element count and the position of a nil decide what the engine does. Usage: + + fake_redis.py [ ] + +Every MGET is answered with `max(0, len(keys) + delta)` elements: the requested keys echoed back +as their own values, followed by RESP nils. `` and `` are comma-separated +key names; when they are given, SCAN answers with those keys and MGET answers nil for the names +in ``, so a full scan meets a fixed valid/nil sequence rather than Redis's own key +order. Anything else is answered `+OK`. +""" + +import socket +import sys +import threading + + +def read_exactly(stream, size): + data = stream.read(size) + if len(data) != size: + raise EOFError + return data + + +def read_line(stream): + line = stream.readline() + if not line: + raise EOFError + return line.rstrip(b"\r\n") + + +def read_command(stream): + """Read one RESP array. + + Argument payloads are read by declared length, never up to a newline: the keys on the wire + are ClickHouse's serializeBinary output and may contain \\r, \\n or non-UTF-8 bytes. + """ + header = read_line(stream) + if not header.startswith(b"*"): + return None + args = [] + for _ in range(int(header[1:])): + arg_header = read_line(stream) + if not arg_header.startswith(b"$"): + raise EOFError + args.append(read_exactly(stream, int(arg_header[1:]))) + read_exactly(stream, 2) # trailing CRLF + return args + + +def serialize_string(name): + """A String shorter than 128 bytes as ClickHouse serializes it: one length byte, then bytes. + + The engine deserializes the keys SCAN reports and the values MGET returns, so both have to + arrive in that form. See serialize_binary_for_string in test.py. + """ + return bytes([len(name)]) + name.encode() + + +def mget_reply(keys, delta, nil_keys): + count = max(0, len(keys) + delta) + out = [b"*%d\r\n" % count] + for i in range(count): + if i < len(keys) and keys[i] not in nil_keys: + out.append(b"$%d\r\n%s\r\n" % (len(keys[i]), keys[i])) + else: + out.append(b"$-1\r\n") + return b"".join(out) + + +def scan_reply(scan_keys): + """Cursor 0 with every key, so the engine reads the whole keyspace in one batch.""" + out = [b"*2\r\n$1\r\n0\r\n", b"*%d\r\n" % len(scan_keys)] + for key in scan_keys: + out.append(b"$%d\r\n%s\r\n" % (len(key), key)) + return b"".join(out) + + +def handle(conn, delta, scan_keys, nil_keys): + try: + with conn.makefile("rb") as stream: + while True: + args = read_command(stream) + if not args: + break + command = args[0].upper() + if command == b"MGET": + conn.sendall(mget_reply(args[1:], delta, nil_keys)) + elif command == b"SCAN" and scan_keys: + conn.sendall(scan_reply(scan_keys)) + else: + conn.sendall(b"+OK\r\n") + except (EOFError, OSError, ValueError): + pass + finally: + conn.close() + + +def parse_names(argument): + return [serialize_string(name) for name in argument.split(",") if name] + + +def main(): + port = int(sys.argv[1]) + delta = int(sys.argv[2]) + scan_keys = parse_names(sys.argv[3]) if len(sys.argv) > 3 else [] + nil_keys = parse_names(sys.argv[4]) if len(sys.argv) > 4 else [] + + server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + server.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + server.bind(("0.0.0.0", port)) + # The engine holds a connection pool and a direct join reads on several pipeline threads, so + # connections have to be served concurrently. + server.listen(64) + print(f"listening on {port}, MGET reply length = requested + {delta}", flush=True) + + while True: + conn, _ = server.accept() + threading.Thread( + target=handle, args=(conn, delta, scan_keys, nil_keys), daemon=True + ).start() + + +if __name__ == "__main__": + main() diff --git a/tests/integration/test_storage_redis/test.py b/tests/integration/test_storage_redis/test.py index 5eef47f2b0737..ee1d0e1c10cc2 100644 --- a/tests/integration/test_storage_redis/test.py +++ b/tests/integration/test_storage_redis/test.py @@ -1,5 +1,6 @@ ## sudo -H pip install redis import json +import os import struct import sys @@ -8,7 +9,9 @@ from helpers.client import QueryRuntimeException from helpers.cluster import ClickHouseCluster -from helpers.test_tools import TSV +from helpers.test_tools import TSV, wait_condition + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) cluster = ClickHouseCluster(__file__) @@ -464,3 +467,161 @@ def check_query(query, read_type, keys_count, rows_read): plan = node.query("EXPLAIN actions=1, optimize=0 SELECT * FROM test_get_keys") assert 'ReadType: FullScan' in plan + + +# Ports of the three fake Redis endpoints, inside the ClickHouse container. +FAKE_REDIS_PORT_OK = 16379 +FAKE_REDIS_PORT_LONG = 16380 +FAKE_REDIS_PORT_SHORT = 16381 +FAKE_REDIS_PORT_SCAN = 16382 + + +def start_fake_redis(port, delta, keys=""): + node.exec_in_container( + [ + "bash", + "-c", + f"python3 /fake_redis.py {port} {delta} {keys}" + f" > /var/log/clickhouse-server/fake_redis_{port}.log 2>&1", + ], + detach=True, + user="root", + ) + wait_condition( + lambda: node.exec_in_container( + ["bash", "-c", f"exec 3<>/dev/tcp/127.0.0.1/{port} && echo OK"], + nothrow=True, + ), + lambda r: "OK" in r, + max_attempts=40, + delay=0.5, + ) + + +def test_malformed_mget_reply(started_cluster): + """An MGET reply whose element count differs from the request must be rejected. + + The result loop of StorageRedis was bounded by the reply length while the null map it + indexes is sized from the request, so an over-long reply read and wrote past the end of the + null map. A short reply produced fewer rows than keys, which breaks the row-per-key contract + that IKeyValueEntity::getByKeys promises to a direct join. + """ + tables = ("redis_fake_ok", "redis_fake_long", "redis_fake_short", "t_fake_left") + for table in tables: + drop_table(table) + + node.copy_file_to_container( + os.path.join(SCRIPT_DIR, "fake_redis.py"), "/fake_redis.py" + ) + start_fake_redis(FAKE_REDIS_PORT_OK, 0) + start_fake_redis(FAKE_REDIS_PORT_LONG, 1) + start_fake_redis(FAKE_REDIS_PORT_SHORT, -1) + + node.query( + f""" + CREATE TABLE redis_fake_ok (key String, value String) + Engine=Redis('127.0.0.1:{FAKE_REDIS_PORT_OK}') PRIMARY KEY (key); + + CREATE TABLE redis_fake_long (key String, value String) + Engine=Redis('127.0.0.1:{FAKE_REDIS_PORT_LONG}') PRIMARY KEY (key); + + CREATE TABLE redis_fake_short (key String, value String) + Engine=Redis('127.0.0.1:{FAKE_REDIS_PORT_SHORT}') PRIMARY KEY (key); + + CREATE TABLE t_fake_left (k String) ENGINE = TinyLog; + INSERT INTO t_fake_left VALUES ('a'), ('b'); + """ + ) + + def direct_join(table): + return node.query( + f"SELECT key, value FROM (SELECT k AS key FROM t_fake_left) AS t " + f"INNER JOIN {table} USING (key) ORDER BY key " + f"SETTINGS join_algorithm = 'direct' FORMAT TSV" + ) + + # An endpoint that answers with one element per key still works, so a mock that never + # listens or frames RESP wrongly reddens here instead of green-washing the arms below. + assert TSV.toMat(direct_join("redis_fake_ok")) == [["a", "a"], ["b", "b"]] + + with pytest.raises(QueryRuntimeException) as long_join: + direct_join("redis_fake_long") + assert "INTERNAL_REDIS_ERROR" in str(long_join.value) + assert "for MGET of" in str(long_join.value) + + with pytest.raises(QueryRuntimeException) as short_join: + direct_join("redis_fake_short") + assert "INTERNAL_REDIS_ERROR" in str(short_join.value) + + # The other caller of the reply reads it with no null map at all. + with pytest.raises(QueryRuntimeException) as long_in: + node.query("SELECT * FROM redis_fake_long WHERE key IN ('a', 'b')") + assert "INTERNAL_REDIS_ERROR" in str(long_in.value) + + # A zero-element reply is a null array in Poco, not an empty one, so it is the isNull() + # term of the guard that rejects it. One key against the delta = -1 endpoint produces it. + with pytest.raises(QueryRuntimeException) as zero_in: + node.query("SELECT * FROM redis_fake_short WHERE key IN ('a')") + assert "INTERNAL_REDIS_ERROR" in str(zero_in.value) + assert "returned 0 values" in str(zero_in.value) + + for table in tables: + drop_table(table) + + +def test_full_scan_skips_missing_values(started_cluster): + """A full scan must skip the keys MGET answers with nil, not stop at the first one. + + MGET answers by position, and a key SCAN listed can hold a non-string type or expire + before the MGET runs, so a nil marks one absent value and not the end of the batch. + """ + address = get_address_for_ch() + table = "test_full_scan_missing" + fake_table = "redis_fake_scan" + + client = get_redis_connection(db_id=4) + client.flushdb() + drop_table(table) + drop_table(fake_table) + + # Redis alone decides in what order SCAN reports keys, so the mock pins the one thing this + # test is about: a nil arriving before a key that still has a value. + node.copy_file_to_container( + os.path.join(SCRIPT_DIR, "fake_redis.py"), "/fake_redis.py" + ) + start_fake_redis(FAKE_REDIS_PORT_SCAN, 0, "a,b,c b") + + node.query( + f""" + CREATE TABLE {fake_table} (key String, value String) + Engine=Redis('127.0.0.1:{FAKE_REDIS_PORT_SCAN}') PRIMARY KEY (key); + """ + ) + rows = node.query(f"SELECT key, value FROM {fake_table} ORDER BY key FORMAT TSV") + assert TSV.toMat(rows) == [["a", "a"], ["c", "c"]] + + # The same thing on a real Redis, which answers nil for any key that holds another type. + node.query( + f""" + CREATE TABLE {table}(k String, v String) + Engine=Redis('{address}', 4, 'clickhouse') PRIMARY KEY (k); + + INSERT INTO {table} SELECT toString(number), toString(number) FROM numbers(16); + """ + ) + + # Control: every row is readable before the keys below exist, so a fixture that writes + # nothing reddens here instead of leaving the assertion after it vacuous. + assert int(node.query(f"SELECT uniqExact(k) FROM {table}")) == 16 + + for i in range(16): + client.rpush(f"list_{i}", "x") + + # SCAN can report a key twice while Redis rehashes and no read path dedupes the rows, so the + # oracle here is the key set: a lost key is the defect, a repeated one is not. + keys = node.query(f"SELECT DISTINCT k FROM {table} ORDER BY toUInt32(k) FORMAT TSV") + assert TSV.toMat(keys) == [[str(i)] for i in range(16)] + + client.flushdb() + drop_table(table) + drop_table(fake_table) diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index 021e73d39235e..eb975f0522518 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -58,7 +58,10 @@ def started_cluster(): cluster = ClickHouseCluster(__file__) cluster.add_instance( "restricted_dummy", - main_configs=["configs/config_for_test_remote_host_filter.xml"], + main_configs=[ + "configs/config_for_test_remote_host_filter.xml", + "configs/remote_servers.xml", + ], with_minio=True, ) cluster.add_instance( @@ -690,21 +693,21 @@ def test_multipart(started_cluster, maybe_auth, positive): def test_remote_host_filter(started_cluster): instance = started_cluster.instances["restricted_dummy"] format = "column1 UInt32, column2 UInt32, column3 UInt32" - - query = "select *, column1*column2*column3 from s3('http://{}:{}/{}/test.csv', 'CSV', '{}')".format( - "invalid_host", MINIO_INTERNAL_PORT, started_cluster.minio_bucket, format - ) - assert "not allowed in configuration file" in instance.query_and_get_error(query) - other_values = "(1, 1, 1), (1, 1, 1), (11, 11, 11)" - query = "insert into table function s3('http://{}:{}/{}/test.csv', 'CSV', '{}') values {}".format( - "invalid_host", - MINIO_INTERNAL_PORT, - started_cluster.minio_bucket, - format, - other_values, - ) - assert "not allowed in configuration file" in instance.query_and_get_error(query) + blocked_url = f"http://invalid_host:{MINIO_INTERNAL_PORT}/{started_cluster.minio_bucket}" + queries = ( + f"DESCRIBE TABLE s3('{blocked_url}/test.csv', 'CSV')", + f"SELECT count() FROM s3Cluster('cluster', '{blocked_url}/test.csv', 'CSV')", + f"DESCRIBE TABLE icebergS3('{blocked_url}/')", + f"SELECT count() FROM icebergS3('{blocked_url}/')", + f"SELECT count() FROM icebergS3Cluster('cluster', '{blocked_url}/')", + f"CREATE TABLE remote_host_filter_iceberg (x UInt32) ENGINE = IcebergS3('{blocked_url}/')", + f"SELECT *, column1 * column2 * column3 FROM s3('{blocked_url}/test.csv', 'CSV', '{format}')", + f"INSERT INTO TABLE FUNCTION s3('{blocked_url}/test.csv', 'CSV', '{format}') VALUES {other_values}", + ) + + for query in queries: + assert "UNACCEPTABLE_URL" in instance.query_and_get_error(query), query def test_wrong_s3_syntax(started_cluster): diff --git a/tests/integration/test_storage_url/configs/users.xml b/tests/integration/test_storage_url/configs/users.xml index 4b6ba057ecb16..2f9966e1d71a2 100644 --- a/tests/integration/test_storage_url/configs/users.xml +++ b/tests/integration/test_storage_url/configs/users.xml @@ -4,6 +4,7 @@ default 1 + 1 diff --git a/tests/integration/test_table_function_mongodb/configs/users.xml b/tests/integration/test_table_function_mongodb/configs/users.xml index 4b6ba057ecb16..2f9966e1d71a2 100644 --- a/tests/integration/test_table_function_mongodb/configs/users.xml +++ b/tests/integration/test_table_function_mongodb/configs/users.xml @@ -4,6 +4,7 @@ default 1 + 1 diff --git a/tests/integration/test_totp_auth/config/.gitignore b/tests/integration/test_totp_auth/config/.gitignore deleted file mode 100644 index 3248ca3f0df3b..0000000000000 --- a/tests/integration/test_totp_auth/config/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# Generated in runtime by test itself -users.xml diff --git a/tests/integration/test_totp_auth/config/users.xml b/tests/integration/test_totp_auth/config/users.xml new file mode 100644 index 0000000000000..33441ee8cea8b --- /dev/null +++ b/tests/integration/test_totp_auth/config/users.xml @@ -0,0 +1,86 @@ + + + + + + + + + aa+bb + + TWLIAMWVKGJ7BYIZ4YNBECJJ7LOSCM5R + 60 + 9 + SHA256 + + + 1 + + ::/0 + + default + default + + + + aa+bb + + AL4V2=== + 60 + 9 + SHA256 + + + + aa+bb + + 5WDWWCWTIMA24564NRP2IMJN6L4QIJUYNREA5SIMX6AAJFMRAMVNCOBP277PBIJEVW637SHOZFRBAPATQ3ZIC2AVHC45UQSCZ5XLOBA= + 60 + 9 + SHA256 + + + + aa+bb + + XODSLCLKMZD44NXE + 60 + 9 + SHA256 + + + + + + + BSKB6BCNUK2Q==== + + + + + + RZ6YAUM5Q35H45XM7QLXBGXLEDAD27ELDU3WDSPRO2IZBYNQAST76=== + + + + + + + PS3T7LI= + + + + + + NCUPAGTUZL7KOUF5IET3L3OOAQ====== + + + + + pw + + 6P3SDQCE + + + + diff --git a/tests/integration/test_totp_auth/test_totp.py b/tests/integration/test_totp_auth/test_totp.py index 44abfc46b3131..73f52725f13c5 100644 --- a/tests/integration/test_totp_auth/test_totp.py +++ b/tests/integration/test_totp_auth/test_totp.py @@ -2,11 +2,10 @@ import hashlib import hmac import os -import random import re import struct import time -from fnmatch import fnmatch +import xml.etree.ElementTree as ET import pytest @@ -14,30 +13,46 @@ from helpers.uclient import client, prompt SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -TOTP_SECRET = base64.b32encode(random.randbytes(random.randint(3, 64))).decode() +USERS_CONFIG = os.path.join(SCRIPT_DIR, "config/users.xml") + +# The server accepts each code at most once and rejects codes for time steps at or before +# the last accepted one (RFC 6238, Section 5.2). Therefore every test uses dedicated users, +# with at most two logins per user: first with the code for the current time step, then with +# the code for the next time step, which is within the server time tolerance. +INTERACTIVE_USERS = [f"totuser_interactive_{i}" for i in range(3)] +NO_PASSWORD_USERS = [f"totuser_no_password_{i}" for i in range(2)] +EMPTY_PASSWORD_USERS = [f"totuser_empty_password_{i}" for i in range(2)] cluster = ClickHouseCluster(__file__) node = cluster.add_instance( "node", user_configs=["config/users.xml"], + stay_alive=True, ) @pytest.fixture(scope="module") def started_cluster(): try: - create_config(TOTP_SECRET) cluster.start() yield cluster finally: cluster.shutdown() +@pytest.fixture(autouse=True) +def fresh_server(started_cluster): + # The server keeps the used codes in memory to enforce their single use, so a repeated + # run of a test (e.g. with --count) would see its codes already consumed. + # Restart the server to reset that state. + node.restart_clickhouse() + + def get_one_time_password( secret, interval=30, digits=6, sha_version=hashlib.sha1, timepoint=None ): key = base64.b32decode(secret, casefold=True) - time_step = int(timepoint or time.time() / interval) + time_step = int((timepoint or time.time()) / interval) msg = struct.pack(">Q", time_step) hmac_hash = hmac.new(key, msg, sha_version).digest() offset = hmac_hash[-1] & 0x0F @@ -46,74 +61,32 @@ def get_one_time_password( return f"{otp:0{digits}d}" -def create_config(totp_secret): - config = f""" - - - - - - - - aa+bb - - {totp_secret} - 60 - 9 - SHA256 - - - 1 - - ::/0 - - default - default - - - - - - GEZDGNBVGY3TQOJQGEZDGNBVGY3TQOJQ - - - - - - - GEZDGNBVGY3TQOJQGEZDGNBVGY3TQOJQ - - - - -""".lstrip() - - with open(os.path.join(SCRIPT_DIR, "config/users.xml"), "w") as f: - f.write(config) - - -def get_totp_for_config(**kwargs): +def get_otp(user, offset_steps=0): + """The code of the user for the current time step shifted by `offset_steps`, + computed with the TOTP parameters of the user from the users config.""" + params = ET.parse(USERS_CONFIG).find(f"./users/{user}/time_based_one_time_password") + period = int(params.findtext("period", default="30")) return get_one_time_password( - secret=TOTP_SECRET, - interval=60, - digits=9, - sha_version=hashlib.sha256, - **kwargs, + secret=params.findtext("secret"), + interval=period, + digits=int(params.findtext("digits", default="6")), + sha_version=getattr( + hashlib, params.findtext("algorithm", default="SHA1").lower() + ), + timepoint=time.time() + offset_steps * period, + ) + + +def client_command(user): + return ( + f"{cluster.get_client_cmd()} --highlight=0 --host {node.ip_address} -u {user}" ) def test_one_time_password(started_cluster): query_text = "SELECT currentUser() || toString(42)" - totuser_secret = { - "secret": TOTP_SECRET, - "interval": 10, - "digits": 9, - "sha_version": hashlib.sha256, - } - - old_password = get_totp_for_config( - timepoint=time.time() - 3 * totuser_secret["interval"] - ) + + old_password = get_otp("totuser", offset_steps=-3) assert "AUTHENTICATION_FAILED" in node.query_and_get_error( query_text, user="totuser", password=f"aa+bb+{old_password}" ) @@ -123,7 +96,7 @@ def test_one_time_password(started_cluster): ) assert "totuser42\n" == node.query( - query_text, user="totuser", password=f"aa+bb+{get_totp_for_config()}" + query_text, user="totuser", password=f"aa+bb+{get_otp('totuser')}" ) resp = node.query( @@ -138,79 +111,79 @@ def test_one_time_password(started_cluster): FROM system.users WHERE name = 'totuser' """, user="totuser", - password=f"aa+bb+{get_totp_for_config()}", + password=f"aa+bb+{get_otp('totuser', offset_steps=1)}", ) assert "totuser\tplaintext_password\tone_time_password\tSHA256\t9\t60" in resp def test_interactive_totp_authentication(started_cluster): """Test TOTP authentication in interactive client mode.""" - client_command = f"{started_cluster.get_client_cmd()} --highlight=0 --host {node.ip_address} -u totuser" + user0, user1, user2 = INTERACTIVE_USERS # Password and TOTP provided in command line arguments with client( - command=f"{client_command} --password aa+bb+{get_totp_for_config()}" + command=f"{client_command(user0)} --password aa+bb+{get_otp(user0)}" ) as c: c.expect(prompt) c.send("SELECT currentUser() || '42' FORMAT TSVRaw;") - c.expect("totuser42") + c.expect(f"{user0}42") c.expect(prompt) with client( - command=f"{client_command} --password aa+bb --one-time-password {get_totp_for_config()}" + command=f"{client_command(user0)} --password aa+bb --one-time-password {get_otp(user0, offset_steps=1)}" ) as c: c.expect(prompt) c.send("SELECT currentUser() || '42' FORMAT TSVRaw;") - c.expect("totuser42") + c.expect(f"{user0}42") c.expect(prompt) # No password provided in command line arguments - with client(command=f"{client_command}") as c: + with client(command=f"{client_command(user1)}") as c: # Enter password + TOTP when prompted c.expect("Password.*:") - c.send(f"aa+bb+{get_totp_for_config()}", eol="\r") + c.send(f"aa+bb+{get_otp(user1)}", eol="\r") c.expect(prompt) c.send("SELECT currentUser() || '42' FORMAT TSVRaw;") - c.expect("totuser42") + c.expect(f"{user1}42") c.expect(prompt) - with client(command=f"{client_command}") as c: + with client(command=f"{client_command(user1)}") as c: # Enter password when prompted first c.expect("Password.*:") c.send(f"aa+bb", eol="\r") # Then enter TOTP when prompted c.expect("TOTP.*:") - c.send(get_totp_for_config(), eol="\r") + c.send(get_otp(user1, offset_steps=1), eol="\r") c.expect(prompt) c.send("SELECT currentUser() || '42' FORMAT TSVRaw;") - c.expect("totuser42") + c.expect(f"{user1}42") c.expect(prompt) # Password provided in command line arguments, then only TOTP prompted - with client(command=f"{client_command} --password aa+bb") as c: + with client(command=f"{client_command(user2)} --password aa+bb") as c: c.expect("TOTP.*:") - c.send(get_totp_for_config(), eol="\r") + c.send(get_otp(user2), eol="\r") c.expect(prompt) c.send("SELECT currentUser() || '42' FORMAT TSVRaw;") - c.expect("totuser42") + c.expect(f"{user2}42") c.expect(prompt) with client( - command=f"{client_command} --one-time-password {get_totp_for_config()}" + command=f"{client_command(user2)} --one-time-password {get_otp(user2, offset_steps=1)}" ) as c: # Enter only password, TOTP is provided in command line arguments c.expect("Password.*:") c.send(f"aa+bb", eol="\r") c.expect(prompt) c.send("SELECT currentUser() || '42' FORMAT TSVRaw;") - c.expect("totuser42") + c.expect(f"{user2}42") c.expect(prompt) # Errors: expected_error = re.compile(r"Authentication failed|password is incorrect") - with client(command=f"{client_command}") as c: + with client(command=f"{client_command(user0)}") as c: c.expect("Password.*:") c.send(f"aa+bb", eol="\r") @@ -219,73 +192,105 @@ def test_interactive_totp_authentication(started_cluster): c.send(f"000000", eol="\r") c.expect(expected_error) - with client(command=f"{client_command} --password aa+bb+000000") as c: + with client(command=f"{client_command(user0)} --password aa+bb+000000") as c: c.expect(expected_error) - with client(command=f"{client_command} --password wrongpwd") as c: + with client(command=f"{client_command(user0)} --password wrongpwd") as c: c.expect(expected_error) with client( - command=f"{client_command} --password wrongpwd+{get_totp_for_config()}" + command=f"{client_command(user0)} --password wrongpwd+{get_otp(user0)}" ) as c: c.expect(expected_error) def test_one_time_only_no_password(started_cluster): query_text = "SELECT currentUser() || toString(42)" + user0, user1 = NO_PASSWORD_USERS assert "AUTHENTICATION_FAILED" in node.query_and_get_error( - query_text, user="totuser_no_password", password="000000" + query_text, user=user0, password="000000" ) - secret = "GEZDGNBVGY3TQOJQGEZDGNBVGY3TQOJQ" - get_otp = lambda: get_one_time_password(secret=secret) - assert "totuser_no_password42\n" == node.query( - query_text, user="totuser_no_password", password=get_otp() - ) + assert f"{user0}42\n" == node.query(query_text, user=user0, password=get_otp(user0)) - client_command = f"{started_cluster.get_client_cmd()} --highlight=0 --host {node.ip_address} -u totuser_no_password" - with client(command=f"{client_command}") as c: + with client(command=f"{client_command(user0)}") as c: c.expect("TOTP.*:") - c.send(get_otp(), eol="\r") + c.send(get_otp(user0, offset_steps=1), eol="\r") c.expect(prompt) c.send("SELECT currentUser() || '42' FORMAT TSVRaw;") - c.expect("totuser_no_password42") + c.expect(f"{user0}42") c.expect(prompt) # When main password is empty TOTP works in both places: - with client(command=f"{client_command} --password {get_otp()}") as c: + with client(command=f"{client_command(user1)} --password {get_otp(user1)}") as c: c.send("SELECT currentUser() || '42' FORMAT TSVRaw;") - c.expect("totuser_no_password42") + c.expect(f"{user1}42") c.expect(prompt) - with client(command=f"{client_command} --one-time-password {get_otp()}") as c: + with client( + command=f"{client_command(user1)} --one-time-password {get_otp(user1, offset_steps=1)}" + ) as c: c.send("SELECT currentUser() || '42' FORMAT TSVRaw;") - c.expect("totuser_no_password42") + c.expect(f"{user1}42") c.expect(prompt) def test_empty_password_with_otp_cli_option(started_cluster): """Test that --one-time-password works for a user with empty plaintext password and TOTP.""" - secret = "GEZDGNBVGY3TQOJQGEZDGNBVGY3TQOJQ" - get_otp = lambda: get_one_time_password(secret=secret) - query_text = "SELECT currentUser() || toString(42)" + user0, user1 = EMPTY_PASSWORD_USERS - assert "totuser_empty_password42\n" == node.query( - query_text, user="totuser_empty_password", password=f"+{get_otp()}" + assert f"{user0}42\n" == node.query( + query_text, user=user0, password=f"+{get_otp(user0)}" ) - client_command = f"{started_cluster.get_client_cmd()} --highlight=0 --host {node.ip_address} -u totuser_empty_password" - - with client(command=f"{client_command} --one-time-password {get_otp()}") as c: + with client( + command=f"{client_command(user0)} --one-time-password {get_otp(user0, offset_steps=1)}" + ) as c: c.send("SELECT currentUser() || '42' FORMAT TSVRaw;") - c.expect("totuser_empty_password42") + c.expect(f"{user0}42") c.expect(prompt) with client( - command=f'{client_command} --password "" --one-time-password {get_otp()}' + command=f'{client_command(user1)} --password "" --one-time-password {get_otp(user1)}' ) as c: c.send("SELECT currentUser() || '42' FORMAT TSVRaw;") - c.expect("totuser_empty_password42") + c.expect(f"{user1}42") c.expect(prompt) + + +def test_code_single_use(started_cluster): + """Each code is accepted at most once (RFC 6238, Section 5.2), and codes for time steps + at or before the last accepted one are rejected.""" + query_text = "SELECT currentUser() || toString(42)" + user = "totuser_single_use" + + # A failed attempt (wrong password with a valid code) does not consume the code + code = get_otp(user) + assert "AUTHENTICATION_FAILED" in node.query_and_get_error( + query_text, user=user, password=f"wrongpwd+{code}" + ) + assert f"{user}42\n" == node.query(query_text, user=user, password=f"pw+{code}") + + # The same code is rejected on the second use, from any interface + assert "AUTHENTICATION_FAILED" in node.query_and_get_error( + query_text, user=user, password=f"pw+{code}" + ) + assert "AUTHENTICATION_FAILED" in node.http_query_and_get_error( + query_text, user=user, password=f"pw+{code}" + ) + + # A code for an earlier time step than the last accepted one is rejected even if it was never used + assert "AUTHENTICATION_FAILED" in node.query_and_get_error( + query_text, user=user, password=f"pw+{get_otp(user, offset_steps=-1)}" + ) + + # The code for the next time step is within the server time tolerance: accepted once, then rejected + next_code = get_otp(user, offset_steps=1) + assert f"{user}42\n" == node.query( + query_text, user=user, password=f"pw+{next_code}" + ) + assert "AUTHENTICATION_FAILED" in node.query_and_get_error( + query_text, user=user, password=f"pw+{next_code}" + ) diff --git a/tests/performance/array_join.xml b/tests/performance/array_join.xml index b3fc24d316652..af586967907b0 100644 --- a/tests/performance/array_join.xml +++ b/tests/performance/array_join.xml @@ -1,4 +1,6 @@ + CREATE TABLE array_join_wide_chain (A Array(String), B Array(String), C Array(String)) ENGINE = MergeTree ORDER BY tuple() + INSERT INTO array_join_wide_chain SELECT arrayMap(i -> concat('a', toString(number), '_', toString(i)), range(100)), arrayMap(i -> concat('b', toString(number), '_', toString(i)), range(100)), arrayMap(i -> concat('c', toString(number), '_', toString(i)), range(100)) FROM numbers(100000) @@ -10,4 +12,10 @@ SELECT count() FROM (SELECT [number] a, [number * 2, number] b FROM numbers(10000000)) AS t LEFT ARRAY JOIN a, b WHERE NOT ignore(a + b) SETTINGS enable_unaligned_array_join = 1 with 'clickhouse' as str select arrayJoin(range(number % 10)), materialize(str) from numbers(10000000) + + + SELECT a, b, c FROM array_join_wide_chain ARRAY JOIN A AS a ARRAY JOIN B AS b ARRAY JOIN C AS c LIMIT 31 SETTINGS max_threads = 1 FORMAT Null + SELECT a, b, c FROM array_join_wide_chain LEFT ARRAY JOIN A AS a LEFT ARRAY JOIN B AS b LEFT ARRAY JOIN C AS c LIMIT 31 SETTINGS max_threads = 1 FORMAT Null + + DROP TABLE IF EXISTS array_join_wide_chain diff --git a/tests/performance/iceberg_manifest_entries.xml b/tests/performance/iceberg_manifest_entries.xml new file mode 100644 index 0000000000000..a3eddb8dec204 --- /dev/null +++ b/tests/performance/iceberg_manifest_entries.xml @@ -0,0 +1,31 @@ + + + 1 + none + 1 + + + + + CREATE TABLE iceberg_manifest_entries (c0 Int64) + ENGINE = IcebergLocal((SELECT value FROM system.server_settings WHERE name = 'user_files_path') || '/iceberg_manifest_entries_' || toString(toUnixTimestamp64Micro(now64(6))) || '/') + + + + + INSERT INTO iceberg_manifest_entries SELECT number FROM numbers(100000) + SETTINGS allow_experimental_insert_into_iceberg = 1, iceberg_insert_max_rows_in_data_file = 1, + max_block_size = 1, min_insert_block_size_rows = 1, min_insert_block_size_bytes = 1 + + + + SELECT count() FROM iceberg_manifest_entries WHERE c0 < 0 + + DROP TABLE IF EXISTS iceberg_manifest_entries + diff --git a/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.reference b/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.reference index ff9c6824f00ad..c7200be9f3178 100644 --- a/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.reference +++ b/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.reference @@ -15,3 +15,4 @@ Finishing alters 0 1 0 +1 diff --git a/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh b/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh index d995dbf731e7f..9e2b81423db37 100755 --- a/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh +++ b/tests/queries/0_stateless/01076_parallel_alter_replicated_zookeeper.sh @@ -48,14 +48,13 @@ done INITIAL_SUM=$($CLICKHOUSE_CLIENT --query "SELECT SUM(value1) FROM concurrent_mutate_mt_1") -# Run mutation on random replica +# Run mutation on replica 1, which detach_attach_thread never takes down function correct_alter_thread() { local TIMELIMIT=$((SECONDS+TIMEOUT)) while [ $SECONDS -lt "$TIMELIMIT" ] do - REPLICA=$(($RANDOM % 5 + 1)) - $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_mutate_mt_$REPLICA UPDATE value1 = value1 + 1 WHERE 1"; + $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_mutate_mt_1 UPDATE value1 = value1 + 1 WHERE 1"; sleep 1 done } @@ -79,7 +78,9 @@ function detach_attach_thread() local TIMELIMIT=$((SECONDS+TIMEOUT)) while [ $SECONDS -lt "$TIMELIMIT" ] do - REPLICA=$(($RANDOM % 5 + 1)) + # Replicas 2.. only: correct_alter_thread needs replica 1 attached, or an ALTER that draws a + # DETACHed replica fails with UNKNOWN_TABLE and the run assigns no mutation to check. + REPLICA=$(($RANDOM % (REPLICAS - 1) + 2)) $CLICKHOUSE_CLIENT --query "DETACH TABLE concurrent_mutate_mt_$REPLICA" sleep 0.$RANDOM sleep 0.$RANDOM @@ -142,10 +143,15 @@ for i in $(seq $REPLICAS); do $CLICKHOUSE_CLIENT --query "SYSTEM SYNC REPLICA concurrent_mutate_mt_$i" $CLICKHOUSE_CLIENT --query "CHECK TABLE concurrent_mutate_mt_$i" &> /dev/null # if we will remove something the output of select will be wrong $CLICKHOUSE_CLIENT --query "SELECT SUM(toUInt64(value1)) > $INITIAL_SUM FROM concurrent_mutate_mt_$i" - $CLICKHOUSE_CLIENT --query "SELECT COUNT() FROM system.mutations WHERE table='concurrent_mutate_mt_$i' and is_done=0" # all mutations have to be done - $CLICKHOUSE_CLIENT --query "SELECT * FROM system.mutations WHERE table='concurrent_mutate_mt_$i' and is_done=0" # for verbose output + $CLICKHOUSE_CLIENT --query "SELECT COUNT() FROM system.mutations WHERE database='${CLICKHOUSE_DATABASE}' and table='concurrent_mutate_mt_$i' and is_done=0" # all mutations have to be done + $CLICKHOUSE_CLIENT --query "SELECT * FROM system.mutations WHERE database='${CLICKHOUSE_DATABASE}' and table='concurrent_mutate_mt_$i' and is_done=0" # for verbose output done +# The stress phase must have assigned at least one mutation: every check above is also satisfied by a +# run that assigned none, because the concurrent inserts alone raise the sum and a run without +# mutations has none unfinished. The SYSTEM SYNC REPLICA calls above make the entries visible here. +$CLICKHOUSE_CLIENT --query "SELECT count() > 0 FROM system.mutations WHERE database='${CLICKHOUSE_DATABASE}' AND table LIKE 'concurrent_mutate_mt_%'" + for i in $(seq $REPLICAS); do $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS concurrent_mutate_mt_$i" done diff --git a/tests/queries/0_stateless/01780_column_sparse.reference b/tests/queries/0_stateless/01780_column_sparse.reference index 3393361a19fb0..7452be01648e6 100644 --- a/tests/queries/0_stateless/01780_column_sparse.reference +++ b/tests/queries/0_stateless/01780_column_sparse.reference @@ -132,7 +132,7 @@ SELECT id % 7, sum(u) FROM t_sparse GROUP BY id % 7 ORDER BY id % 7; 4 190 5 330 6 270 -SELECT arrayFilter(x -> x % 2 = 1, arr2) FROM t_sparse WHERE arr2 != [] LIMIT 5; +SELECT arrayFilter(x -> x % 2 = 1, arr2) FROM t_sparse WHERE arr2 != [] ORDER BY id LIMIT 5; [1] [1,3] [1,3,5] diff --git a/tests/queries/0_stateless/01780_column_sparse.sql b/tests/queries/0_stateless/01780_column_sparse.sql index 8e3c4372d05aa..c5b98b1fd422f 100644 --- a/tests/queries/0_stateless/01780_column_sparse.sql +++ b/tests/queries/0_stateless/01780_column_sparse.sql @@ -27,7 +27,7 @@ SELECT * FROM t_sparse WHERE arr2 != [] ORDER BY id; SELECT sum(u) FROM t_sparse; SELECT id % 7, sum(u) FROM t_sparse GROUP BY id % 7 ORDER BY id % 7; -SELECT arrayFilter(x -> x % 2 = 1, arr2) FROM t_sparse WHERE arr2 != [] LIMIT 5; +SELECT arrayFilter(x -> x % 2 = 1, arr2) FROM t_sparse WHERE arr2 != [] ORDER BY id LIMIT 5; CREATE TABLE t_sparse_1 (id UInt64, v Int64) ENGINE = MergeTree ORDER BY tuple() diff --git a/tests/queries/0_stateless/02344_describe_cache.reference b/tests/queries/0_stateless/02344_describe_cache.reference index 091580f1ed078..e743a4a36d205 100644 --- a/tests/queries/0_stateless/02344_describe_cache.reference +++ b/tests/queries/0_stateless/02344_describe_cache.reference @@ -1,2 +1,2 @@ 1 -02344_describe_cache_test /var/lib/clickhouse/filesystem_caches/02344_describe_cache_test 102400 10000000 33554432 4194304 0 slru 0.6 5 5000 4194304 16 0 0 0 100 0 0 0 268435456 0 0 1000 0 0 0.1 10485760 0.001 1 0 0 +02344_describe_cache_test /var/lib/clickhouse/filesystem_caches/02344_describe_cache_test 102400 10000000 33554432 4194304 0 slru 0.6 5 5000 4194304 16 0 0 0 100 0 0 0 268435456 0 0 1000 0 0 0 0.1 10485760 0.001 1 0 0 diff --git a/tests/queries/0_stateless/02346_text_index_mergeTreeTextIndex_access.reference b/tests/queries/0_stateless/02346_text_index_mergeTreeTextIndex_access.reference index 14efafd0c46c2..d219eb8e6a852 100644 --- a/tests/queries/0_stateless/02346_text_index_mergeTreeTextIndex_access.reference +++ b/tests/queries/0_stateless/02346_text_index_mergeTreeTextIndex_access.reference @@ -9,5 +9,5 @@ ACCESS_DENIED OK ACCESS_DENIED ACCESS_DENIED -OK +ACCESS_DENIED ACCESS_DENIED diff --git a/tests/queries/0_stateless/02346_text_index_mergeTreeTextIndex_access.sh b/tests/queries/0_stateless/02346_text_index_mergeTreeTextIndex_access.sh index 87e86c4edc6c0..ae07f79cd8468 100755 --- a/tests/queries/0_stateless/02346_text_index_mergeTreeTextIndex_access.sh +++ b/tests/queries/0_stateless/02346_text_index_mergeTreeTextIndex_access.sh @@ -66,7 +66,7 @@ check_access "SELECT * FROM mergeTreeTextIndex(currentDatabase(), tab, idx_ab)" ## Row policy tests $CLICKHOUSE_CLIENT -q "GRANT SELECT ON $CLICKHOUSE_DATABASE.tab TO $user_name;" -# Row policy on column `a`: idx_a and idx_ab denied, idx_b allowed +# Row policy on column `a`: every index is denied, because each dictionary contains tokens of the hidden rows $CLICKHOUSE_CLIENT -q "CREATE ROW POLICY p1_03917 ON $CLICKHOUSE_DATABASE.tab FOR SELECT USING a = 'hello' TO $user_name;" check_access "SELECT * FROM mergeTreeTextIndex(currentDatabase(), tab, idx_a)" diff --git a/tests/queries/0_stateless/02454_create_table_with_custom_disk.reference b/tests/queries/0_stateless/02454_create_table_with_custom_disk.reference index 3c0030edc4c62..551d32cdc173c 100644 --- a/tests/queries/0_stateless/02454_create_table_with_custom_disk.reference +++ b/tests/queries/0_stateless/02454_create_table_with_custom_disk.reference @@ -4,7 +4,7 @@ CREATE TABLE test ) ENGINE = MergeTree ORDER BY tuple() -SETTINGS disk = disk(type = local, path = \'not a real path\') +SETTINGS disk = disk(type = local, path = \'[HIDDEN]\') 100 CREATE TABLE default.test\n(\n `a` Int32\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS disk = disk(type = local, path = \'[HIDDEN]\'), index_granularity = 8192 a Int32 diff --git a/tests/queries/0_stateless/02843_backup_use_same_s3_credentials_for_base_backup.reference b/tests/queries/0_stateless/02843_backup_use_same_s3_credentials_for_base_backup.reference index c03ff71b18359..97d141f6b80da 100644 --- a/tests/queries/0_stateless/02843_backup_use_same_s3_credentials_for_base_backup.reference +++ b/tests/queries/0_stateless/02843_backup_use_same_s3_credentials_for_base_backup.reference @@ -6,7 +6,7 @@ BACKUP_CREATED inc_2 BACKUP_CREATED inc_3_bad -The request signature we calculated does not match the signature you provided. Check your key and signing method. (S3_ERROR) +Please check your AWS credentials and permissions. (S3_ERROR) inc_4 BACKUP_CREATED restore inc_1 diff --git a/tests/queries/0_stateless/02843_backup_use_same_s3_credentials_for_base_backup.sh b/tests/queries/0_stateless/02843_backup_use_same_s3_credentials_for_base_backup.sh index 007f7462d9347..1ceb4b0208f3a 100755 --- a/tests/queries/0_stateless/02843_backup_use_same_s3_credentials_for_base_backup.sh +++ b/tests/queries/0_stateless/02843_backup_use_same_s3_credentials_for_base_backup.sh @@ -35,7 +35,7 @@ echo "inc_2" $CLICKHOUSE_CLIENT "${client_opts[@]}" -q "BACKUP TABLE data TO S3($(s3_location inc_2)) SETTINGS base_backup=S3($(s3_location inc_1))" | cut -f2 echo "inc_3_bad" -$CLICKHOUSE_CLIENT "${client_opts[@]}" --format Null -q "BACKUP TABLE data TO S3($(s3_location inc_3_bad)) SETTINGS base_backup=S3($(s3_location_with_invalid_password inc_1))" |& grep -m1 -o 'The request signature we calculated does not match the signature you provided. Check your key and signing method. (S3_ERROR)' +$CLICKHOUSE_CLIENT "${client_opts[@]}" --format Null -q "BACKUP TABLE data TO S3($(s3_location inc_3_bad)) SETTINGS base_backup=S3($(s3_location_with_invalid_password inc_1))" |& grep -m1 -o 'Please check your AWS credentials and permissions. (S3_ERROR)' echo "inc_4" $CLICKHOUSE_CLIENT "${client_opts[@]}" -q "BACKUP TABLE data TO S3($(s3_location inc_4)) SETTINGS base_backup=S3($(s3_location_with_invalid_password inc_1)), use_same_s3_credentials_for_base_backup=1" | cut -f2 diff --git a/tests/queries/0_stateless/02864_statistics_ddl.sql b/tests/queries/0_stateless/02864_statistics_ddl.sql index d0a1c38c4128e..002256e8b85f0 100644 --- a/tests/queries/0_stateless/02864_statistics_ddl.sql +++ b/tests/queries/0_stateless/02864_statistics_ddl.sql @@ -16,6 +16,9 @@ SET allow_statistics = 1; -- Error case: Unknown statistics types are rejected CREATE TABLE tab (col Float64 STATISTICS(no_statistics_type)) Engine = MergeTree() ORDER BY tuple(); -- { serverError INCORRECT_QUERY } +-- Error case: A statistics type written as an expression is rejected as an unknown type +CREATE TABLE tab (col Float64 STATISTICS(trim(BOTH '' FROM 'x'))) Engine = MergeTree() ORDER BY tuple(); -- { serverError INCORRECT_QUERY } + -- Error case: The same statistics type can't exist more than once on a column CREATE TABLE tab (col Float64 STATISTICS(tdigest, tdigest)) Engine = MergeTree() ORDER BY tuple(); -- { serverError INCORRECT_QUERY } diff --git a/tests/queries/0_stateless/02883_named_collections_override.reference b/tests/queries/0_stateless/02883_named_collections_override.reference index 78b3eeda98739..84bbb8c1c4007 100644 --- a/tests/queries/0_stateless/02883_named_collections_override.reference +++ b/tests/queries/0_stateless/02883_named_collections_override.reference @@ -1,32 +1,22 @@ -allow_named_collection_override_by_default=1 u1 +u1 1\n 1\n 1 -allow_named_collection_override_by_default=1 u2 +u2 1\n 1\n 2\n 1 -allow_named_collection_override_by_default=0 u1 -1\n -1 -allow_named_collection_override_by_default=0 u2 -1\n Test ALTER 2\n 1\n Test XML collections -allow_named_collection_override_by_default=1 url_override1 +url_override1 1\n 1\n 1 -allow_named_collection_override_by_default=1 url_override2 +url_override2 1\n 1\n 2\n 1 -allow_named_collection_override_by_default=0 url_override1 -1\n -1 -allow_named_collection_override_by_default=0 url_override2 -1\n diff --git a/tests/queries/0_stateless/02883_named_collections_override.sh b/tests/queries/0_stateless/02883_named_collections_override.sh index 915ce28022622..371574f1f2427 100755 --- a/tests/queries/0_stateless/02883_named_collections_override.sh +++ b/tests/queries/0_stateless/02883_named_collections_override.sh @@ -15,78 +15,53 @@ DROP NAMED COLLECTION IF EXISTS $u2; DROP NAMED COLLECTION IF EXISTS $u3; CREATE NAMED COLLECTION IF NOT EXISTS $u1 AS - url = 'http://127.0.0.1:8123?query=select+1' NOT OVERRIDABLE, + url = 'http://127.0.0.1:${CLICKHOUSE_PORT_HTTP}?query=select+1' NOT OVERRIDABLE, format = 'RawBLOB' OVERRIDABLE; CREATE NAMED COLLECTION IF NOT EXISTS $u2 AS - url = 'http://127.0.0.1:8123?query=select+1', + url = 'http://127.0.0.1:${CLICKHOUSE_PORT_HTTP}?query=select+1', format = 'RawBLOB'; -SET allow_named_collection_override_by_default=1; -SELECT 'allow_named_collection_override_by_default=1 u1'; +SELECT 'u1'; SELECT * FROM url($u1); SELECT * FROM url($u1, headers('Accept'='text/csv; charset=utf-8')); -SELECT * FROM url($u1, url='http://127.0.0.1:8123?query=select+2'); -- { serverError BAD_ARGUMENTS } +SELECT * FROM url($u1, url='http://127.0.0.1:${CLICKHOUSE_PORT_HTTP}?query=select+2'); -- { serverError BAD_ARGUMENTS } SELECT * FROM url($u1, format='CSV'); -SELECT 'allow_named_collection_override_by_default=1 u2'; +SELECT 'u2'; SELECT * FROM url($u2); SELECT * FROM url($u2, headers('Accept'='text/csv; charset=utf-8')); -SELECT * FROM url($u2, url='http://127.0.0.1:8123?query=select+2'); +SELECT * FROM url($u2, url='http://127.0.0.1:${CLICKHOUSE_PORT_HTTP}?query=select+2'); SELECT * FROM url($u2, format='CSV'); -SET allow_named_collection_override_by_default=0; -SELECT 'allow_named_collection_override_by_default=0 u1'; -SELECT * FROM url($u1); -SELECT * FROM url($u1, headers('Accept'='text/csv; charset=utf-8')); -- { serverError BAD_ARGUMENTS } -SELECT * FROM url($u1, url='http://127.0.0.1:8123?query=select+2'); -- { serverError BAD_ARGUMENTS } -SELECT * FROM url($u1, format='CSV'); -SELECT 'allow_named_collection_override_by_default=0 u2'; -SELECT * FROM url($u2); -SELECT * FROM url($u2, headers('Accept'='text/csv; charset=utf-8')); -- { serverError BAD_ARGUMENTS } -SELECT * FROM url($u2, url='http://127.0.0.1:8123?query=select+2'); -- { serverError BAD_ARGUMENTS } -SELECT * FROM url($u2, format='CSV'); -- { serverError BAD_ARGUMENTS } - SELECT 'Test ALTER'; CREATE NAMED COLLECTION IF NOT EXISTS $u3 AS - url = 'http://127.0.0.1:8123?query=select+1' NOT OVERRIDABLE, + url = 'http://127.0.0.1:${CLICKHOUSE_PORT_HTTP}?query=select+1' NOT OVERRIDABLE, format = 'RawBLOB' OVERRIDABLE; ALTER NAMED COLLECTION $u3 SET - url = 'http://127.0.0.1:8123?query=select+2' OVERRIDABLE, + url = 'http://127.0.0.1:${CLICKHOUSE_PORT_HTTP}?query=select+2' OVERRIDABLE, format = 'RawBLOB' NOT OVERRIDABLE; SELECT * FROM url($u3); -SELECT * FROM url($u3, url='http://127.0.0.1:8123?query=select+1'); +SELECT * FROM url($u3, url='http://127.0.0.1:${CLICKHOUSE_PORT_HTTP}?query=select+1'); SELECT * FROM url($u3, format='CSV'); -- { serverError BAD_ARGUMENTS } DROP NAMED COLLECTION IF EXISTS $u1; DROP NAMED COLLECTION IF EXISTS $u2; -DROP NAMED COLLECTION IF EXISTS $u2; +DROP NAMED COLLECTION IF EXISTS $u3; SELECT 'Test XML collections'; -SET allow_named_collection_override_by_default=1; -SELECT 'allow_named_collection_override_by_default=1 url_override1'; +SELECT 'url_override1'; SELECT * FROM url(url_override1); SELECT * FROM url(url_override1, headers('Accept'='text/csv; charset=utf-8')); -SELECT * FROM url(url_override1, url='http://127.0.0.1:8123?query=select+2'); -- { serverError BAD_ARGUMENTS } +SELECT * FROM url(url_override1, url='http://127.0.0.1:${CLICKHOUSE_PORT_HTTP}?query=select+2'); -- { serverError BAD_ARGUMENTS } SELECT * FROM url(url_override1, format='CSV'); -SELECT 'allow_named_collection_override_by_default=1 url_override2'; +SELECT 'url_override2'; SELECT * FROM url(url_override2); SELECT * FROM url(url_override2, headers('Accept'='text/csv; charset=utf-8')); -SELECT * FROM url(url_override2, url='http://127.0.0.1:8123?query=select+2'); +SELECT * FROM url(url_override2, url='http://127.0.0.1:${CLICKHOUSE_PORT_HTTP}?query=select+2'); SELECT * FROM url(url_override2, format='CSV'); -SET allow_named_collection_override_by_default=0; -SELECT 'allow_named_collection_override_by_default=0 url_override1'; -SELECT * FROM url(url_override1); -SELECT * FROM url(url_override1, headers('Accept'='text/csv; charset=utf-8')); -- { serverError BAD_ARGUMENTS } -SELECT * FROM url(url_override1, url='http://127.0.0.1:8123?query=select+2'); -- { serverError BAD_ARGUMENTS } -SELECT * FROM url(url_override1, format='CSV'); -SELECT 'allow_named_collection_override_by_default=0 url_override2'; -SELECT * FROM url(url_override2); -SELECT * FROM url(url_override2, headers('Accept'='text/csv; charset=utf-8')); -- { serverError BAD_ARGUMENTS } -SELECT * FROM url(url_override2, url='http://127.0.0.1:8123?query=select+2'); -- { serverError BAD_ARGUMENTS } -SELECT * FROM url(url_override2, format='CSV'); -- { serverError BAD_ARGUMENTS } " diff --git a/tests/queries/0_stateless/02916_another_move_partition_inactive_replica.reference b/tests/queries/0_stateless/02916_another_move_partition_inactive_replica.reference index 361a0d1e9bba2..cd7f071944b6f 100644 --- a/tests/queries/0_stateless/02916_another_move_partition_inactive_replica.reference +++ b/tests/queries/0_stateless/02916_another_move_partition_inactive_replica.reference @@ -1,4 +1,3 @@ -all_0_1_1 1 -all_2_3_1 1 +2 0 40 1580 diff --git a/tests/queries/0_stateless/02916_another_move_partition_inactive_replica.sql b/tests/queries/0_stateless/02916_another_move_partition_inactive_replica.sql index fa5b635781134..936bb26a4707f 100644 --- a/tests/queries/0_stateless/02916_another_move_partition_inactive_replica.sql +++ b/tests/queries/0_stateless/02916_another_move_partition_inactive_replica.sql @@ -33,7 +33,7 @@ drop table if exists shard_0.from_1; drop table if exists shard_1.from_1; OPTIMIZE TABLE shard_0.to; OPTIMIZE TABLE shard_0.to; -select name, active from system.parts where database='shard_0' and table='to' and active order by name; +select count() from system.parts where database='shard_0' and table='to' and active; -- If moved parts are not merged by OPTIMIZE or background merge restart -- can log Warning about metadata version on disk. It's normal situation diff --git a/tests/queries/0_stateless/02921_database_filesystem_path_check.sql b/tests/queries/0_stateless/02921_database_filesystem_path_check.sql index d62b629df7b7d..f5fd2f848d235 100644 --- a/tests/queries/0_stateless/02921_database_filesystem_path_check.sql +++ b/tests/queries/0_stateless/02921_database_filesystem_path_check.sql @@ -1,2 +1,18 @@ -create database db_filesystem ENGINE=Filesystem('/etc'); -- { serverError BAD_ARGUMENTS } -create database db_filesystem ENGINE=Filesystem('../../../../../../../../etc'); -- { serverError BAD_ARGUMENTS } \ No newline at end of file +create database {CLICKHOUSE_DATABASE_1:Identifier} ENGINE=Filesystem('/etc'); -- { serverError BAD_ARGUMENTS } +create database {CLICKHOUSE_DATABASE_1:Identifier} ENGINE=Filesystem('../../../../../../../../etc'); -- { serverError BAD_ARGUMENTS } +-- A path inside user_files that does not exist. A definition the user supplies now is rejected whether the +-- statement is CREATE or ATTACH; only the replay of a definition already stored on this server may load. +attach database {CLICKHOUSE_DATABASE_1:Identifier} ENGINE=Filesystem('no_such_dir_in_user_files'); -- { serverError BAD_ARGUMENTS } +-- A wrapper that runs its children as internal queries must not turn a user-supplied definition into a +-- metadata replay. The sibling has an empty pipeline and cannot fail, so the assertion is order-independent. +attach database {CLICKHOUSE_DATABASE_1:Identifier} ENGINE=Filesystem('no_such_dir_in_user_files') +parallel with drop table if exists no_such_table_02921; -- { serverError BAD_ARGUMENTS } +-- Same wrapper, but now the name already has a stored definition, which a detach leaves behind. What the +-- server stored is that definition, not the one below, so the path still has to exist. +create database {CLICKHOUSE_DATABASE_1:Identifier}; +detach database {CLICKHOUSE_DATABASE_1:Identifier}; +attach database {CLICKHOUSE_DATABASE_1:Identifier} ENGINE=Filesystem('no_such_dir_in_user_files') +parallel with drop table if exists no_such_table_02921; -- { serverError BAD_ARGUMENTS } +-- Re-attach before dropping: a detached database keeps its metadata file, which every later restart replays. +attach database {CLICKHOUSE_DATABASE_1:Identifier}; +drop database {CLICKHOUSE_DATABASE_1:Identifier}; diff --git a/tests/queries/0_stateless/02941_variant_type_alters.reference b/tests/queries/0_stateless/02941_variant_type_alters.reference index 52c834e455bf6..7e402108591a4 100644 --- a/tests/queries/0_stateless/02941_variant_type_alters.reference +++ b/tests/queries/0_stateless/02941_variant_type_alters.reference @@ -60,51 +60,51 @@ insert after alter modify column 1 20 20 20 \N 20 \N 21 21 str_21 str_21 \N \N alter modify column 2 -0 0 \N \N \N \N \N \N -1 1 \N \N \N \N \N \N -2 2 \N \N \N \N \N \N -3 3 \N \N 3 \N 3 \N -4 4 \N \N 4 \N 4 \N -5 5 \N \N 5 \N 5 \N -6 6 \N \N str_6 str_6 \N \N -7 7 \N \N str_7 str_7 \N \N -8 8 \N \N str_8 str_8 \N \N -9 9 \N \N \N \N \N \N -10 10 \N \N \N \N \N \N -11 11 \N \N \N \N \N \N -12 12 \N \N 12 \N 12 \N -13 13 \N \N str_13 str_13 \N \N -14 14 \N \N \N \N \N \N -15 15 \N \N 1970-01-16 \N \N 1970-01-16 -16 16 \N \N 1970-01-17 \N \N 1970-01-17 -17 17 \N \N 1970-01-18 \N \N 1970-01-18 -18 18 \N \N 1970-01-19 \N \N 1970-01-19 -19 19 \N \N \N \N \N \N -20 20 \N \N 20 \N 20 \N -21 21 \N \N str_21 str_21 \N \N +0 0 0 \N \N \N \N \N +1 1 1 \N \N \N \N \N +2 2 2 \N \N \N \N \N +3 3 3 \N 3 \N 3 \N +4 4 4 \N 4 \N 4 \N +5 5 5 \N 5 \N 5 \N +6 6 6 \N str_6 str_6 \N \N +7 7 7 \N str_7 str_7 \N \N +8 8 8 \N str_8 str_8 \N \N +9 9 9 \N \N \N \N \N +10 10 10 \N \N \N \N \N +11 11 11 \N \N \N \N \N +12 12 12 \N 12 \N 12 \N +13 13 13 \N str_13 str_13 \N \N +14 14 14 \N \N \N \N \N +15 15 15 \N 1970-01-16 \N \N 1970-01-16 +16 16 16 \N 1970-01-17 \N \N 1970-01-17 +17 17 17 \N 1970-01-18 \N \N 1970-01-18 +18 18 18 \N 1970-01-19 \N \N 1970-01-19 +19 19 19 \N \N \N \N \N +20 20 20 \N 20 \N 20 \N +21 21 21 \N str_21 str_21 \N \N insert after alter modify column 2 -0 0 \N \N \N \N \N \N -1 1 \N \N \N \N \N \N -2 2 \N \N \N \N \N \N -3 3 \N \N 3 \N 3 \N -4 4 \N \N 4 \N 4 \N -5 5 \N \N 5 \N 5 \N -6 6 \N \N str_6 str_6 \N \N -7 7 \N \N str_7 str_7 \N \N -8 8 \N \N str_8 str_8 \N \N -9 9 \N \N \N \N \N \N -10 10 \N \N \N \N \N \N -11 11 \N \N \N \N \N \N -12 12 \N \N 12 \N 12 \N -13 13 \N \N str_13 str_13 \N \N -14 14 \N \N \N \N \N \N -15 15 \N \N 1970-01-16 \N \N 1970-01-16 -16 16 \N \N 1970-01-17 \N \N 1970-01-17 -17 17 \N \N 1970-01-18 \N \N 1970-01-18 -18 18 \N \N 1970-01-19 \N \N 1970-01-19 -19 19 \N \N \N \N \N \N -20 20 \N \N 20 \N 20 \N -21 21 \N \N str_21 str_21 \N \N +0 0 0 \N \N \N \N \N +1 1 1 \N \N \N \N \N +2 2 2 \N \N \N \N \N +3 3 3 \N 3 \N 3 \N +4 4 4 \N 4 \N 4 \N +5 5 5 \N 5 \N 5 \N +6 6 6 \N str_6 str_6 \N \N +7 7 7 \N str_7 str_7 \N \N +8 8 8 \N str_8 str_8 \N \N +9 9 9 \N \N \N \N \N +10 10 10 \N \N \N \N \N +11 11 11 \N \N \N \N \N +12 12 12 \N 12 \N 12 \N +13 13 13 \N str_13 str_13 \N \N +14 14 14 \N \N \N \N \N +15 15 15 \N 1970-01-16 \N \N 1970-01-16 +16 16 16 \N 1970-01-17 \N \N 1970-01-17 +17 17 17 \N 1970-01-18 \N \N 1970-01-18 +18 18 18 \N 1970-01-19 \N \N 1970-01-19 +19 19 19 \N \N \N \N \N +20 20 20 \N 20 \N 20 \N +21 21 21 \N str_21 str_21 \N \N 22 str_22 \N str_22 \N \N \N \N 23 \N \N \N \N \N \N \N 24 24 24 \N \N \N \N \N diff --git a/tests/queries/0_stateless/02941_variant_type_alters.sh b/tests/queries/0_stateless/02941_variant_type_alters.sh index d93e7d82a4b93..54735b66b8f2a 100755 --- a/tests/queries/0_stateless/02941_variant_type_alters.sh +++ b/tests/queries/0_stateless/02941_variant_type_alters.sh @@ -1,6 +1,5 @@ #!/usr/bin/env bash # Tags: long, memory-engine -# memory-engine: minor data inconsistency after alter CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/02998_analyzer_secret_args_tree_node.reference b/tests/queries/0_stateless/02998_analyzer_secret_args_tree_node.reference index 67ef38093d4c9..a558fa1930594 100644 --- a/tests/queries/0_stateless/02998_analyzer_secret_args_tree_node.reference +++ b/tests/queries/0_stateless/02998_analyzer_secret_args_tree_node.reference @@ -25,12 +25,15 @@ QUERY id: 0 CONSTANT id: 13, constant_value: [HIDDEN id: 2], constant_value_type: String JOIN TREE TABLE id: 14, alias: __table1, table_name: system.one +-- The session setting alone must not reveal secrets: the `display_secrets_in_show_and_select` +-- server setting and the `displaySecretsInShowAndSelect` privilege are also required, and the +-- stateless test server has neither. SET format_display_secrets_in_show_and_select = 1; EXPLAIN QUERY TREE SELECT encrypt('aes-256-ofb', (SELECT 'qwerty'), '12345678901234567890123456789012'), encrypt('aes-256-ofb', (SELECT 'asdf'), '12345678901234567890123456789012'); QUERY id: 0 PROJECTION COLUMNS - encrypt(\'aes-256-ofb\', _subquery_1, \'12345678901234567890123456789012\') Nullable(String) - encrypt(\'aes-256-ofb\', _subquery_2, \'12345678901234567890123456789012\') Nullable(String) + encrypt(\'aes-256-ofb\', [HIDDEN id: 1], [HIDDEN id: 2]) Nullable(String) + encrypt(\'aes-256-ofb\', [HIDDEN id: 3], [HIDDEN id: 2]) Nullable(String) PROJECTION LIST id: 1, nodes: 2 CONSTANT id: 2, constant_value: \'\\nãì&\', constant_value_type: Nullable(String) @@ -39,33 +42,15 @@ QUERY id: 0 ARGUMENTS LIST id: 4, nodes: 3 CONSTANT id: 5, constant_value: \'aes-256-ofb\', constant_value_type: String - CONSTANT id: 6, constant_value: \'qwerty\', constant_value_type: Nullable(String) - EXPRESSION - QUERY id: 7, is_subquery: 1 - PROJECTION COLUMNS - \'qwerty\' String - PROJECTION - LIST id: 8, nodes: 1 - CONSTANT id: 9, constant_value: \'qwerty\', constant_value_type: String - JOIN TREE - TABLE id: 10, table_name: system.one - CONSTANT id: 11, constant_value: \'12345678901234567890123456789012\', constant_value_type: String - CONSTANT id: 12, constant_value: \'çø\', constant_value_type: Nullable(String) + CONSTANT id: 6, constant_value: [HIDDEN id: 1], constant_value_type: Nullable(String) + CONSTANT id: 7, constant_value: [HIDDEN id: 2], constant_value_type: String + CONSTANT id: 8, constant_value: \'çø\', constant_value_type: Nullable(String) EXPRESSION - FUNCTION id: 13, function_name: encrypt, function_type: ordinary, result_type: Nullable(String) + FUNCTION id: 9, function_name: encrypt, function_type: ordinary, result_type: Nullable(String) ARGUMENTS - LIST id: 14, nodes: 3 - CONSTANT id: 15, constant_value: \'aes-256-ofb\', constant_value_type: String - CONSTANT id: 16, constant_value: \'asdf\', constant_value_type: Nullable(String) - EXPRESSION - QUERY id: 17, is_subquery: 1 - PROJECTION COLUMNS - \'asdf\' String - PROJECTION - LIST id: 18, nodes: 1 - CONSTANT id: 19, constant_value: \'asdf\', constant_value_type: String - JOIN TREE - TABLE id: 20, table_name: system.one - CONSTANT id: 21, constant_value: \'12345678901234567890123456789012\', constant_value_type: String + LIST id: 10, nodes: 3 + CONSTANT id: 11, constant_value: \'aes-256-ofb\', constant_value_type: String + CONSTANT id: 12, constant_value: [HIDDEN id: 3], constant_value_type: Nullable(String) + CONSTANT id: 13, constant_value: [HIDDEN id: 2], constant_value_type: String JOIN TREE - TABLE id: 22, alias: __table1, table_name: system.one + TABLE id: 14, alias: __table1, table_name: system.one diff --git a/tests/queries/0_stateless/02998_analyzer_secret_args_tree_node.sql b/tests/queries/0_stateless/02998_analyzer_secret_args_tree_node.sql index a216f886f8ad4..332f54b6b432b 100644 --- a/tests/queries/0_stateless/02998_analyzer_secret_args_tree_node.sql +++ b/tests/queries/0_stateless/02998_analyzer_secret_args_tree_node.sql @@ -6,6 +6,9 @@ SET enable_analyzer = 1; EXPLAIN QUERY TREE SELECT encrypt('aes-256-ofb', (SELECT 'qwerty'), '12345678901234567890123456789012'), encrypt('aes-256-ofb', (SELECT 'asdf'), '12345678901234567890123456789012'); +-- The session setting alone must not reveal secrets: the `display_secrets_in_show_and_select` +-- server setting and the `displaySecretsInShowAndSelect` privilege are also required, and the +-- stateless test server has neither. SET format_display_secrets_in_show_and_select = 1; EXPLAIN QUERY TREE SELECT encrypt('aes-256-ofb', (SELECT 'qwerty'), '12345678901234567890123456789012'), encrypt('aes-256-ofb', (SELECT 'asdf'), '12345678901234567890123456789012'); diff --git a/tests/queries/0_stateless/03279_database_backup_database_disk_engine.reference b/tests/queries/0_stateless/03279_database_backup_database_disk_engine.reference index 4f8c3ae7dc13c..80083db7617a9 100644 --- a/tests/queries/0_stateless/03279_database_backup_database_disk_engine.reference +++ b/tests/queries/0_stateless/03279_database_backup_database_disk_engine.reference @@ -20,7 +20,7 @@ 8 1500 9 1500 -- -CREATE DATABASE default_inner_backup_database\nENGINE = Backup(\'default_inner\', \'Disk(\\\'backups\\\', \\\'default_inner\\\')\') +CREATE DATABASE default_inner_backup_database\nENGINE = Backup(\'default_inner\', Disk(\'backups\', \'default_inner\')) test_table_1 15000 test_table_2 15000 -- diff --git a/tests/queries/0_stateless/03595_equality_deletes_simple.reference b/tests/queries/0_stateless/03595_equality_deletes_simple.reference index 864c8ae319893..3ccc2b85e7200 100644 --- a/tests/queries/0_stateless/03595_equality_deletes_simple.reference +++ b/tests/queries/0_stateless/03595_equality_deletes_simple.reference @@ -1,3 +1,6 @@ 448755 448755 448755 881 +--- equality deletes with mismatched nullability --- +bar +world diff --git a/tests/queries/0_stateless/03595_equality_deletes_simple.sql b/tests/queries/0_stateless/03595_equality_deletes_simple.sql index b70348337f005..c934376d69293 100644 --- a/tests/queries/0_stateless/03595_equality_deletes_simple.sql +++ b/tests/queries/0_stateless/03595_equality_deletes_simple.sql @@ -3,3 +3,12 @@ SELECT sum(id) FROM icebergS3(s3_conn, filename = 'deletes_db/eq_deletes_table'); SELECT sum(id) FROM icebergS3Cluster('test_cluster_two_shards_localhost', s3_conn, filename = 'deletes_db/eq_deletes_table'); SELECT sum(id), count(name) FROM icebergS3(s3_conn, filename = 'deletes_db/eq_deletes_table'); + +-- Equality deletes on the `data` column, which is required in the table schema +-- but optional in the delete file schema (their nullability may legally differ). +SELECT '--- equality deletes with mismatched nullability ---'; +-- The table's metadata declares `location` as `/out/warehouse/default/eq_deletes_required_table`, +-- so the queried path is kept as a component-aligned suffix of it. This branch resolves a data path +-- with `getProperFilePathFromMetadataInfo`, whose fast path needs `table_location` to end with the +-- queried path; `IcebergPathResolver`, which does not, is not in 26.3. +SELECT data FROM icebergS3(s3_conn, filename = 'default/eq_deletes_required_table') ORDER BY data; diff --git a/tests/queries/0_stateless/03737_ttl_delete_vertical_merge.reference b/tests/queries/0_stateless/03737_ttl_delete_vertical_merge.reference index dec1ec938abef..a8321f1de6737 100644 --- a/tests/queries/0_stateless/03737_ttl_delete_vertical_merge.reference +++ b/tests/queries/0_stateless/03737_ttl_delete_vertical_merge.reference @@ -7,7 +7,6 @@ test1_parts 1 test1_algo Vertical test2_before 2000 test2_after 0 -test2_algo Vertical test3_before 2000 test3_after 1000 test3_check_cols 4995000 49950000 499500000 4995000000 diff --git a/tests/queries/0_stateless/03737_ttl_delete_vertical_merge.sql b/tests/queries/0_stateless/03737_ttl_delete_vertical_merge.sql index 83802b5a44df7..8431f03bddb9c 100644 --- a/tests/queries/0_stateless/03737_ttl_delete_vertical_merge.sql +++ b/tests/queries/0_stateless/03737_ttl_delete_vertical_merge.sql @@ -48,8 +48,11 @@ SELECT 'test1_check_cols', sum(c1), sum(c2), sum(c3), sum(c4) FROM t_ttl_vert_1; SELECT 'test1_parts', count() FROM system.parts WHERE database = currentDatabase() AND table = 't_ttl_vert_1' AND active; SYSTEM FLUSH LOGS part_log; +-- Exclude TTLDropMerge: a background TTLDrop merge may race with OPTIMIZE FINAL +-- and TTLDrop merges always use the Horizontal algorithm. SELECT 'test1_algo', merge_algorithm FROM system.part_log WHERE database = currentDatabase() AND table = 't_ttl_vert_1' AND event_type = 'MergeParts' + AND merge_reason != 'TTLDropMerge' ORDER BY event_time_microseconds LIMIT 1; DROP TABLE IF EXISTS t_ttl_vert_1; @@ -94,11 +97,6 @@ OPTIMIZE TABLE t_ttl_vert_2 FINAL; SELECT 'test2_after', count() FROM t_ttl_vert_2; -SYSTEM FLUSH LOGS part_log; -SELECT 'test2_algo', merge_algorithm FROM system.part_log - WHERE database = currentDatabase() AND table = 't_ttl_vert_2' AND event_type = 'MergeParts' - ORDER BY event_time_microseconds LIMIT 1; - DROP TABLE IF EXISTS t_ttl_vert_2; -- Test 3: Disabled setting falls back to horizontal merge (still produces correct results) diff --git a/tests/queries/0_stateless/03801_domain_empty_string_msan.reference b/tests/queries/0_stateless/03801_domain_empty_string_msan.reference index a02e234508177..d906074e6f176 100644 --- a/tests/queries/0_stateless/03801_domain_empty_string_msan.reference +++ b/tests/queries/0_stateless/03801_domain_empty_string_msan.reference @@ -14,3 +14,12 @@ + + + + + +0 + + + diff --git a/tests/queries/0_stateless/03801_domain_empty_string_msan.sql b/tests/queries/0_stateless/03801_domain_empty_string_msan.sql index fa8ec2c850f61..19de201184a15 100644 --- a/tests/queries/0_stateless/03801_domain_empty_string_msan.sql +++ b/tests/queries/0_stateless/03801_domain_empty_string_msan.sql @@ -19,3 +19,16 @@ SELECT domainWithoutWWW(''); SELECT domainWithoutWWW('a'); SELECT domainWithoutWWW('/'); SELECT domainWithoutWWW('//'); + +-- An authority that ends exactly at the scheme prefix leaves the host scan positioned at the end of +-- the input. `ColumnString` is not zero-terminated, so the byte one past the input is uninitialised +-- heap. The returned host is empty either way, so only an MSan build can observe the read. +SELECT domainRFC('http://'); +SELECT domainRFC('aa://'); +SELECT domainRFC('a://'); +SELECT domainWithoutWWWRFC('http://'); +SELECT topLevelDomainRFC('http://'); +SELECT portRFC('http://'); +SELECT firstSignificantSubdomainRFC('http://'); +SELECT cutToFirstSignificantSubdomainRFC('http://'); +SELECT domainRFC(materialize('http://')); diff --git a/tests/queries/0_stateless/03990_alter_rename_column_swap_different_types.reference b/tests/queries/0_stateless/03990_alter_rename_column_swap_different_types.reference new file mode 100644 index 0000000000000..c6900cad13b5d --- /dev/null +++ b/tests/queries/0_stateless/03990_alter_rename_column_swap_different_types.reference @@ -0,0 +1,39 @@ +--- compact part: swap ip <-> ip_v6 (UInt32 <-> IPv6) --- +id UInt32 +ip_v4 UInt32 +ip IPv6 +0 3628718300 ::ffff:216.73.216.220 +1 0 :: +1 0 :: +--- wide part: same swap --- +id UInt32 +ip_v4 UInt32 +ip IPv6 +0 3628718300 ::ffff:216.73.216.220 +1 0 :: +1 0 :: +--- wide part: swap where the added column was never materialized --- +id UInt32 +ip_v4 UInt32 +ip IPv6 +0 3628718300 :: +1 0 :: +--- compact part: same unmaterialized swap (must keep working) --- +id UInt32 +ip_v4 UInt32 +ip IPv6 +0 3628718300 :: +1 0 :: +--- wide part: unmaterialized swap with a part rewrite forced in the same ALTER --- +id UInt32 +ip_v4 UInt32 +ip IPv6 +payload String +0 3628718300 :: b +1 3628718300 :: b +--- DROP a + RENAME b TO a, different types (must keep working) --- +id UInt32 +a IPv6 +0 ::ffff:1.2.3.4 +1 ::ffff:1.2.3.4 +2 ::ffff:1.2.3.4 diff --git a/tests/queries/0_stateless/03990_alter_rename_column_swap_different_types.sql b/tests/queries/0_stateless/03990_alter_rename_column_swap_different_types.sql new file mode 100644 index 0000000000000..6fa04959b18e1 --- /dev/null +++ b/tests/queries/0_stateless/03990_alter_rename_column_swap_different_types.sql @@ -0,0 +1,113 @@ +-- Tags: no-random-merge-tree-settings +-- ^ wide/compact split is asserted explicitly via min_bytes_for_wide_part. + +-- Regression test for https://github.com/ClickHouse/ClickHouse/issues/103247 +-- A compound RENAME that reuses a freed column name (a "swap") between columns +-- of DIFFERENT types used to write the wrong type into the renamed part's +-- columns.txt: the target of "RENAME ip_v6 TO ip" inherited the type of the +-- same-named source column "ip" that was being renamed away, instead of the +-- type of "ip_v6". The data files were renamed correctly, so the part ended +-- up with an IPv6 column declared as UInt32 -> read-time CAST UInt32 -> IPv6 +-- (Code 48) or an assertion on part load in debug builds. + +DROP TABLE IF EXISTS rename_swap_compact; +DROP TABLE IF EXISTS rename_swap_wide; +DROP TABLE IF EXISTS rename_drop_into; + +SELECT '--- compact part: swap ip <-> ip_v6 (UInt32 <-> IPv6) ---'; + +CREATE TABLE rename_swap_compact (id UInt32, ip UInt32) +ENGINE = MergeTree ORDER BY id +SETTINGS min_bytes_for_wide_part = 1000000000 +AS SELECT * FROM values((0, 3628718300), (1, 0)); + +ALTER TABLE rename_swap_compact ADD COLUMN ip_v6 IPv6 AFTER ip SETTINGS mutations_sync = 2; +ALTER TABLE rename_swap_compact UPDATE ip_v6 = toIPv6(IPv4NumToString(ip)) WHERE ip != 0 SETTINGS mutations_sync = 2; +ALTER TABLE rename_swap_compact RENAME COLUMN ip TO ip_v4, RENAME COLUMN ip_v6 TO ip SETTINGS mutations_sync = 2; + +DESCRIBE rename_swap_compact; +SELECT id, ip_v4, ip FROM rename_swap_compact ORDER BY id; +SELECT id, ip_v4, ip FROM rename_swap_compact WHERE ip_v4 = 0 ORDER BY id; + +SELECT '--- wide part: same swap ---'; + +CREATE TABLE rename_swap_wide (id UInt32, ip UInt32) +ENGINE = MergeTree ORDER BY id +SETTINGS min_bytes_for_wide_part = 0 +AS SELECT * FROM values((0, 3628718300), (1, 0)); + +ALTER TABLE rename_swap_wide ADD COLUMN ip_v6 IPv6 AFTER ip SETTINGS mutations_sync = 2; +ALTER TABLE rename_swap_wide UPDATE ip_v6 = toIPv6(IPv4NumToString(ip)) WHERE ip != 0 SETTINGS mutations_sync = 2; +ALTER TABLE rename_swap_wide RENAME COLUMN ip TO ip_v4, RENAME COLUMN ip_v6 TO ip SETTINGS mutations_sync = 2; + +DESCRIBE rename_swap_wide; +SELECT id, ip_v4, ip FROM rename_swap_wide ORDER BY id; +SELECT id, ip_v4, ip FROM rename_swap_wide WHERE ip_v4 = 0 ORDER BY id; + +SELECT '--- wide part: swap where the added column was never materialized ---'; +-- ip_v6 is ADDed but never UPDATEd, so existing parts have no ip_v6 data. The +-- swap then renames the unmaterialized ip_v6 onto the freed name ip. The new +-- ip must be dropped from the part and default-filled by the reader, while old +-- ip still becomes ip_v4. This used to throw LOGICAL_ERROR in part writing. + +CREATE TABLE rename_swap_wide_unmat (id UInt32, ip UInt32) +ENGINE = MergeTree ORDER BY id +SETTINGS min_bytes_for_wide_part = 0 +AS SELECT * FROM values((0, 3628718300), (1, 0)); + +ALTER TABLE rename_swap_wide_unmat ADD COLUMN ip_v6 IPv6 AFTER ip SETTINGS mutations_sync = 2; +ALTER TABLE rename_swap_wide_unmat RENAME COLUMN ip TO ip_v4, RENAME COLUMN ip_v6 TO ip SETTINGS mutations_sync = 2; + +DESCRIBE rename_swap_wide_unmat; +SELECT id, ip_v4, ip FROM rename_swap_wide_unmat ORDER BY id; + +SELECT '--- compact part: same unmaterialized swap (must keep working) ---'; + +CREATE TABLE rename_swap_compact_unmat (id UInt32, ip UInt32) +ENGINE = MergeTree ORDER BY id +SETTINGS min_bytes_for_wide_part = 1000000000 +AS SELECT * FROM values((0, 3628718300), (1, 0)); + +ALTER TABLE rename_swap_compact_unmat ADD COLUMN ip_v6 IPv6 AFTER ip SETTINGS mutations_sync = 2; +ALTER TABLE rename_swap_compact_unmat RENAME COLUMN ip TO ip_v4, RENAME COLUMN ip_v6 TO ip SETTINGS mutations_sync = 2; + +DESCRIBE rename_swap_compact_unmat; +SELECT id, ip_v4, ip FROM rename_swap_compact_unmat ORDER BY id; + +SELECT '--- wide part: unmaterialized swap with a part rewrite forced in the same ALTER ---'; +-- Same as above but a non-key UPDATE forces the part to be rewritten by the +-- mutation, so getColumnsForNewDataPart actually processes the part while the +-- renamed-from column (ip_v6) has no data in it. The new ip must be dropped +-- from the part (reader default-fills it) and old ip must become ip_v4. + +CREATE TABLE rename_swap_wide_unmat_rewrite (id UInt32, ip UInt32, payload String) +ENGINE = MergeTree ORDER BY id +SETTINGS min_bytes_for_wide_part = 0 +AS SELECT number, 3628718300, 'a' FROM numbers(2); + +ALTER TABLE rename_swap_wide_unmat_rewrite ADD COLUMN ip_v6 IPv6 AFTER ip SETTINGS mutations_sync = 2; +ALTER TABLE rename_swap_wide_unmat_rewrite + RENAME COLUMN ip TO ip_v4, RENAME COLUMN ip_v6 TO ip, UPDATE payload = 'b' WHERE 1 + SETTINGS mutations_sync = 2; + +DESCRIBE rename_swap_wide_unmat_rewrite; +SELECT id, ip_v4, ip, payload FROM rename_swap_wide_unmat_rewrite ORDER BY id; + +SELECT '--- DROP a + RENAME b TO a, different types (must keep working) ---'; + +CREATE TABLE rename_drop_into (id UInt32, a UInt32, b IPv6) +ENGINE = MergeTree ORDER BY id +SETTINGS min_bytes_for_wide_part = 0 +AS SELECT number, number, toIPv6('::ffff:1.2.3.4') FROM numbers(3); + +ALTER TABLE rename_drop_into DROP COLUMN a, RENAME COLUMN b TO a SETTINGS mutations_sync = 2; + +DESCRIBE rename_drop_into; +SELECT id, a FROM rename_drop_into ORDER BY id; + +DROP TABLE rename_swap_compact; +DROP TABLE rename_swap_wide; +DROP TABLE rename_swap_wide_unmat; +DROP TABLE rename_swap_compact_unmat; +DROP TABLE rename_swap_wide_unmat_rewrite; +DROP TABLE rename_drop_into; diff --git a/tests/queries/0_stateless/04004_integral_col_comparison_with_float_key_condition.reference b/tests/queries/0_stateless/04004_integral_col_comparison_with_float_key_condition.reference index 766cf2c807527..f6c9e1150a6c1 100644 --- a/tests/queries/0_stateless/04004_integral_col_comparison_with_float_key_condition.reference +++ b/tests/queries/0_stateless/04004_integral_col_comparison_with_float_key_condition.reference @@ -1,5 +1,6 @@ -- { echo } - +SET max_insert_threads = 1; +SET enable_parallel_replicas = 0; DROP TABLE IF EXISTS test_int64; CREATE TABLE test_int64 (uid Int64) ENGINE = MergeTree ORDER BY uid SETTINGS index_granularity = 8192; INSERT INTO test_int64 SELECT number FROM numbers(200000); diff --git a/tests/queries/0_stateless/04004_integral_col_comparison_with_float_key_condition.sql b/tests/queries/0_stateless/04004_integral_col_comparison_with_float_key_condition.sql index 9b9122cc09a11..da2db206d160b 100644 --- a/tests/queries/0_stateless/04004_integral_col_comparison_with_float_key_condition.sql +++ b/tests/queries/0_stateless/04004_integral_col_comparison_with_float_key_condition.sql @@ -1,4 +1,8 @@ +-- Tags: no-random-settings + -- { echo } +SET max_insert_threads = 1; +SET enable_parallel_replicas = 0; DROP TABLE IF EXISTS test_int64; CREATE TABLE test_int64 (uid Int64) ENGINE = MergeTree ORDER BY uid SETTINGS index_granularity = 8192; diff --git a/tests/queries/0_stateless/04077_datetime32_overflow.reference b/tests/queries/0_stateless/04077_datetime32_overflow.reference new file mode 100644 index 0000000000000..3a59ae1d69f12 --- /dev/null +++ b/tests/queries/0_stateless/04077_datetime32_overflow.reference @@ -0,0 +1,29 @@ +3 +2 +2 +3 +3 +3 +3 +3 +3 +3 +3 +3 +3 +3 +3 +3 +hello +world +3 +2 +3 +2 +3 +2 +2 +0 +0 +2 +3 diff --git a/tests/queries/0_stateless/04077_datetime32_overflow.sql b/tests/queries/0_stateless/04077_datetime32_overflow.sql new file mode 100644 index 0000000000000..5df6f259d7d4c --- /dev/null +++ b/tests/queries/0_stateless/04077_datetime32_overflow.sql @@ -0,0 +1,222 @@ +-- Verify that TTL expressions on `Date` / `DateTime` (16/32-bit) columns are evaluated +-- in widened types (`Date32` / `DateTime64`) so arithmetic with large intervals does not +-- silently wrap. Previously, `day + toIntervalDay(46000)` or `ts + INTERVAL 100 YEAR` +-- would overflow, corrupt part min/max TTL metadata, and drop entire parts during merge. +-- +-- The fix widens `Date` -> `Date32` and `DateTime` -> `DateTime64(0, tz)` in the +-- column list passed to the TTL expression analyzer, so arithmetic runs in the 64-bit +-- domain. At execution time the narrow block columns are cast to the widened types +-- before the expression runs. + +-- Case 1: rows TTL with a `Date` column and a >100-year interval. +DROP TABLE IF EXISTS t_ttl_overflow; +CREATE TABLE t_ttl_overflow (day Date, value UInt64) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t_ttl_overflow VALUES ('2024-01-01', 1), ('2024-06-15', 2), ('2025-12-31', 3); +ALTER TABLE t_ttl_overflow MODIFY TTL day + toIntervalDay(46000); +OPTIMIZE TABLE t_ttl_overflow FINAL; +-- TTL evaluates to year ~2150; data is retained. +SELECT count() FROM t_ttl_overflow; +DROP TABLE t_ttl_overflow; + +-- Case 2: same with `materialize_ttl_after_modify = 0` then an explicit `MATERIALIZE TTL`. +DROP TABLE IF EXISTS t_ttl_overflow2; +CREATE TABLE t_ttl_overflow2 (day Date, value UInt64) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t_ttl_overflow2 VALUES ('2024-01-01', 10), ('2024-06-15', 20); +SET materialize_ttl_after_modify = 0; +ALTER TABLE t_ttl_overflow2 MODIFY TTL day + toIntervalDay(46000); +SELECT count() FROM t_ttl_overflow2; +SET mutations_sync = 1; +ALTER TABLE t_ttl_overflow2 MATERIALIZE TTL; +SELECT count() FROM t_ttl_overflow2; +SET materialize_ttl_after_modify = 1; +-- Keep `mutations_sync = 2` for the rest of the file so that the implicit +-- `MATERIALIZE TTL` mutation fired by `MODIFY TTL` (with the default +-- `materialize_ttl_after_modify = 1`) completes synchronously. Otherwise the +-- `SELECT count()` right after each `ALTER` could run before the TTL is actually +-- evaluated and pass spuriously even if the 32-bit overflow path were still live. +SET mutations_sync = 2; +DROP TABLE t_ttl_overflow2; + +-- Case 3: every `add*` granularity must widen for `Date` columns. +DROP TABLE IF EXISTS t_ttl_date_fns; +CREATE TABLE t_ttl_date_fns (day Date, value UInt64) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t_ttl_date_fns VALUES ('2024-01-01', 1), ('2024-06-15', 2), ('2025-12-31', 3); + +ALTER TABLE t_ttl_date_fns MODIFY TTL day + INTERVAL 200 YEAR; +SELECT count() FROM t_ttl_date_fns; +ALTER TABLE t_ttl_date_fns MODIFY TTL addYears(day, 150); +SELECT count() FROM t_ttl_date_fns; +ALTER TABLE t_ttl_date_fns MODIFY TTL addMonths(day, 2000); +SELECT count() FROM t_ttl_date_fns; +ALTER TABLE t_ttl_date_fns MODIFY TTL addWeeks(day, 6600); +SELECT count() FROM t_ttl_date_fns; +ALTER TABLE t_ttl_date_fns MODIFY TTL addDays(day, 46000); +SELECT count() FROM t_ttl_date_fns; +DROP TABLE t_ttl_date_fns; + +-- Case 4: every `add*` granularity must widen for `DateTime` columns. +DROP TABLE IF EXISTS t_ttl_datetime_fns; +CREATE TABLE t_ttl_datetime_fns (ts DateTime, value UInt64) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t_ttl_datetime_fns VALUES ('2034-01-01 00:00:00', 1), ('2034-06-15 12:00:00', 2), ('2035-12-31 23:59:59', 3); + +ALTER TABLE t_ttl_datetime_fns MODIFY TTL ts + INTERVAL 100 YEAR; +SELECT count() FROM t_ttl_datetime_fns; +ALTER TABLE t_ttl_datetime_fns MODIFY TTL addYears(ts, 100); +SELECT count() FROM t_ttl_datetime_fns; +ALTER TABLE t_ttl_datetime_fns MODIFY TTL addMonths(ts, 1200); +SELECT count() FROM t_ttl_datetime_fns; +ALTER TABLE t_ttl_datetime_fns MODIFY TTL addWeeks(ts, 5200); +SELECT count() FROM t_ttl_datetime_fns; +ALTER TABLE t_ttl_datetime_fns MODIFY TTL addDays(ts, 36500); +SELECT count() FROM t_ttl_datetime_fns; +ALTER TABLE t_ttl_datetime_fns MODIFY TTL addHours(ts, 876000); +SELECT count() FROM t_ttl_datetime_fns; +ALTER TABLE t_ttl_datetime_fns MODIFY TTL addMinutes(ts, 52560000); +SELECT count() FROM t_ttl_datetime_fns; +ALTER TABLE t_ttl_datetime_fns MODIFY TTL addSeconds(ts, 3153600000); +SELECT count() FROM t_ttl_datetime_fns; +DROP TABLE t_ttl_datetime_fns; + +-- Case 5: column TTL (`TTLColumnAlgorithm`). +DROP TABLE IF EXISTS t_ttl_col_algo; +CREATE TABLE t_ttl_col_algo (day Date, val String) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t_ttl_col_algo VALUES ('2024-01-01', 'hello'), ('2024-06-15', 'world'); +ALTER TABLE t_ttl_col_algo MODIFY COLUMN val String TTL day + toIntervalDay(46000); +SELECT val FROM t_ttl_col_algo ORDER BY val; +DROP TABLE t_ttl_col_algo; + +-- Case 6: GROUP BY TTL (`TTLAggregationAlgorithm`). +DROP TABLE IF EXISTS t_ttl_groupby_algo; +CREATE TABLE t_ttl_groupby_algo (day Date, key UInt64, val UInt64) ENGINE = MergeTree ORDER BY key; +INSERT INTO t_ttl_groupby_algo VALUES ('2024-01-01', 1, 10), ('2024-01-01', 1, 20), ('2024-06-15', 2, 30); +ALTER TABLE t_ttl_groupby_algo MODIFY TTL day + toIntervalDay(46000) GROUP BY key SET val = max(val); +-- Aggregation does not run because TTL is in the future; row count stays 3. +SELECT count() FROM t_ttl_groupby_algo; +DROP TABLE t_ttl_groupby_algo; + +-- Case 7: RECOMPRESS TTL (`TTLUpdateInfoAlgorithm`). `materialize_ttl_after_modify` +-- is disabled here because `MATERIALIZE TTL` expects a rows TTL — recompression-only +-- TTLs don't have one, and the auto-fired materialization mutation would fail flakily +-- under TSan with `Cannot MATERIALIZE TTL as there is no TTL set`. +DROP TABLE IF EXISTS t_ttl_recompress_algo; +CREATE TABLE t_ttl_recompress_algo (day Date, val UInt64) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t_ttl_recompress_algo VALUES ('2024-01-01', 1), ('2024-06-15', 2); +SET materialize_ttl_after_modify = 0; +ALTER TABLE t_ttl_recompress_algo MODIFY TTL day + toIntervalDay(46000) RECOMPRESS CODEC(ZSTD(1)); +SET materialize_ttl_after_modify = 1; +SELECT count() FROM t_ttl_recompress_algo; +DROP TABLE t_ttl_recompress_algo; + +-- LowCardinality on temporal types is not on by default — these cases need the +-- explicit opt-in. +SET allow_suspicious_low_cardinality_types = 1; + +-- Case 8: `LowCardinality(Date)` — widening must apply after unwrapping the LC layer. +DROP TABLE IF EXISTS t_ttl_lc_date; +CREATE TABLE t_ttl_lc_date (day LowCardinality(Date), value UInt64) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t_ttl_lc_date VALUES ('2024-01-01', 1), ('2024-06-15', 2), ('2025-12-31', 3); +ALTER TABLE t_ttl_lc_date MODIFY TTL day + toIntervalDay(46000); +SELECT count() FROM t_ttl_lc_date; +DROP TABLE t_ttl_lc_date; + +-- Case 9: `LowCardinality(DateTime)` — same check for the 32-bit timestamp path. +DROP TABLE IF EXISTS t_ttl_lc_datetime; +CREATE TABLE t_ttl_lc_datetime (ts LowCardinality(DateTime), value UInt64) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t_ttl_lc_datetime VALUES ('2034-01-01 00:00:00', 1), ('2034-06-15 12:00:00', 2); +ALTER TABLE t_ttl_lc_datetime MODIFY TTL ts + INTERVAL 100 YEAR; +SELECT count() FROM t_ttl_lc_datetime; +DROP TABLE t_ttl_lc_datetime; + +SET allow_suspicious_low_cardinality_types = 0; + +-- Case 9a: `Nullable(Date)` — the `Nullable` wrapper must be preserved during widening, +-- otherwise the analyzer sees the column as non-null and `isNull` / `ifNull` get +-- constant-folded, silently changing TTL decisions for `NULL` rows. +DROP TABLE IF EXISTS t_ttl_nullable_date; +CREATE TABLE t_ttl_nullable_date (day Nullable(Date), value UInt64) ENGINE = MergeTree ORDER BY tuple() SETTINGS allow_nullable_key = 1; +INSERT INTO t_ttl_nullable_date VALUES ('2024-01-01', 1), (NULL, 2), ('2025-12-31', 3); +ALTER TABLE t_ttl_nullable_date MODIFY TTL ifNull(day, toDate('1970-01-02')) + toIntervalDay(46000); +OPTIMIZE TABLE t_ttl_nullable_date FINAL; +-- All rows retained: TTL resolves to year ~2150 for non-null and to ~2095 for the NULL row +-- (1970-01-02 + 46000 days). +SELECT count() FROM t_ttl_nullable_date; +DROP TABLE t_ttl_nullable_date; + +-- Case 9b: `Nullable(DateTime)` — same check for the 32-bit timestamp path. Without +-- preserving `Nullable`, `isNull(ts)` would be constant-folded to `false`. +DROP TABLE IF EXISTS t_ttl_nullable_datetime; +CREATE TABLE t_ttl_nullable_datetime (ts Nullable(DateTime), value UInt64) ENGINE = MergeTree ORDER BY tuple() SETTINGS allow_nullable_key = 1; +INSERT INTO t_ttl_nullable_datetime VALUES ('2034-01-01 00:00:00', 1), (NULL, 2); +ALTER TABLE t_ttl_nullable_datetime MODIFY TTL ifNull(ts, toDateTime('2034-01-01 00:00:00')) + INTERVAL 100 YEAR; +SELECT count() FROM t_ttl_nullable_datetime; +DROP TABLE t_ttl_nullable_datetime; + +-- Case 10: a lambda parameter that shadows a table column must keep its lambda-local +-- type. The TTL expression contains `arrayFilter(day -> day != '', arr)` where the +-- lambda's `day` is a String element of `arr`, not the `Date` column `day`. Because the +-- widening happens at column-type level (not AST rewriting), the analyzer's normal name +-- resolution already binds `day` inside the lambda to the lambda parameter rather than +-- to the outer column, so this case is handled automatically. +DROP TABLE IF EXISTS t_ttl_lambda_shadow; +CREATE TABLE t_ttl_lambda_shadow (day Date, arr Array(String), value UInt64) ENGINE = MergeTree ORDER BY tuple(); +-- Future dates so the resulting TTL is not yet expired — the count assertion below then +-- meaningfully checks that the ALTER succeeded and rows survive (rather than collapsing +-- to 0 from immediate expiration). +INSERT INTO t_ttl_lambda_shadow VALUES ('2099-01-01', ['a', 'b'], 1), ('2099-06-15', ['', 'c'], 2); +ALTER TABLE t_ttl_lambda_shadow + MODIFY TTL day + toIntervalDay(length(arrayFilter(day -> day != '', arr))); +SELECT count() FROM t_ttl_lambda_shadow; +DROP TABLE t_ttl_lambda_shadow; + +-- Case 11: negative control — normal TTL expiration must still drop expired rows. +-- Year 2000 + 1 day is far in the past, so all rows should be removed at merge time. +DROP TABLE IF EXISTS t_ttl_normal; +CREATE TABLE t_ttl_normal (day Date, value UInt64) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t_ttl_normal VALUES ('2000-01-01', 100); +ALTER TABLE t_ttl_normal MODIFY TTL day + INTERVAL 1 DAY; +OPTIMIZE TABLE t_ttl_normal FINAL; +SELECT count() FROM t_ttl_normal; +DROP TABLE t_ttl_normal; + +-- Case 12: `DELETE WHERE` must keep the original source-column types. Only the +-- timestamp expression is widened; widening this predicate would make the condition +-- below false (`DateTime64` instead of `DateTime`) and incorrectly retain the row. +DROP TABLE IF EXISTS t_ttl_delete_where_type; +CREATE TABLE t_ttl_delete_where_type (ts DateTime, value UInt64) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t_ttl_delete_where_type VALUES ('2000-01-01 00:00:00', 1); +ALTER TABLE t_ttl_delete_where_type MODIFY TTL ts + INTERVAL 1 DAY DELETE WHERE toTypeName(ts) = 'DateTime'; +OPTIMIZE TABLE t_ttl_delete_where_type FINAL; +SELECT count() FROM t_ttl_delete_where_type; +DROP TABLE t_ttl_delete_where_type; + +-- Case 13: backward compatibility — a TTL using a function that accepts only the narrow +-- `DateTime` type (e.g. `tumbleStart`) must keep working. Widening the source column to +-- `DateTime64` makes `tumbleStart` reject it, so analysis falls back to the original +-- column types for such expressions: they explicitly operate in the narrow domain and +-- are out of scope for the overflow fix. Without the fallback this `ALTER` (and `ATTACH` +-- of a legacy table with such a TTL) would fail with an illegal-argument-type exception. +DROP TABLE IF EXISTS t_ttl_tumble; +CREATE TABLE t_ttl_tumble (ts DateTime, value UInt64) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t_ttl_tumble VALUES ('2099-01-01 00:00:00', 1), ('2099-06-15 12:00:00', 2); +ALTER TABLE t_ttl_tumble MODIFY TTL tumbleStart(ts, INTERVAL 1 HOUR) + INTERVAL 1 DAY; +OPTIMIZE TABLE t_ttl_tumble FINAL; +SELECT count() FROM t_ttl_tumble; +DROP TABLE t_ttl_tumble; + +-- Case 14: a bare `LowCardinality(DateTime)` column as the whole TTL expression +-- (found by AST fuzzer). Analysis widens it to `DateTime64` (dropping `LowCardinality`), +-- so the `CREATE` is accepted, but the runtime block still holds the original +-- `LowCardinality` column and takes the shortcut path in `executeExpressionAndGetColumn` +-- (the result column is present in the block, so the expression does not run). Without +-- a cast to the expression's result type there, the `INSERT` threw a logical error +-- `Unexpected type (LowCardinality(UInt32)) of result TTL column` when computing the +-- part TTL info. +DROP TABLE IF EXISTS t_ttl_lc_bare; +SET allow_suspicious_low_cardinality_types = 1; +CREATE TABLE t_ttl_lc_bare (id UInt64, ts LowCardinality(DateTime)) ENGINE = MergeTree ORDER BY id TTL ts; +INSERT INTO t_ttl_lc_bare SELECT number, toDateTime('2106-01-01 00:00:00', 'UTC') FROM numbers(3); +INSERT INTO t_ttl_lc_bare SELECT number + 10, toDateTime(1, 'UTC') FROM numbers(2); +OPTIMIZE TABLE t_ttl_lc_bare FINAL; +-- The two rows from 1970 are expired and dropped by the merge; the 2106 rows survive. +SELECT count() FROM t_ttl_lc_bare; +DROP TABLE t_ttl_lc_bare; diff --git a/tests/queries/0_stateless/04077_part_moves_between_shards_keeper_component.reference b/tests/queries/0_stateless/04077_part_moves_between_shards_keeper_component.reference new file mode 100644 index 0000000000000..aa47d0d46d47a --- /dev/null +++ b/tests/queries/0_stateless/04077_part_moves_between_shards_keeper_component.reference @@ -0,0 +1,2 @@ +0 +0 diff --git a/tests/queries/0_stateless/04077_part_moves_between_shards_keeper_component.sql b/tests/queries/0_stateless/04077_part_moves_between_shards_keeper_component.sql new file mode 100644 index 0000000000000..bcc4a8ff27712 --- /dev/null +++ b/tests/queries/0_stateless/04077_part_moves_between_shards_keeper_component.sql @@ -0,0 +1,23 @@ +-- Tags: zookeeper + +-- Regression test for missing Coordination component in PartMovesBetweenShardsOrchestrator. +-- When enforce_keeper_component_tracking is enabled, querying system.part_moves_between_shards +-- would crash with "Current component is empty" because the orchestrator methods did not set +-- the coordination component before making ZooKeeper requests. + +DROP TABLE IF EXISTS test_part_moves_keeper_component; + +CREATE TABLE test_part_moves_keeper_component (id UInt64) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/test_part_moves_keeper_component', 'r1') +ORDER BY id; + +INSERT INTO test_part_moves_keeper_component VALUES (1); + +-- This query triggers PartMovesBetweenShardsOrchestrator::getEntries() -> syncStateFromZK() +-- which makes ZooKeeper calls. Without the fix, this crashes with LOGICAL_ERROR. +SELECT count() FROM system.part_moves_between_shards WHERE database = currentDatabase(); + +-- Also test with a subquery filter (matching the stress test pattern from 02841_not_ready_set_bug.sh) +SELECT count() FROM system.part_moves_between_shards WHERE database IN (SELECT currentDatabase()); + +DROP TABLE test_part_moves_keeper_component; diff --git a/tests/queries/0_stateless/04104_any_join_semi_anti_not_ready_set.reference b/tests/queries/0_stateless/04104_any_join_semi_anti_not_ready_set.reference new file mode 100644 index 0000000000000..06f8170447155 --- /dev/null +++ b/tests/queries/0_stateless/04104_any_join_semi_anti_not_ready_set.reference @@ -0,0 +1,8 @@ +1 10 100 +2 20 200 +3 30 0 +1 10 100 +2 20 200 +3 30 0 +1 10 +2 20 diff --git a/tests/queries/0_stateless/04104_any_join_semi_anti_not_ready_set.sql b/tests/queries/0_stateless/04104_any_join_semi_anti_not_ready_set.sql new file mode 100644 index 0000000000000..a80909756712c --- /dev/null +++ b/tests/queries/0_stateless/04104_any_join_semi_anti_not_ready_set.sql @@ -0,0 +1,67 @@ +-- tryConvertAnyJoinToSemiOrAntiJoin used to call ActionsDAG::evaluatePartialResult on a WHERE DAG +-- containing IN (subquery); on debug/sanitizer builds FunctionIn on a not-ready Set aborted via +-- LOGICAL_ERROR before the surrounding try/catch could intercept + +DROP TABLE IF EXISTS t_left_004104; +DROP TABLE IF EXISTS t_right_004104; +DROP TABLE IF EXISTS t_subq_004104; + +CREATE TABLE t_left_04104 (a UInt64, b UInt64) ENGINE = MergeTree ORDER BY a; +CREATE TABLE t_right_04104 (a UInt64, c UInt64) ENGINE = MergeTree ORDER BY a; +CREATE TABLE t_subq_04104 (x UInt64) ENGINE = MergeTree ORDER BY x; + +INSERT INTO t_left_04104 VALUES (1, 10), (2, 20), (3, 30); +INSERT INTO t_right_04104 VALUES (1, 100), (2, 200); +INSERT INTO t_subq_04104 VALUES (1); + +SELECT a, b, c +FROM t_left_04104 +ANY LEFT JOIN t_right_04104 USING (a) +WHERE toUInt64(1 IN (SELECT x FROM t_subq_04104)) > 0 +ORDER BY a +SETTINGS + query_plan_filter_push_down = 0, + query_plan_split_filter = 0, + query_plan_merge_filters = 0; + +SELECT a, b, c +FROM t_right_04104 +ANY RIGHT JOIN t_left_04104 USING (a) +WHERE toUInt64(1 IN (SELECT x FROM t_subq_04104)) > 0 +ORDER BY a +SETTINGS + query_plan_filter_push_down = 0, + query_plan_split_filter = 0, + query_plan_merge_filters = 0; + +TRUNCATE TABLE t_subq_04104; + +SELECT a, b +FROM t_left_04104 +ANY LEFT JOIN ( + SELECT a, notIn(1, (SELECT x FROM t_subq_04104)) AS flag FROM t_right_04104 +) AS rr USING (a) +WHERE rr.flag +ORDER BY a +SETTINGS + query_plan_filter_push_down = 0, + query_plan_split_filter = 0, + query_plan_merge_filters = 0; + +INSERT INTO t_subq_04104 VALUES (1); + +SELECT a, b +FROM t_left_04104 +ANY LEFT JOIN ( + SELECT a, notIn(1, (SELECT x FROM t_subq_04104)) AS flag FROM t_right_04104 +) AS rr USING (a) +WHERE rr.flag +ORDER BY a +SETTINGS + query_plan_filter_push_down = 0, + query_plan_split_filter = 0, + query_plan_merge_filters = 0; + +DROP TABLE t_left_04104; +DROP TABLE t_right_04104; +DROP TABLE t_subq_04104; diff --git a/tests/queries/0_stateless/04161_null_subcolumn_after_metadata_only_nullable_alter.reference b/tests/queries/0_stateless/04161_null_subcolumn_after_metadata_only_nullable_alter.reference new file mode 100644 index 0000000000000..9ec7614951e0a --- /dev/null +++ b/tests/queries/0_stateless/04161_null_subcolumn_after_metadata_only_nullable_alter.reference @@ -0,0 +1,36 @@ +compact, DEFAULT NULL +0 +0 +42 0 +compact, no default +1 +1 +1 +1 +1 +1 +1 42 0 +2 \N 1 +wide +0 +0 +42 0 +string +1 +1 +1 a 0 +2 \N 1 +memory +1 +1 +1 +1 +1 42 0 +2 \N 1 +apply_mutations_on_fly +1 +1 +1 0 0 +2 \N 1 +tuple element +7 diff --git a/tests/queries/0_stateless/04161_null_subcolumn_after_metadata_only_nullable_alter.sql b/tests/queries/0_stateless/04161_null_subcolumn_after_metadata_only_nullable_alter.sql new file mode 100644 index 0000000000000..5dd8b9c667eff --- /dev/null +++ b/tests/queries/0_stateless/04161_null_subcolumn_after_metadata_only_nullable_alter.sql @@ -0,0 +1,126 @@ +DROP TABLE IF EXISTS t_null_sub_evolved; + +-- A part written before a metadata-only `T` -> `Nullable(T)` ALTER stores non-nullable +-- data and has no `.null` substream. Reading the `.null` subcolumn (as done by +-- optimize_functions_to_subcolumns for isNull/isNotNull/count) must derive it from the +-- present parent column, giving null-map = 0 for the existing rows, not the storage-type +-- default (NULL). + +CREATE TABLE t_null_sub_evolved (id UInt8, x UInt256) +ENGINE = MergeTree ORDER BY tuple() SETTINGS auto_statistics_types = ''; + +INSERT INTO t_null_sub_evolved VALUES (1, 42); + +ALTER TABLE t_null_sub_evolved + MODIFY COLUMN x Nullable(UInt256) DEFAULT NULL +SETTINGS mutations_sync = 2; + +SELECT 'compact, DEFAULT NULL'; +SELECT count() FROM t_null_sub_evolved WHERE id = 1 AND x IS NULL SETTINGS optimize_functions_to_subcolumns = 1; +SELECT count() FROM t_null_sub_evolved WHERE id = 1 AND x IS NULL SETTINGS optimize_functions_to_subcolumns = 0; +SELECT x, x.null FROM t_null_sub_evolved ORDER BY id; + +DROP TABLE t_null_sub_evolved; + +-- No DEFAULT, add a genuine NULL in a new part; check isNull/isNotNull/count. +CREATE TABLE t_null_sub_evolved (id UInt8, x UInt256) +ENGINE = MergeTree ORDER BY tuple() SETTINGS auto_statistics_types = ''; + +INSERT INTO t_null_sub_evolved VALUES (1, 42); +ALTER TABLE t_null_sub_evolved MODIFY COLUMN x Nullable(UInt256) SETTINGS mutations_sync = 2; +INSERT INTO t_null_sub_evolved VALUES (2, NULL); + +SELECT 'compact, no default'; +SELECT count() FROM t_null_sub_evolved WHERE x IS NULL SETTINGS optimize_functions_to_subcolumns = 1; +SELECT count() FROM t_null_sub_evolved WHERE x IS NULL SETTINGS optimize_functions_to_subcolumns = 0; +SELECT count() FROM t_null_sub_evolved WHERE x IS NOT NULL SETTINGS optimize_functions_to_subcolumns = 1; +SELECT count() FROM t_null_sub_evolved WHERE x IS NOT NULL SETTINGS optimize_functions_to_subcolumns = 0; +SELECT count(x) FROM t_null_sub_evolved SETTINGS optimize_functions_to_subcolumns = 1; +SELECT count(x) FROM t_null_sub_evolved SETTINGS optimize_functions_to_subcolumns = 0; +SELECT id, x, x.null FROM t_null_sub_evolved ORDER BY id; + +DROP TABLE t_null_sub_evolved; + +-- Wide part. +CREATE TABLE t_null_sub_evolved (id UInt8, x UInt256) +ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0, auto_statistics_types = ''; + +INSERT INTO t_null_sub_evolved VALUES (1, 42); +ALTER TABLE t_null_sub_evolved MODIFY COLUMN x Nullable(UInt256) SETTINGS mutations_sync = 2; + +SELECT 'wide'; +SELECT count() FROM t_null_sub_evolved WHERE x IS NULL SETTINGS optimize_functions_to_subcolumns = 1; +SELECT count() FROM t_null_sub_evolved WHERE x IS NULL SETTINGS optimize_functions_to_subcolumns = 0; +SELECT x, x.null FROM t_null_sub_evolved ORDER BY id; + +DROP TABLE t_null_sub_evolved; + +-- String -> Nullable(String) evolution. +CREATE TABLE t_null_sub_evolved (id UInt8, x String) +ENGINE = MergeTree ORDER BY tuple() SETTINGS auto_statistics_types = ''; + +INSERT INTO t_null_sub_evolved VALUES (1, 'a'); +ALTER TABLE t_null_sub_evolved MODIFY COLUMN x Nullable(String) SETTINGS mutations_sync = 2; +INSERT INTO t_null_sub_evolved VALUES (2, NULL); + +SELECT 'string'; +SELECT count() FROM t_null_sub_evolved WHERE x IS NULL SETTINGS optimize_functions_to_subcolumns = 1; +SELECT count() FROM t_null_sub_evolved WHERE x IS NULL SETTINGS optimize_functions_to_subcolumns = 0; +SELECT id, x, x.null FROM t_null_sub_evolved ORDER BY id; + +DROP TABLE t_null_sub_evolved; + +-- Memory engine: MODIFY COLUMN is metadata-only (in-RAM blocks are not rewritten), so a block +-- inserted before the ALTER holds non-nullable data with no `.null` substream. Reading `.null` +-- must derive it from the converted parent (null-map = 0), not crash and not return NULL. +DROP TABLE IF EXISTS t_null_sub_mem; +CREATE TABLE t_null_sub_mem (id UInt8, x UInt256) ENGINE = Memory; + +INSERT INTO t_null_sub_mem VALUES (1, 42); +ALTER TABLE t_null_sub_mem MODIFY COLUMN x Nullable(UInt256); +INSERT INTO t_null_sub_mem VALUES (2, NULL); + +SELECT 'memory'; +SELECT count() FROM t_null_sub_mem WHERE x IS NULL SETTINGS optimize_functions_to_subcolumns = 1; +SELECT count() FROM t_null_sub_mem WHERE x IS NULL SETTINGS optimize_functions_to_subcolumns = 0; +SELECT count() FROM t_null_sub_mem WHERE x IS NOT NULL SETTINGS optimize_functions_to_subcolumns = 1; +SELECT count(x) FROM t_null_sub_mem SETTINGS optimize_functions_to_subcolumns = 1; +SELECT id, x, x.null FROM t_null_sub_mem ORDER BY id; + +DROP TABLE t_null_sub_mem; + +-- apply_mutations_on_fly: an on-fly UPDATE produces the full `x` in an earlier pipeline step, +-- then a metadata-only MODIFY COLUMN makes it Nullable. The later step reads only `x.null`; it +-- must be derived from the parent produced by the on-fly step (null-map = 0), not default-filled +-- to all-NULL (issue #110555 corner case). +DROP TABLE IF EXISTS t_null_sub_amof; +CREATE TABLE t_null_sub_amof (id UInt8, x UInt8) +ENGINE = MergeTree ORDER BY id SETTINGS auto_statistics_types = ''; + +INSERT INTO t_null_sub_amof VALUES (1, 5); +SYSTEM STOP MERGES t_null_sub_amof; +ALTER TABLE t_null_sub_amof UPDATE x = 0 WHERE 1 SETTINGS mutations_sync = 0; +ALTER TABLE t_null_sub_amof MODIFY COLUMN x Nullable(UInt8) SETTINGS mutations_sync = 0, alter_sync = 0; +INSERT INTO t_null_sub_amof VALUES (2, NULL); + +SELECT 'apply_mutations_on_fly'; +SELECT count() FROM t_null_sub_amof WHERE x IS NULL SETTINGS apply_mutations_on_fly = 1, optimize_functions_to_subcolumns = 1; +SELECT count() FROM t_null_sub_amof WHERE x IS NULL SETTINGS apply_mutations_on_fly = 1, optimize_functions_to_subcolumns = 0; +SELECT id, x, x.null FROM t_null_sub_amof ORDER BY id SETTINGS apply_mutations_on_fly = 1; + +DROP TABLE t_null_sub_amof; + +-- Tuple subcolumn that STILL exists in the old block type after a sibling element's +-- metadata-only type change. Reading `t.a` must extract it directly and must NOT cast the whole +-- tuple (which would throw on the non-convertible old `b` value). Memory engine keeps the +-- pre-ALTER block, so this exercises the old-type-still-has-subcolumn path. +DROP TABLE IF EXISTS t_tuple_elem_mem; +CREATE TABLE t_tuple_elem_mem (id UInt8, t Tuple(a UInt8, b String)) ENGINE = Memory; + +INSERT INTO t_tuple_elem_mem VALUES (1, (7, 'x')); +ALTER TABLE t_tuple_elem_mem MODIFY COLUMN t Tuple(a UInt8, b UInt64); + +SELECT 'tuple element'; +SELECT t.a FROM t_tuple_elem_mem; + +DROP TABLE t_tuple_elem_mem; diff --git a/tests/queries/0_stateless/04307_03285_3bba_interval_binary_encoding_invalid_kind.reference b/tests/queries/0_stateless/04307_03285_3bba_interval_binary_encoding_invalid_kind.reference new file mode 100644 index 0000000000000..e1669917121b7 --- /dev/null +++ b/tests/queries/0_stateless/04307_03285_3bba_interval_binary_encoding_invalid_kind.reference @@ -0,0 +1,3 @@ +Unknown IntervalKind during Interval type decoding: 0x0b. (INCORRECT_DATA) +Unknown IntervalKind during Interval type decoding: 0x7f. (INCORRECT_DATA) +Unknown IntervalKind during Interval type decoding: 0xff. (INCORRECT_DATA) diff --git a/tests/queries/0_stateless/04307_03285_3bba_interval_binary_encoding_invalid_kind.sh b/tests/queries/0_stateless/04307_03285_3bba_interval_binary_encoding_invalid_kind.sh new file mode 100755 index 0000000000000..afc12d59b39d2 --- /dev/null +++ b/tests/queries/0_stateless/04307_03285_3bba_interval_binary_encoding_invalid_kind.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Valid `IntervalKind::Kind` values are 0x00..0x0A. Any byte above 0x0A is +# rejected by `decodeDataType` so it cannot reach `magic_enum::enum_name` on +# an out-of-range enum (which used to feed a null `std::string_view` into +# `SerializationInterval::getHash` and trip UBSan in `SipHash::update`). + +# Bytes per column: `01` num_columns, `01` name_len, `74` ('t'), `22` +# `BinaryTypeIndex::Interval`, then the `IntervalKind` byte. + +$CLICKHOUSE_LOCAL -q "select x'010174220b' format TSVRaw" | $CLICKHOUSE_LOCAL --input-format RowBinaryWithNamesAndTypes --input_format_binary_decode_types_in_binary_format -q "select * from table" 2>&1 | grep "Unknown IntervalKind.*INCORRECT_DATA)" -o +$CLICKHOUSE_LOCAL -q "select x'010174227f' format TSVRaw" | $CLICKHOUSE_LOCAL --input-format RowBinaryWithNamesAndTypes --input_format_binary_decode_types_in_binary_format -q "select * from table" 2>&1 | grep "Unknown IntervalKind.*INCORRECT_DATA)" -o +$CLICKHOUSE_LOCAL -q "select x'01017422ff' format TSVRaw" | $CLICKHOUSE_LOCAL --input-format RowBinaryWithNamesAndTypes --input_format_binary_decode_types_in_binary_format -q "select * from table" 2>&1 | grep "Unknown IntervalKind.*INCORRECT_DATA)" -o diff --git a/tests/queries/0_stateless/04357_analysisOfVariance_deserialize_size.reference b/tests/queries/0_stateless/04357_analysisOfVariance_deserialize_size.reference new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/queries/0_stateless/04357_analysisOfVariance_deserialize_size.sql b/tests/queries/0_stateless/04357_analysisOfVariance_deserialize_size.sql new file mode 100644 index 0000000000000..5474dc68c5192 --- /dev/null +++ b/tests/queries/0_stateless/04357_analysisOfVariance_deserialize_size.sql @@ -0,0 +1,10 @@ +-- analysisOfVariance keeps three parallel arrays in its aggregate state (one entry per group). +-- The finalize path iterates one array and indexes the others at the same position, so a state +-- deserialized from raw bytes with mismatched array lengths reads out of bounds. Such a state is +-- reachable from any user via CAST of a String to AggregateFunction. + +-- Normal aggregation builds equal-length arrays and must keep working. +SELECT analysisOfVariance(number, number % 3) FROM numbers(30) FORMAT Null; + +-- xs1 and xs2 have two groups, ns has one -> ns is read past its end. +SELECT finalizeAggregation(CAST(unhex('02000000000000f03f000000000000f03f02000000000000f03f000000000000f03f010500000000000000') AS AggregateFunction(analysisOfVariance, Float64, UInt8))); -- { serverError INCORRECT_DATA } diff --git a/tests/queries/0_stateless/04401_has_column_in_table_access_check.reference b/tests/queries/0_stateless/04401_has_column_in_table_access_check.reference new file mode 100644 index 0000000000000..ddea6a6a2c695 --- /dev/null +++ b/tests/queries/0_stateless/04401_has_column_in_table_access_check.reference @@ -0,0 +1,22 @@ +-- without SHOW COLUMNS: denied (existing column) +ACCESS_DENIED +-- without SHOW COLUMNS: denied for a non-existing table (no existence oracle) +ACCESS_DENIED +-- without SHOW COLUMNS: denied for a non-existing database (no existence oracle) +ACCESS_DENIED +-- without SHOW COLUMNS: localhost overload denied (existing column) +ACCESS_DENIED +-- without SHOW COLUMNS: localhost overload denied for a non-existing table (no existence oracle) +ACCESS_DENIED +-- without SHOW COLUMNS: non-loopback remote-to-self denied (existing column) +ACCESS_DENIED +-- without SHOW COLUMNS: non-loopback remote-to-self denied for a non-existing table (no existence oracle) +ACCESS_DENIED +-- with SHOW COLUMNS: existing column +1 +-- with SHOW COLUMNS: non-existing column +0 +-- with SHOW COLUMNS: localhost overload existing column +1 +-- with SHOW COLUMNS: localhost overload non-existing column +0 diff --git a/tests/queries/0_stateless/04401_has_column_in_table_access_check.sh b/tests/queries/0_stateless/04401_has_column_in_table_access_check.sh new file mode 100755 index 0000000000000..595df508e21e0 --- /dev/null +++ b/tests/queries/0_stateless/04401_has_column_in_table_access_check.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# hasColumnInTable must require SHOW COLUMNS, like DESCRIBE / SHOW CREATE TABLE. +# User and table names are scoped to ${CLICKHOUSE_DATABASE} so the test is parallel-safe. + +user="user_${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" +${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED WITH plaintext_password BY 'password'" +${CLICKHOUSE_CLIENT} --query "GRANT SELECT ON system.one TO ${user}" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE ${CLICKHOUSE_DATABASE}.secret (id UInt64, password String) ENGINE = MergeTree ORDER BY id" + +run_as_user() { ${CLICKHOUSE_CLIENT} --user "${user}" --password password "$@"; } + +echo "-- without SHOW COLUMNS: denied (existing column)" +run_as_user --query "SELECT hasColumnInTable('${CLICKHOUSE_DATABASE}', 'secret', 'password')" 2>&1 | grep -Fo "ACCESS_DENIED" | uniq +echo "-- without SHOW COLUMNS: denied for a non-existing table (no existence oracle)" +run_as_user --query "SELECT hasColumnInTable('${CLICKHOUSE_DATABASE}', 'no_such_table', 'id')" 2>&1 | grep -Fo "ACCESS_DENIED" | uniq +echo "-- without SHOW COLUMNS: denied for a non-existing database (no existence oracle)" +run_as_user --query "SELECT hasColumnInTable('no_such_db_${CLICKHOUSE_DATABASE}', 'secret', 'id')" 2>&1 | grep -Fo "ACCESS_DENIED" | uniq + +# The 'localhost' overload collapses to a local shard, so the check must gate it too +# (getStructureOfRemoteTableInShard checks SHOW_COLUMNS on the caller context for local shards). +echo "-- without SHOW COLUMNS: localhost overload denied (existing column)" +run_as_user --query "SELECT hasColumnInTable('localhost', '${CLICKHOUSE_DATABASE}', 'secret', 'password')" 2>&1 | grep -Fo "ACCESS_DENIED" | uniq +echo "-- without SHOW COLUMNS: localhost overload denied for a non-existing table (no existence oracle)" +run_as_user --query "SELECT hasColumnInTable('localhost', '${CLICKHOUSE_DATABASE}', 'no_such_table', 'id')" 2>&1 | grep -Fo "ACCESS_DENIED" | uniq + +# A non-loopback host (127.0.0.2) is not matched by isLocalAddress, so it routes through the real +# remote path connecting as "default". The check now runs before dispatch, so this is denied without +# any connection attempt (deterministic regardless of what the server listens on). +echo "-- without SHOW COLUMNS: non-loopback remote-to-self denied (existing column)" +run_as_user --query "SELECT hasColumnInTable('127.0.0.2', '${CLICKHOUSE_DATABASE}', 'secret', 'password')" 2>&1 | grep -Fo "ACCESS_DENIED" | uniq +echo "-- without SHOW COLUMNS: non-loopback remote-to-self denied for a non-existing table (no existence oracle)" +run_as_user --query "SELECT hasColumnInTable('127.0.0.2', '${CLICKHOUSE_DATABASE}', 'no_such_table', 'id')" 2>&1 | grep -Fo "ACCESS_DENIED" | uniq + +${CLICKHOUSE_CLIENT} --query "GRANT SHOW COLUMNS ON ${CLICKHOUSE_DATABASE}.secret TO ${user}" + +echo "-- with SHOW COLUMNS: existing column" +run_as_user --query "SELECT hasColumnInTable('${CLICKHOUSE_DATABASE}', 'secret', 'password')" +echo "-- with SHOW COLUMNS: non-existing column" +run_as_user --query "SELECT hasColumnInTable('${CLICKHOUSE_DATABASE}', 'secret', 'not_a_column')" +echo "-- with SHOW COLUMNS: localhost overload existing column" +run_as_user --query "SELECT hasColumnInTable('localhost', '${CLICKHOUSE_DATABASE}', 'secret', 'password')" +echo "-- with SHOW COLUMNS: localhost overload non-existing column" +run_as_user --query "SELECT hasColumnInTable('localhost', '${CLICKHOUSE_DATABASE}', 'secret', 'not_a_column')" + +${CLICKHOUSE_CLIENT} --query "DROP TABLE ${CLICKHOUSE_DATABASE}.secret" +${CLICKHOUSE_CLIENT} --query "DROP USER ${user}" diff --git a/tests/queries/0_stateless/04409_explain_actions_secret_args.reference b/tests/queries/0_stateless/04409_explain_actions_secret_args.reference index 88ad32d9cb6d2..e1c0257b5bdb6 100644 --- a/tests/queries/0_stateless/04409_explain_actions_secret_args.reference +++ b/tests/queries/0_stateless/04409_explain_actions_secret_args.reference @@ -71,7 +71,9 @@ Actions: INPUT : 0 -> number UInt64 : 0 ALIAS hex(HMAC(\'sha256\'_String, toString(__table1.number), [HIDDEN id: 1]_String)) :: 2 -> hex(HMAC(\'sha256\', toString(number), [HIDDEN id: 1])) String : 3 Positions: 3 ReadFromSystemNumbers --- With the setting enabled the key is shown verbatim. +-- The session setting alone must not reveal the key: the `display_secrets_in_show_and_select` +-- server setting and the `displaySecretsInShowAndSelect` privilege are also required, and the +-- stateless test server has neither. SET format_display_secrets_in_show_and_select = 1; EXPLAIN actions = 1 SELECT encrypt('aes-128-ecb', toString(number), 'MY_SUPER_SECRET_KEY_123') @@ -79,10 +81,10 @@ FROM numbers(3); Expression ((Project names + (Projection + Change column names to column identifiers))) Actions: INPUT : 0 -> number UInt64 : 0 COLUMN Const(String) -> \'aes-128-ecb\'_String String : 1 - COLUMN Const(String) -> \'MY_SUPER_SECRET_KEY_123\'_String String : 2 + COLUMN Const(String) -> [HIDDEN id: 1]_String String : 2 ALIAS number :: 0 -> __table1.number UInt64 : 3 FUNCTION toString(__table1.number :: 3) -> toString(__table1.number) String : 0 - FUNCTION encrypt(\'aes-128-ecb\'_String :: 1, toString(__table1.number) :: 0, \'MY_SUPER_SECRET_KEY_123\'_String :: 2) -> encrypt(\'aes-128-ecb\'_String, toString(__table1.number), \'MY_SUPER_SECRET_KEY_123\'_String) String : 3 - ALIAS encrypt(\'aes-128-ecb\'_String, toString(__table1.number), \'MY_SUPER_SECRET_KEY_123\'_String) :: 3 -> encrypt(\'aes-128-ecb\', toString(number), \'MY_SUPER_SECRET_KEY_123\') String : 2 + FUNCTION encrypt(\'aes-128-ecb\'_String :: 1, toString(__table1.number) :: 0, [HIDDEN id: 1]_String :: 2) -> encrypt(\'aes-128-ecb\'_String, toString(__table1.number), [HIDDEN id: 1]_String) String : 3 + ALIAS encrypt(\'aes-128-ecb\'_String, toString(__table1.number), [HIDDEN id: 1]_String) :: 3 -> encrypt(\'aes-128-ecb\', toString(number), [HIDDEN id: 1]) String : 2 Positions: 2 ReadFromSystemNumbers diff --git a/tests/queries/0_stateless/04409_explain_actions_secret_args.sql b/tests/queries/0_stateless/04409_explain_actions_secret_args.sql index fcf81a67b05bc..4a4c2590f608a 100644 --- a/tests/queries/0_stateless/04409_explain_actions_secret_args.sql +++ b/tests/queries/0_stateless/04409_explain_actions_secret_args.sql @@ -28,7 +28,9 @@ EXPLAIN actions = 1 SELECT hex(HMAC('sha256', toString(number), 'MY_SUPER_SECRET_KEY_123')) FROM numbers(3); --- With the setting enabled the key is shown verbatim. +-- The session setting alone must not reveal the key: the `display_secrets_in_show_and_select` +-- server setting and the `displaySecretsInShowAndSelect` privilege are also required, and the +-- stateless test server has neither. SET format_display_secrets_in_show_and_select = 1; EXPLAIN actions = 1 diff --git a/tests/queries/0_stateless/04411_ttl_drop_not_vertical.reference b/tests/queries/0_stateless/04411_ttl_drop_not_vertical.reference new file mode 100644 index 0000000000000..876d94d1857a2 --- /dev/null +++ b/tests/queries/0_stateless/04411_ttl_drop_not_vertical.reference @@ -0,0 +1,4 @@ +TTLDropMerge Horizontal 0 +0 +TTLDropMerge Horizontal 0 +0 diff --git a/tests/queries/0_stateless/04411_ttl_drop_not_vertical.sh b/tests/queries/0_stateless/04411_ttl_drop_not_vertical.sh new file mode 100755 index 0000000000000..d34ea2f74dc6a --- /dev/null +++ b/tests/queries/0_stateless/04411_ttl_drop_not_vertical.sh @@ -0,0 +1,154 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +set -e + +TABLE=t_ttl_drop_not_vertical + +function wait_for_ttl_drop_merge() +{ + for _ in $(seq 1 600); do + ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS part_log" + local merge_count + merge_count=$(${CLICKHOUSE_CLIENT} -q " + SELECT count() + FROM system.part_log + WHERE database = currentDatabase() + AND table = '${TABLE}' + AND event_type = 'MergeParts' + AND merge_reason = 'TTLDropMerge'") + + if [ "${merge_count}" -gt "0" ]; then + return + fi + sleep 0.1 + done + + echo "Timed out waiting for a TTLDropMerge of ${TABLE}" >&2 + exit 1 +} + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ${TABLE}" + +${CLICKHOUSE_CLIENT} -q " + CREATE TABLE ${TABLE} + ( + id UInt64, + d DateTime DEFAULT '2000-01-01 00:00:00', + c1 UInt64, + c2 UInt64, + c3 UInt64, + c4 UInt64 + ) + ENGINE = MergeTree + ORDER BY id + TTL d + INTERVAL 1 DAY + SETTINGS + ttl_only_drop_parts = 1, + merge_with_ttl_timeout = 0, + min_bytes_for_wide_part = 0, + min_bytes_for_full_part_storage = 0, + enable_block_number_column = 0, + enable_block_offset_column = 0, + vertical_merge_algorithm_min_rows_to_activate = 1, + vertical_merge_algorithm_min_columns_to_activate = 1, + vertical_merge_optimize_ttl_delete = 1, + ratio_of_defaults_for_sparse_serialization = 1.0" + +${CLICKHOUSE_CLIENT} -q "SYSTEM STOP TTL MERGES ${TABLE}" +${CLICKHOUSE_CLIENT} -q "SYSTEM STOP MERGES ${TABLE}" + +${CLICKHOUSE_CLIENT} -q " + INSERT INTO ${TABLE} + SELECT number, '2000-01-01 00:00:00', number, number, number, number + FROM numbers(1000)" +${CLICKHOUSE_CLIENT} -q " + INSERT INTO ${TABLE} + SELECT number + 1000, '2000-01-01 00:00:00', number, number, number, number + FROM numbers(1000)" + +${CLICKHOUSE_CLIENT} -q "SYSTEM START TTL MERGES ${TABLE}" +${CLICKHOUSE_CLIENT} -q "SYSTEM START MERGES ${TABLE}" + +wait_for_ttl_drop_merge + +${CLICKHOUSE_CLIENT} -q " + SELECT + merge_reason, + merge_algorithm, + rows + FROM system.part_log + WHERE database = currentDatabase() + AND table = '${TABLE}' + AND event_type = 'MergeParts' + AND merge_reason = 'TTLDropMerge' + ORDER BY event_time_microseconds DESC + LIMIT 1" + +${CLICKHOUSE_CLIENT} -q "SELECT count() FROM ${TABLE}" +${CLICKHOUSE_CLIENT} -q "DROP TABLE ${TABLE}" + +TABLE=t_ttl_drop_not_vertical_mixed_ttl + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ${TABLE}" + +${CLICKHOUSE_CLIENT} -q " + CREATE TABLE ${TABLE} + ( + id UInt64, + d DateTime DEFAULT '2000-01-01 00:00:00', + c1 UInt64, + c2 UInt64, + c3 UInt64, + c4 UInt64 + ) + ENGINE = MergeTree + ORDER BY id + TTL d + INTERVAL 1 DAY, d + INTERVAL 2 DAY RECOMPRESS CODEC(ZSTD) + SETTINGS + ttl_only_drop_parts = 1, + merge_with_ttl_timeout = 0, + min_bytes_for_wide_part = 0, + min_bytes_for_full_part_storage = 0, + enable_block_number_column = 0, + enable_block_offset_column = 0, + vertical_merge_algorithm_min_rows_to_activate = 1, + vertical_merge_algorithm_min_columns_to_activate = 1, + vertical_merge_optimize_ttl_delete = 1, + ratio_of_defaults_for_sparse_serialization = 1.0" + +${CLICKHOUSE_CLIENT} -q "SYSTEM STOP TTL MERGES ${TABLE}" +${CLICKHOUSE_CLIENT} -q "SYSTEM STOP MERGES ${TABLE}" + +${CLICKHOUSE_CLIENT} -q " + INSERT INTO ${TABLE} + SELECT number, '2000-01-01 00:00:00', number, number, number, number + FROM numbers(1000)" +${CLICKHOUSE_CLIENT} -q " + INSERT INTO ${TABLE} + SELECT number + 1000, '2000-01-01 00:00:00', number, number, number, number + FROM numbers(1000)" + +${CLICKHOUSE_CLIENT} -q "SYSTEM START TTL MERGES ${TABLE}" +${CLICKHOUSE_CLIENT} -q "SYSTEM START MERGES ${TABLE}" + +wait_for_ttl_drop_merge + +${CLICKHOUSE_CLIENT} -q " + SELECT + merge_reason, + merge_algorithm, + rows + FROM system.part_log + WHERE database = currentDatabase() + AND table = '${TABLE}' + AND event_type = 'MergeParts' + AND merge_reason = 'TTLDropMerge' + ORDER BY event_time_microseconds DESC + LIMIT 1" + +${CLICKHOUSE_CLIENT} -q "SELECT count() FROM ${TABLE}" +${CLICKHOUSE_CLIENT} -q "DROP TABLE ${TABLE}" diff --git a/tests/queries/0_stateless/04489_constraint_comparison_wrong_arity.reference b/tests/queries/0_stateless/04489_constraint_comparison_wrong_arity.reference new file mode 100644 index 0000000000000..9766475a4185a --- /dev/null +++ b/tests/queries/0_stateless/04489_constraint_comparison_wrong_arity.reference @@ -0,0 +1 @@ +ok diff --git a/tests/queries/0_stateless/04489_constraint_comparison_wrong_arity.sql b/tests/queries/0_stateless/04489_constraint_comparison_wrong_arity.sql new file mode 100644 index 0000000000000..e26158673b7e6 --- /dev/null +++ b/tests/queries/0_stateless/04489_constraint_comparison_wrong_arity.sql @@ -0,0 +1,23 @@ +-- Regression test: a comparison function with the wrong number of arguments used as a +-- constraint expression must not crash the server. It used to trigger an out-of-bounds +-- access in `ComparisonGraph::normalizeAtom`, reached via `ConstraintsDescription::buildGraph`, +-- because the unanalyzed AST `less(a)` (a unary `less`) was treated as a binary comparison and +-- its second argument was accessed unconditionally. + +DROP TABLE IF EXISTS t_constraint_arity; + +-- Unary `less` in a CHECK constraint at CREATE time. +CREATE TABLE t_constraint_arity (a UInt32, b UInt32, CONSTRAINT c0 CHECK less(a)) ENGINE = MergeTree ORDER BY a; + +-- The same added via ALTER, as CHECK and as ASSUME. +ALTER TABLE t_constraint_arity ADD CONSTRAINT c1 CHECK less(a); +ALTER TABLE t_constraint_arity ADD CONSTRAINT c2 ASSUME less(a); + +-- Other wrong arities and relations must be handled the same way. +ALTER TABLE t_constraint_arity ADD CONSTRAINT c3 CHECK lessOrEquals(a); +ALTER TABLE t_constraint_arity ADD CONSTRAINT c4 CHECK less(); +ALTER TABLE t_constraint_arity ADD CONSTRAINT c5 CHECK greater(a, b, a); + +SELECT 'ok'; + +DROP TABLE t_constraint_arity; diff --git a/tests/queries/0_stateless/04499_map_subcolumn_incorrect_column_default.reference b/tests/queries/0_stateless/04499_map_subcolumn_incorrect_column_default.reference new file mode 100644 index 0000000000000..792f36995c9e8 --- /dev/null +++ b/tests/queries/0_stateless/04499_map_subcolumn_incorrect_column_default.reference @@ -0,0 +1,4 @@ +BEFORE MATERIALIZE COLUMN +10 +AFTER MATERIALIZE COLUMN +10 diff --git a/tests/queries/0_stateless/04499_map_subcolumn_incorrect_column_default.sql b/tests/queries/0_stateless/04499_map_subcolumn_incorrect_column_default.sql new file mode 100644 index 0000000000000..d073f02a93024 --- /dev/null +++ b/tests/queries/0_stateless/04499_map_subcolumn_incorrect_column_default.sql @@ -0,0 +1,39 @@ +-- Reading a subcolumn of a column that is missing in a part must evaluate the column's DEFAULT +-- expression (reading its source columns from the part) instead of substituting type defaults. +-- https://github.com/ClickHouse/ClickHouse/issues/120326 +DROP TABLE IF EXISTS t_map_subcolumn_default; + +CREATE TABLE t_map_subcolumn_default +( + id UInt64, + `a.key` Array(String), + `a.value` Array(String) +) +ENGINE = MergeTree +ORDER BY id; + + +INSERT INTO t_map_subcolumn_default +SELECT number, ['k1', 'k2'], ['v1', 'v2'] +FROM numbers(10); + +ALTER TABLE t_map_subcolumn_default +ADD COLUMN m Map(String, Nullable(String)) +DEFAULT mapFromArrays(`a.key`, `a.value`); + +SELECT 'BEFORE MATERIALIZE COLUMN'; + +SELECT count() +FROM t_map_subcolumn_default +WHERE m['k1'] = 'v1'; + +ALTER TABLE t_map_subcolumn_default MATERIALIZE COLUMN m SETTINGS mutations_sync = 1; + +SELECT 'AFTER MATERIALIZE COLUMN'; + +SELECT count() +FROM t_map_subcolumn_default +WHERE m['k1'] = 'v1'; + + +DROP TABLE IF EXISTS t_map_subcolumn_default; diff --git a/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.reference b/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.reference index 702fa6365824d..393391c888dbf 100644 --- a/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.reference +++ b/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.reference @@ -197,6 +197,45 @@ BACKUP TABLE nonexistent_04510 TO S3(\'url_bkp_pos\', \'[HIDDEN]\', \'[HIDDEN]\' BACKUP TABLE nonexistent_04510 TO S3(nc_bkp_missing, \'visible_bkp_dir\', \'[HIDDEN]\') BACKUP TABLE nonexistent_04510 TO S3(nc_bkporder_missing, secret_access_key = \'[HIDDEN]\', \'visible_bkp_dir2\') BACKUP TABLE nonexistent_04510 TO S3(\'url_bkp_mixed\', equals(access_key_id, \'ak\'), \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO File(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO Disk(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO Memory(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO File(\'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO `Null`(\'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO Foo(\'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO Null(); +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureQueue(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'http://localhost:11111/acct\', \'visible_04510_cont\', \'visible_04510_dir/b.zip\'); +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, connection_string = \'DefaultEndpointsProtocol=https;AccountName=visible_04510_acct;AccountKey=[HIDDEN];\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, account_key = \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, equals(storage_account_url, \'http://localhost:11111/visible_04510_url\'), account_key = \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, container = \'visible_04510_c1\', container = \'visible_04510_c2\'); +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(\'http://localhost:11111/visible_04510_acct5\', \'visible_04510_cont5\', \'visible_04510_blob5\', \'visible_04510_acctname5\', \'[HIDDEN]\') +CREATE TABLE t_04510_azte1 (`x` UInt8) ENGINE = AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +CREATE TABLE t_04510_azte2 (`x` UInt8) ENGINE = AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\') +CREATE TABLE t_04510_azte3 (`x` UInt8) ENGINE = AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +CREATE TABLE t_04510_azte4 (`x` UInt8) ENGINE = AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +CREATE TABLE t_04510_azte5 (`x` UInt8) ENGINE = AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +CREATE TABLE t_04510_azte6 (`x` UInt8) ENGINE = AzureBlobStorage(\'DefaultEndpointsProtocol=https;AccountName=a;AccountKey=[HIDDEN];\', \'visible_04510_teco\', \'visible_04510_teblob\') +CREATE TABLE t_04510_azte7 (`x` UInt8) ENGINE = AzureBlobStorage(\'http://localhost:11111/visible_04510_teacct\', \'visible_04510_tec5\', \'visible_04510_teb5\', \'visible_04510_teacctname\', \'[HIDDEN]\') +CREATE TABLE t_04510_azte8 (`x` UInt8) ENGINE = AzureQueue(\'http://localhost:11111/visible_04510_teq/cont/*\', \'[HIDDEN]\') SETTINGS mode = \'unordered\' CREATE DATABASE db_04510_ec ENGINE = Backup(\'\', S3(\'url_dbec\', \'ak\', \'[HIDDEN]\', extra_credentials(external_id = \'[HIDDEN]\'))) CREATE DATABASE db_04510_postok ENGINE = Backup(\'\', S3(\'url_dbpostok\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\')) CREATE DATABASE db_04510_ncpos ENGINE = Backup(\'\', S3(nc_dbnc_missing, \'visible_dbnc_dir\', \'[HIDDEN]\')) @@ -207,6 +246,21 @@ CREATE DATABASE db_04510_mixed ENGINE = Backup(\'\', S3(\'url_dbmixed\', access_ CREATE DATABASE db_04510_ncurl ENGINE = Backup(\'\', S3(nc_dburl_missing, url = \'[HIDDEN]\')) CREATE DATABASE db_04510_hdr ENGINE = Backup(\'\', S3(\'url_dbhdr\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\')) CREATE DATABASE db_04510_expr ENGINE = Backup(\'\', S3(\'url_dbexpr\', \'ak\', \'[HIDDEN]\', \'[HIDDEN]\')) +CREATE DATABASE db_04510_quoted ENGINE = Backup(\'\', \'[HIDDEN]\') +CREATE DATABASE db_04510_tail ENGINE = Backup(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +CREATE DATABASE db_04510_tmap ENGINE = Backup(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\') +CREATE DATABASE db_04510_lone ENGINE = Backup(\'[HIDDEN]\') +CREATE DATABASE db_04510_swap ENGINE = Backup(\'[HIDDEN]\', \'[HIDDEN]\') +CREATE DATABASE db_04510_ident ENGINE = Backup(\'[HIDDEN]\', \'[HIDDEN]\') +CREATE DATABASE db_04510_ftail ENGINE = Backup(\'src_04510\', File(\'[HIDDEN]\', \'[HIDDEN]\')) +CREATE DATABASE db_04510_fover ENGINE = Backup(\'src_04510\', File(\'[HIDDEN]\', \'[HIDDEN]\')) +CREATE DATABASE db_04510_dover ENGINE = Backup(\'src_04510\', Disk(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\')) +CREATE DATABASE db_04510_mover ENGINE = Backup(\'src_04510\', Memory(\'[HIDDEN]\', \'[HIDDEN]\')) +CREATE DATABASE db_04510_fvalid ENGINE = Backup(\'src_04510\', File(\'nonexistent_04510\')); +CREATE DATABASE db_04510_dvalid ENGINE = Backup(\'src_04510\', Disk(\'backups\', \'nonexistent_04510\')); +CREATE DATABASE db_04510_eval ENGINE = Backup(\'\', S3(\'http://localhost:11111/test/04510eval\', \'ak\', \'[HIDDEN]\', extra_credentials(external_id = \'[HIDDEN]\'))) +CREATE DATABASE db_04510_nonlit ENGINE = Backup(\'\', S3(\'[HIDDEN]\', \'[HIDDEN]\')) +CREATE DATABASE db_04510_azure ENGINE = Backup(\'\', AzureBlobStorage(\'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\', \'[HIDDEN]\')) CREATE DATABASE db_04510_s3pos ENGINE = S3(\'url_dbs3pos\', \'ak\', \'[HIDDEN]\', \'[HIDDEN]\') EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3(\'https://[HIDDEN]@localhost:11111/test/04510qt?X-Amz-Signature=[HIDDEN]\', \'ak\', \'[HIDDEN]\', \'TSV\', \'x UInt8\') EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3(\'http://localhost:11111/test/04510qt\', NOSIGN, \'TSV\', \'x UInt8\', headers(\'Authorization\' = \'[HIDDEN]\')) @@ -215,3 +269,4 @@ EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3(\'http://localhost:11111/test EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3(\'http://localhost:11111/test/04510qt\', NOSIGN, \'TSV\', \'x UInt8\', headers(\'Authorization\' = \'[HIDDEN]\')) EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3(nc_04510_missing, url = \'https://[HIDDEN]@localhost:11111/test/04510qt?X-Amz-Signature=[HIDDEN]\', structure = \'x UInt8\') EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM s3(\'http://localhost:11111/test/04510qt\', \'ak\', \'[HIDDEN]\', \'TSV\', \'x UInt8\') UNION ALL SELECT 1 +1 0 0 diff --git a/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.sql b/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.sql index 54a01c7e532ad..33205f8de7c8b 100644 --- a/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.sql +++ b/tests/queries/0_stateless/04510_s3_explicit_url_named_secret_mask.sql @@ -234,6 +234,135 @@ BACKUP TABLE nonexistent_04510 TO S3(nc_bkporder_missing, BACKUP TABLE nonexistent_04510 TO S3('url_bkp_mixed', access_key_id = 'ak', 'SEKRIT_BKPMIX'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +-- The credential-free engines (Disk, File, Memory, Null) are credential-free only in the shape they +-- read: File and Memory take one argument, Disk two, Null none. A surplus argument is rejected after +-- the statement is logged, so it reaches the log holding whatever was written in it, an argument that is +-- no string is read by none of these engines and can carry a string of its own, and an engine name that +-- is not registered at all has no known shape to trust; all three must be hidden. The last statement is +-- the control: the shape Null does read carries nothing to hide, so it stays visible verbatim and gets +-- as far as resolving the table. +BACKUP TABLE nonexistent_04510 TO File('nonexistent_04510', + 'SEKRIT_TOFILEOVER'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +BACKUP TABLE nonexistent_04510 TO Disk('backups', 'nonexistent_04510', + 'SEKRIT_TODISKOVER'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +BACKUP TABLE nonexistent_04510 TO Memory('nonexistent_04510', + 'SEKRIT_TOMEMOVER'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +BACKUP TABLE nonexistent_04510 TO File(['SEKRIT_TOFILEARR']); -- { serverError BAD_GET } +BACKUP TABLE nonexistent_04510 TO Null('SEKRIT_TONULLOVER'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +BACKUP TABLE nonexistent_04510 TO Foo('SEKRIT_TOUNKNOWN'); -- { serverError BACKUP_ENGINE_NOT_FOUND } +BACKUP TABLE nonexistent_04510 TO Null(); -- { serverError UNKNOWN_TABLE } + +-- The AzureBlobStorage backup destination reads a different signature than the table engine of the same +-- name: a named collection with an optional filename, three arguments (connection string or account url, +-- container, path), or five (adding account_name and account_key). An argument outside those shapes is +-- rejected only after the statement is logged, and AzureQueue has no backup engine at all. The last +-- statement is the control: the three-argument shape has nothing to hide, so it stays visible verbatim. +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage('http://localhost:11111/acct', 'cont', 'blob', + 'SEKRIT_AZTO4'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, 'dir', + 'SEKRIT_AZTONCPOS'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureQueue('http://localhost:11111/acct', 'cont', 'blob', + 'SEKRIT_AZQTO'); -- { serverError BACKUP_ENGINE_NOT_FOUND } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage('DefaultEndpointsProtocol=https;AccountName=a;AccountKey=SEKRIT_AZTOCSKEY==;', + 'cont', 'blob', 'acct', 'SEKRIT_AZTOCS5'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage('http://localhost:11111/acct', 'visible_04510_cont', 'visible_04510_dir/b.zip'); -- { serverError STD_EXCEPTION } + +-- A named collection can be overridden per statement, and the destination evaluates those overrides as +-- constant expressions. An override this rule cannot read may hold either credential, and hiding a +-- connection string replaces the whole argument, which cannot be combined with hiding account_key, so +-- both shapes hide the locator whole. The last three statements are the controls: a connection string +-- alone still hides only its AccountKey, account_key alone is hidden by itself, and an account url +-- override does not take the replacement path, so it stays visible next to a hidden account_key. +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + connection_string = concat('DefaultEndpointsProtocol=https;AccountName=a;AccountKey=', + 'SEKRIT_AZNCEXPR')); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + connection_string = 'DefaultEndpointsProtocol=https;AccountName=a;AccountKey=c2VrcmV0Cg==;', + account_key = 'SEKRIT_AZNCBOTH'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + storage_account_url = concat('https://a.blob.core.windows.net/c?sig=', + 'SEKRIT_AZNCURLEXPR')); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + connection_string = 'DefaultEndpointsProtocol=https;AccountName=visible_04510_acct;AccountKey=SEKRIT_AZNCCS==;'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + account_key = 'SEKRIT_AZNCKEY'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + storage_account_url = 'http://localhost:11111/visible_04510_url', + account_key = 'SEKRIT_AZNCURLKEY'); -- { serverError BAD_ARGUMENTS } + +-- An override key can be an expression too, and one this rule cannot read hides which credential the +-- value is: both the computed key and a malformed override hide the locator whole, as the S3 form above +-- already does for a key it cannot read. +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + concat('account_', 'key') = 'SEKRIT_AZNCKEYEXPR'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + equals('account_key', 'SEKRIT_AZNCEQ3', 'surplus')); -- { serverError BAD_ARGUMENTS } + +-- A readable key does not make the override readable: the destination evaluates its value as a constant +-- expression, so a value that is no plain literal or identifier can nest a credential of its own and is +-- formatted verbatim before that evaluation rejects it, as the S3 form above already is for such a value. +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + container = headers('Authorization' = 'SEKRIT_AZNCHDR')); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + blob_path = concat('SEKRIT_AZNCPATHEXPR', '/b.zip')); -- { serverError BAD_ARGUMENTS } + +-- connection_string and storage_account_url are mutually exclusive, and the destination reads at most +-- one of them, so an override it never reads holds whatever was written there. The last statement is +-- the control: a key that carries no credential stays visible however often it is repeated. +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + connection_string = 'http://localhost:11111/acct', + storage_account_url = 'DefaultEndpointsProtocol=https;AccountName=a;AccountKey=SEKRIT_AZNCPAIR==;'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + connection_string = 'http://localhost:11111/acct', + connection_string = 'DefaultEndpointsProtocol=https;AccountName=a;AccountKey=SEKRIT_AZNCDUP==;'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, container = 'visible_04510_c1', container = 'visible_04510_c2'); -- { serverError BAD_ARGUMENTS } + +-- An account url is shown only when it is a plain storage account URL, which is what the destination +-- requires beside explicit credentials: userinfo, a query string (a SAS is a credential) and a fragment +-- each carry a credential of their own, in any shape and under any scheme spelling. A connection value +-- that is no string is read by neither this rule nor the destination, so it is hidden as well. The last +-- statement is the control: a plain url keeps the account url, container, path and account name visible +-- while account_key is hidden, and reaches the account_key decoding that rejects it. +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + storage_account_url = 'http://user:SEKRIT_AZNCUSERINFO@localhost:11111/acct'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(nc_04510_missing, + storage_account_url = 'HTTPS://localhost:11111/acct?sig=SEKRIT_AZNCSAS'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage('http://user:SEKRIT_AZ3USERINFO@localhost:11111/acct', + 'cont', 'blob'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage('http://localhost:11111/acct#f', 'cont', 'blob', + 'acct', 'SEKRIT_AZTO5KEY'); -- { serverError BAD_ARGUMENTS } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage(['SEKRIT_AZ3ARR'], 'cont', 'blob'); -- { serverError BAD_GET } +BACKUP TABLE nonexistent_04510 TO AzureBlobStorage('http://localhost:11111/visible_04510_acct5', + 'visible_04510_cont5', 'visible_04510_blob5', 'visible_04510_acctname5', + 'SEKRIT_AZ5PLAINKEY'); -- { serverError STD_EXCEPTION } + +-- The AzureBlobStorage table engine reads its arguments the same way: an override whose value or key +-- this rule cannot read may hold a credential the engine does read (the parser evaluates both as +-- constant expressions), an account url is shown only when it is a plain storage account url, and a +-- connection value that is no string is read by neither. A connection string is hidden by replacing +-- its whole argument, which cannot be combined with hiding a following account_key. The last two +-- statements are the controls: the connection string alone still hides only its AccountKey, and a +-- plain account url keeps url, container, path and account name visible beside a hidden account_key. +CREATE TABLE t_04510_azte1 (x UInt8) ENGINE = AzureBlobStorage(nc_04510_missing, + connection_string = headers('Authorization' = 'SEKRIT_AZTEHDR')); -- { serverError NAMED_COLLECTION_DOESNT_EXIST } +CREATE TABLE t_04510_azte2 (x UInt8) ENGINE = AzureBlobStorage(nc_04510_missing, + upper('account_key') = 'SEKRIT_AZTEKEY'); -- { serverError NAMED_COLLECTION_DOESNT_EXIST } +CREATE TABLE t_04510_azte3 (x UInt8) ENGINE = AzureBlobStorage(['SEKRIT_AZTEARR'], 'cont', 'blob'); -- { serverError BAD_ARGUMENTS } +CREATE TABLE t_04510_azte4 (x UInt8) ENGINE = AzureBlobStorage('http://localhost:11111/acct?sig=SEKRIT_AZTESAS', + 'cont', 'blob', 'acct', 'SEKRIT_AZTE5KEY'); -- { serverError STD_EXCEPTION } +CREATE TABLE t_04510_azte5 (x UInt8) ENGINE = AzureBlobStorage('DefaultEndpointsProtocol=https;AccountName=a;AccountKey=SEKRIT_AZTECS==;', + 'cont', 'blob', 'acct', 'SEKRIT_AZTEMIXKEY'); -- { serverError STD_EXCEPTION } +CREATE TABLE t_04510_azte6 (x UInt8) ENGINE = AzureBlobStorage('DefaultEndpointsProtocol=https;AccountName=a;AccountKey=SEKRIT_AZTECTLCS==;', + 'visible_04510_teco', 'visible_04510_teblob'); -- { serverError STD_EXCEPTION } +CREATE TABLE t_04510_azte7 (x UInt8) ENGINE = AzureBlobStorage('http://localhost:11111/visible_04510_teacct', + 'visible_04510_tec5', 'visible_04510_teb5', 'visible_04510_teacctname', + 'SEKRIT_AZTECTLKEY'); -- { serverError STD_EXCEPTION } + +-- The two-argument signature takes container and path from the url and a shared access signature +-- beside it, so its second argument is a credential wherever the same rule serves the engine. +CREATE TABLE t_04510_azte8 (x UInt8) ENGINE = AzureQueue('http://localhost:11111/visible_04510_teq/cont/*', + 'SEKRIT_AZTESAS2') SETTINGS mode = 'unordered'; -- { serverError UNKNOWN_FORMAT } + -- Backup database engine reconstructs the nested S3 destination; extra_credentials must be masked. CREATE DATABASE db_04510_ec ENGINE = Backup('', S3('url_dbec', 'ak', 'SEKRIT_SAK', extra_credentials(external_id = 'SEKRIT_EID'))); -- { serverError BAD_ARGUMENTS } @@ -272,14 +401,82 @@ CREATE DATABASE db_04510_mixed ENGINE = Backup('', S3('url_dbmixed', CREATE DATABASE db_04510_ncurl ENGINE = Backup('', S3(nc_dburl_missing, url = concat('https://user:SEKRIT_PW@', 'localhost/x?X-Amz-Signature=SEKRIT_SIG'))); -- { serverError BAD_ARGUMENTS } --- The reconstructor must fail closed on an unsupported tail (headers), not emit it verbatim. +-- The reconstructor must fail closed on an unsupported tail (headers), not emit it verbatim, and the +-- rejection message must not echo it either: a nested map's values are hidden only by its parent's +-- formatter, so such a node formatted on its own carries them in plaintext. CREATE DATABASE db_04510_hdr ENGINE = Backup('', S3('url_dbhdr', 'ak', 'SEKRIT_SAK', - headers('X-Auth' = 'SEKRIT_HDR'))); -- { serverError BAD_ARGUMENTS } + headers('X-Auth' = 'SEKRIT_DBHDR'))); -- { serverError BAD_ARGUMENTS } -- The reconstructor must also fail closed on a constant-expression extra_credentials key. CREATE DATABASE db_04510_expr ENGINE = Backup('', S3('url_dbexpr', 'ak', 'SEKRIT_SAK', extra_credentials(concat('extern', 'al_id') = 'SEKRIT_EXPR'))); -- { serverError BAD_ARGUMENTS } +-- A locator held in a string literal is text this finder cannot parse, and that text can carry an +-- access key or a presigned URL. The `Backup` engine accepts that form only while replaying its own +-- metadata, but a statement carrying it is formatted - by `PARALLEL WITH`, by the distributed DDL +-- queue, by `query_log` - before the engine rejects it, so it must be masked whole. +CREATE DATABASE db_04510_quoted ENGINE = Backup('', + 'S3(\'https://user:SEKRIT_PW@localhost:11111/x?X-Amz-Signature=SEKRIT_SIG\', \'ak\', \'SEKRIT_QUOTED\')'); -- { serverError BAD_ARGUMENTS } + +-- `Backup(database_name, locator)` is the only valid shape. The statement is logged before the arity +-- is validated, so a credential parked in a surplus argument must be hidden as well. +CREATE DATABASE db_04510_tail ENGINE = Backup('', S3('url_dbtail', 'ak', 'SEKRIT_SAK'), + 'SEKRIT_DBTAIL'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +CREATE DATABASE db_04510_tmap ENGINE = Backup('', S3('url_dbtmap', 'ak', 'SEKRIT_SAK'), + extra_credentials(external_id = 'SEKRIT_DBTMAP')); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +CREATE DATABASE db_04510_lone ENGINE = Backup(S3('url_dblone', 'ak', + 'SEKRIT_DBLONE')); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } + +-- A locator is also a credential carrier in the database-name position, where the argument is neither +-- masked by the S3 rule below nor safe to echo: the message that rejects it formats it standalone. +CREATE DATABASE db_04510_swap ENGINE = Backup(S3('url_dbswap', 'ak', 'SEKRIT_DBSWAP'), + ''); -- { serverError BAD_ARGUMENTS } + +-- An identifier is not a literal either, and the parsers evaluate one as a literal, so the +-- database-name position must hold a literal before the rest of the shape is trusted. +CREATE DATABASE db_04510_ident ENGINE = Backup(SEKRIT_DBIDENT, + S3('url_dbident', 'ak', 'SEKRIT_SAK')); -- { serverError BAD_ARGUMENTS } + +-- A credential-free locator (Disk, File, Memory, Null) names its destination with literals and holds +-- no credential, but it keeps a named override or nested map that it never reads, and that can carry +-- one. Both the logged text and the not-found message identify the locator, so both must hide it. +CREATE DATABASE db_04510_ftail ENGINE = Backup('src_04510', + File('nonexistent_04510', extra_credentials(external_id = 'SEKRIT_DBFTAIL'))); -- { serverError BACKUP_NOT_FOUND } + +-- A surplus literal argument reaches the same locator through the shape check instead of the tail, and +-- it is rejected only after the statement is logged, so it has to be hidden as well. The two controls +-- that follow hold the argument counts these engines do read (File one, Disk two), where the locator +-- names a destination only: nothing is masked, so both the logged text and the not-found message keep +-- it visible verbatim, and the transcript records the statement as sent rather than a re-formatted AST. +CREATE DATABASE db_04510_fover ENGINE = Backup('src_04510', + File('nonexistent_04510', 'SEKRIT_DBFILEOVER')); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +CREATE DATABASE db_04510_dover ENGINE = Backup('src_04510', + Disk('backups', 'nonexistent_04510', 'SEKRIT_DBDISKOVER')); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +CREATE DATABASE db_04510_mover ENGINE = Backup('src_04510', + Memory('nonexistent_04510', 'SEKRIT_DBMEMOVER')); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH } +CREATE DATABASE db_04510_fvalid ENGINE = Backup('src_04510', File('nonexistent_04510')); -- { serverError BACKUP_NOT_FOUND } +CREATE DATABASE db_04510_dvalid ENGINE = Backup('src_04510', Disk('backups', 'nonexistent_04510')); -- { serverError BACKUP_NOT_FOUND } + +-- A credential value given as an expression is hidden in the logged text, but evaluating it can fail +-- with a message that quotes its input, so the rejection must not carry that message either. The url +-- has to parse for the locator to reach credential evaluation at all. +CREATE DATABASE db_04510_eval ENGINE = Backup('', S3('http://localhost:11111/test/04510eval', 'ak', 'SEKRIT_SAK', + extra_credentials(external_id = toUInt64('SEKRIT_DBEVAL')))); -- { serverError BAD_ARGUMENTS } + +-- A non-tail argument that is neither a literal nor `key = value` cannot be reconstructed either. It +-- is rejected by a different message than the quoted locator above, and that message used to echo the +-- offending argument verbatim, so it needs its own tag. +CREATE DATABASE db_04510_nonlit ENGINE = Backup('', S3(concat('SEKRIT_NONLIT', 'x'), + 'url_dbnonlit')); -- { serverError BAD_ARGUMENTS } + +-- S3 is not the only backup engine whose locator carries credentials: AzureBlobStorage takes an +-- account_key and accepts connection strings and named-collection overrides that carry one too. +-- Only S3 is reconstructed above, so an Azure locator keeps its engine name and argument count +-- (neither is a secret) and every argument is hidden. The url carries a query string, which the +-- engine rejects before it reaches the network. +CREATE DATABASE db_04510_azure ENGINE = Backup('', AzureBlobStorage('http://localhost:11111/acct?sig=x', + 'cont', 'blob', 'account', 'SEKRIT_AZUREKEY')); -- { serverError BAD_ARGUMENTS } + -- The S3 database engine accepts no positional beyond secret_access_key; an extra positional must -- be masked in the logged query text. CREATE DATABASE db_04510_s3pos ENGINE = S3('url_dbs3pos', 'ak', 'SEKRIT_SAK', @@ -323,3 +520,21 @@ WHERE current_database = currentDatabase() AND query NOT ILIKE 'SYSTEM FLUSH%' -- its own terminal event races with the flush it performs AND event_date >= yesterday() AND event_time > now() - INTERVAL 5 MINUTE ORDER BY event_time_microseconds; + +-- The transcript above is scoped to the statements this test issued. A Replicated database also +-- logs each DDL from the replay worker, which re-masks a rewritten AST independently, so assert +-- the masking property over every row this test produced, replay rows included. count() > 0 keeps +-- an empty row set from passing vacuously. +-- The third column covers the recorded exception messages of the seven unreconstructible-locator +-- statements above, whose rejections used to echo the locator verbatim - one tag per message, since +-- they are thrown at different sites. It is scoped to those tags because widening it to every +-- deliberately-failing statement here would report unrelated pre-existing echoes. +SELECT count() > 0, countIf(query LIKE '%SEKRIT%'), + countIf(exception LIKE '%SEKRIT_QUOTED%' OR exception LIKE '%SEKRIT_NONLIT%' + OR exception LIKE '%SEKRIT_DBHDR%' OR exception LIKE '%SEKRIT_DBSWAP%' + OR exception LIKE '%SEKRIT_DBEVAL%' OR exception LIKE '%SEKRIT_DBFTAIL%' + OR exception LIKE '%SEKRIT_DBIDENT%') +FROM system.query_log +WHERE current_database = currentDatabase() + AND type != 'QueryStart' + AND event_date >= yesterday() AND event_time > now() - INTERVAL 5 MINUTE; diff --git a/tests/queries/0_stateless/04512_mutation_temp_dir_cleanup_race.reference b/tests/queries/0_stateless/04512_mutation_temp_dir_cleanup_race.reference new file mode 100644 index 0000000000000..3ea124c3c2635 --- /dev/null +++ b/tests/queries/0_stateless/04512_mutation_temp_dir_cleanup_race.reference @@ -0,0 +1,4 @@ +1 0 +0 +999 +1 diff --git a/tests/queries/0_stateless/04512_mutation_temp_dir_cleanup_race.sh b/tests/queries/0_stateless/04512_mutation_temp_dir_cleanup_race.sh new file mode 100755 index 0000000000000..7fb4467d3effe --- /dev/null +++ b/tests/queries/0_stateless/04512_mutation_temp_dir_cleanup_race.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash +# Tags: zookeeper, no-parallel, no-shared-merge-tree, no-replicated-database +# Tag no-parallel: the failpoints pause every ReplicatedMergeTree mutation on the server +# Tag no-shared-merge-tree: the failpoints are in the ReplicatedMergeTree mutation task +# Tag no-replicated-database: additional replicas execute the same mutation + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# shellcheck source=./mergetree_mutations.lib +. "$CUR_DIR"/mergetree_mutations.lib + +set -e + +# Both failpoints are server-global and pause every `ReplicatedMergeTree` mutation, so they must be +# disabled even when the test fails in the middle: otherwise the next tests on the same server hang. +# Disabling a pauseable failpoint also resumes the threads that are currently paused at it. +function disable_failpoints() +{ + $CLICKHOUSE_CLIENT --query " + SYSTEM DISABLE FAILPOINT rmt_mutate_task_pause_before_rename_part; + SYSTEM DISABLE FAILPOINT rmt_mutate_task_pause_after_temporary_part_released; + " ||: +} + +trap disable_failpoints EXIT + +# Scope all text_log reads below to this run of the test, so that stale rows from a previous run +# (e.g. under a fixed --database, where the logger names are the same) cannot satisfy the checks. +start_time=$($CLICKHOUSE_CLIENT --query "SELECT now64(6)") + +# `temporary_directories_lifetime = 1` makes the cleanup thread consider the temporary directory of +# an in-flight mutation old enough to be removed, which is exactly what the test needs to check. +$CLICKHOUSE_CLIENT --query " + SET insert_keeper_fault_injection_probability = 0; + + CREATE TABLE rmt (num UInt32, num2 UInt32) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/rmt/', '1') + ORDER BY num + SETTINGS min_bytes_for_wide_part = 0, + temporary_directories_lifetime = 1, + cleanup_delay_period = 1, + cleanup_delay_period_random_add = 0, + max_cleanup_delay_period = 1; + + INSERT INTO rmt SELECT number, number + 1 FROM numbers(1000); +" + +CLEANUP_ROWS="FROM system.text_log + WHERE event_time_microseconds >= toDateTime64('$start_time', 6) + AND logger_name LIKE '${CLICKHOUSE_DATABASE}.rmt%' AND message LIKE '%tmp_mut_%' + AND (message LIKE '%is in use (by merge/mutation/INSERT)%' OR message LIKE '%Removing temporary directory%')" + +# Every finished iteration of the cleanup thread logs this message, so it tells the test that the +# cleanup thread has actually looked at the data directory, instead of just waiting for a while. +CLEANUP_ITERATIONS="SELECT count() FROM system.text_log + WHERE event_time_microseconds >= toDateTime64('$start_time', 6) + AND logger_name = '${CLICKHOUSE_DATABASE}.rmt (CleanupThread)' AND message LIKE 'Scheduling next cleanup%'" + +# Wait until the cleanup thread finishes at least $1 more iterations than it had at the moment of the call. +function wait_for_cleanup_iterations() +{ + local wanted=$1 + local before + $CLICKHOUSE_CLIENT --query "SYSTEM FLUSH LOGS text_log" + before=$($CLICKHOUSE_CLIENT --query "$CLEANUP_ITERATIONS") + + for _ in {1..300} + do + sleep 0.3 + $CLICKHOUSE_CLIENT --query "SYSTEM FLUSH LOGS text_log" + if [[ $($CLICKHOUSE_CLIENT --query "$CLEANUP_ITERATIONS") -ge $((before + wanted)) ]] + then + return 0 + fi + done + + echo "The cleanup thread did not run $wanted times" >&2 + return 1 +} + +# Phase 1: pause the mutation right before its temporary part directory is renamed to the persistent +# name. The directory is still owned by the mutation task, so the cleanup thread must skip it. +$CLICKHOUSE_CLIENT --query " + SYSTEM ENABLE FAILPOINT rmt_mutate_task_pause_before_rename_part; + ALTER TABLE rmt RENAME COLUMN num2 TO foo2 SETTINGS alter_sync = 0; +" + +wait_for_mutation_in_progress "rmt" "0000000000" +timeout 120 $CLICKHOUSE_CLIENT --query "SYSTEM WAIT FAILPOINT rmt_mutate_task_pause_before_rename_part PAUSE" + +wait_for_cleanup_iterations 2 + +$CLICKHOUSE_CLIENT --query " + SELECT + countIf(message LIKE '%is in use (by merge/mutation/INSERT)%') > 0 AS kept, + countIf(message LIKE '%Removing temporary directory%') AS removed + $CLEANUP_ROWS; +" + +# Phase 2: this is the window the fix is about. Let the mutation continue up to the point where the +# `TemporaryParts` guard has just been released by `mutate_task.reset()`, and hold it there while the +# cleanup thread makes a few more passes over the data directory. +# +# With the fix, the temporary directory has already been renamed to the persistent name under the +# guard, so there is nothing left for the cleanup thread to remove. With the previous ordering, where +# `transaction_ptr->renameParts()` ran after `mutate_task.reset()`, the directory is still there and +# no longer registered in `TemporaryParts`, so the cleanup thread removes it right from under the +# rename: `removed` below becomes non-zero and the checks at the end of the test fail. +$CLICKHOUSE_CLIENT --query " + SYSTEM ENABLE FAILPOINT rmt_mutate_task_pause_after_temporary_part_released; + SYSTEM DISABLE FAILPOINT rmt_mutate_task_pause_before_rename_part; +" + +timeout 120 $CLICKHOUSE_CLIENT --query "SYSTEM WAIT FAILPOINT rmt_mutate_task_pause_after_temporary_part_released PAUSE" + +wait_for_cleanup_iterations 2 + +$CLICKHOUSE_CLIENT --query " + SELECT countIf(message LIKE '%Removing temporary directory%') AS removed_after_release + $CLEANUP_ROWS; + + SYSTEM DISABLE FAILPOINT rmt_mutate_task_pause_after_temporary_part_released; +" + +wait_for_mutation "rmt" "0000000000" + +# The mutated part must be complete: every file listed in its checksums has to exist on disk. +$CLICKHOUSE_CLIENT --query " + SELECT count() FROM rmt WHERE foo2 % 1000 > 0; + CHECK TABLE rmt SETTINGS check_query_single_value_result = 1; + DROP TABLE rmt SYNC; +" diff --git a/tests/queries/0_stateless/04603_rowbinary_header_too_many_columns.reference b/tests/queries/0_stateless/04603_rowbinary_header_too_many_columns.reference new file mode 100644 index 0000000000000..6cc8f3258bf9e --- /dev/null +++ b/tests/queries/0_stateless/04603_rowbinary_header_too_many_columns.reference @@ -0,0 +1,3 @@ +1 +TOO_LARGE_ARRAY_SIZE +TOO_LARGE_ARRAY_SIZE diff --git a/tests/queries/0_stateless/04603_rowbinary_header_too_many_columns.sh b/tests/queries/0_stateless/04603_rowbinary_header_too_many_columns.sh new file mode 100755 index 0000000000000..627efc72b50a2 --- /dev/null +++ b/tests/queries/0_stateless/04603_rowbinary_header_too_many_columns.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Tests that a RowBinaryWithNames[AndTypes] header with a suspiciously large column +# count is rejected with TOO_LARGE_ARRAY_SIZE instead of amplifying into a huge +# allocation. See https://github.com/ClickHouse/clickhouse-private/issues/69219 + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS t_rowbinary_header" +${CLICKHOUSE_CLIENT} -q "CREATE TABLE t_rowbinary_header (x UInt64) ENGINE = Memory" + +# LEB128 encoder. +varint() { + python3 -c ' +import sys +n = int(sys.argv[1]) +out = bytearray() +while True: + b = n & 0x7F + n >>= 7 + if n: + b |= 0x80 + out.append(b) + if not n: + break +sys.stdout.buffer.write(bytes(out)) +' "$1" +} + +# Negative control: a well-formed one-column header (name "x", type "UInt64") plus one +# 8-byte UInt64 row inserts fine. +{ varint 1; printf '\x01x\x06UInt64'; printf '\x01\x00\x00\x00\x00\x00\x00\x00'; } \ + | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=INSERT+INTO+t_rowbinary_header+FORMAT+RowBinaryWithNamesAndTypes" --data-binary @- +${CLICKHOUSE_CLIENT} -q "SELECT count() FROM t_rowbinary_header" + +# Attack: a header claiming far more columns than the ceiling (1'000'000). +# The tiny body must be rejected up-front, not amplified. +varint 2000000 \ + | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=INSERT+INTO+t_rowbinary_header+FORMAT+RowBinaryWithNamesAndTypes" --data-binary @- 2>&1 \ + | grep -o "TOO_LARGE_ARRAY_SIZE" | head -n1 + +# Same for the WithNames variant (the names header is a separate read path). +varint 2000000 \ + | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=INSERT+INTO+t_rowbinary_header+FORMAT+RowBinaryWithNames" --data-binary @- 2>&1 \ + | grep -o "TOO_LARGE_ARRAY_SIZE" | head -n1 + +${CLICKHOUSE_CLIENT} -q "DROP TABLE t_rowbinary_header" diff --git a/tests/queries/0_stateless/04611_iceberg_parquet_schema_evolution_prewhere_rename.reference b/tests/queries/0_stateless/04611_iceberg_parquet_schema_evolution_prewhere_rename.reference new file mode 100644 index 0000000000000..76b2a858b4b7b --- /dev/null +++ b/tests/queries/0_stateless/04611_iceberg_parquet_schema_evolution_prewhere_rename.reference @@ -0,0 +1,6 @@ +89 +9 +4895 +50 +89 +89 diff --git a/tests/queries/0_stateless/04611_iceberg_parquet_schema_evolution_prewhere_rename.sh b/tests/queries/0_stateless/04611_iceberg_parquet_schema_evolution_prewhere_rename.sh new file mode 100755 index 0000000000000..b060df73530af --- /dev/null +++ b/tests/queries/0_stateless/04611_iceberg_parquet_schema_evolution_prewhere_rename.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel-replicas +# `no-parallel-replicas`: see comment in `04071_iceberg_orc_prewhere_crash.sh`. +# `StorageObjectStorageCluster` (used when `parallel_replicas_for_cluster_engines = 1`, +# default) does not delegate `supportsPrewhere` to its underlying configuration, +# so explicit `PREWHERE` against `icebergLocal` is rejected by the analyzer. +# +# Regression test for the schema-changed branch of the Iceberg PARQUET `PREWHERE` +# path. When an Iceberg table has gone through a column RENAME and then a query +# uses `PREWHERE` on the renamed column against PARQUET data files, the source: +# +# 1) reads the parquet file using a `ColumnMapper` that resolves parquet +# field-ids to the FILE-side (old) clickhouse names (the schema id current +# when the file was written), +# 2) applies a `schema_transform` `ExpressionTransform` that renames the +# file-side names into the QUERY-side names of the current schema. +# +# Parquet supports `PREWHERE`, so before the fix the `PREWHERE` / row-level +# filter were pushed INTO the parquet reader together with that old-schema +# mapper. `PREWHERE` references the CURRENT name (`renamed_c0`) which the +# old-schema mapper does not know, so the reader materialized an absent/default +# column and the predicate matched nothing -> the query silently returned 0 +# rows (a wrong result, not an error). `WHERE` on the same column was correct +# because it runs as a `FilterTransform` after `schema_transform`. +# +# The fix strips `PREWHERE` / row-level filter on the schema-changed Iceberg +# path (regardless of format) and replays them as fallback `FilterTransform`s +# after `schema_transform` aliases the names, exactly like the ORC path. This +# is the PARQUET data-file counterpart of the ORC-only coverage in +# `04147_iceberg_orc_schema_evolution_row_policy.sh`. +# +# Without the fix (parquet data files): +# `SELECT count() FROM t PREWHERE renamed_c0 > 10` -> 0 (should be 89) + +# `input_format_parquet_use_native_reader_v3` gates `PREWHERE` support for `Parquet`, and +# `clickhouse-test` randomizes it to `0`, which makes `IcebergLocal` throw `ILLEGAL_PREWHERE`. +# Pin it to `1`, including in the `CREATE TABLE` session: `supports_prewhere` is computed in +# the `StorageObjectStorage` constructor. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +ICEBERG_PATH="${CLICKHOUSE_USER_FILES}/lakehouses/${CLICKHOUSE_DATABASE}_parquet_evol" +TEST_USER="${CLICKHOUSE_DATABASE}_user_pq_evol" +TEST_POLICY="${CLICKHOUSE_DATABASE}_policy_pq_evol" +TEST_TABLE="t_ice_parquet_evol" + +rm -rf "${ICEBERG_PATH}" + +# Create an Iceberg table with PARQUET data files under its first schema +# (c0 Int64, c1 String, c2 Int32). One data file, 100 rows. +${CLICKHOUSE_CLIENT} --query " + SET allow_experimental_insert_into_iceberg = 1, input_format_parquet_use_native_reader_v3 = 1; + + CREATE TABLE ${TEST_TABLE} (c0 Int64, c1 String, c2 Int32) + ENGINE = IcebergLocal('${ICEBERG_PATH}', 'Parquet'); + + INSERT INTO ${TEST_TABLE} + SELECT number, toString(number), toInt32(number) FROM numbers(100); +" + +# Rename c0 -> renamed_c0. The existing parquet data file still carries the +# column-id under its old name `c0`; the current snapshot has it as +# `renamed_c0`. Reads now go through the schema-changed path. +${CLICKHOUSE_CLIENT} --query " + SET allow_insert_into_iceberg = 1, input_format_parquet_use_native_reader_v3 = 1; + ALTER TABLE ${TEST_TABLE} RENAME COLUMN c0 TO renamed_c0; +" + +# 1) PREWHERE on the renamed column, not in the SELECT list +# (remove_prewhere_column = true). Rows with renamed_c0 in (10..99] -> 89. +${CLICKHOUSE_CLIENT} --query " + SELECT count() FROM ${TEST_TABLE} PREWHERE renamed_c0 > 10 + SETTINGS input_format_parquet_use_native_reader_v3 = 1, enable_analyzer = 1 +" + +# 2) PREWHERE on the renamed column plus a WHERE on another (unrenamed) column +# kept in the SELECT list. Rows with renamed_c0 > 10 AND c1 starting with +# '1' -> {1x} in (10..99]: 12..19 and 100 excluded -> 9. +${CLICKHOUSE_CLIENT} --query " + SELECT count() FROM ${TEST_TABLE} PREWHERE renamed_c0 > 10 WHERE startsWith(c1, '1') + SETTINGS input_format_parquet_use_native_reader_v3 = 1, enable_analyzer = 1 +" + +# 3) Read another column with PREWHERE on the renamed column only (the renamed +# column is dropped after PREWHERE). sum of c1 (as Int) for renamed_c0 in +# (10..99] = sum(11..99) = 4895. +${CLICKHOUSE_CLIENT} --query " + SELECT sum(toInt64(c1)) FROM ${TEST_TABLE} PREWHERE renamed_c0 > 10 + SETTINGS input_format_parquet_use_native_reader_v3 = 1, enable_analyzer = 1 +" + +# 4) Control: PREWHERE on a NON-renamed column (c2) must stay correct. Rows +# with c2 < 50 -> 50. +${CLICKHOUSE_CLIENT} --query " + SELECT count() FROM ${TEST_TABLE} PREWHERE c2 < 50 + SETTINGS input_format_parquet_use_native_reader_v3 = 1, enable_analyzer = 1 +" + +# 5) Same predicate as (1) but via WHERE — always worked (fallback filter runs +# after schema_transform). Kept as an oracle: PREWHERE must equal WHERE. 89. +${CLICKHOUSE_CLIENT} --query " + SELECT count() FROM ${TEST_TABLE} WHERE renamed_c0 > 10 + SETTINGS input_format_parquet_use_native_reader_v3 = 1, enable_analyzer = 1 +" + +# 6) Row policy on the renamed column, combined with PREWHERE on it. The policy +# keeps renamed_c0 > 5; PREWHERE keeps renamed_c0 > 10; intersection is +# (10..99] -> 89. Exercises row_level_filter on the schema-changed parquet +# path (same reader-side name-resolution as PREWHERE). +${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${TEST_USER}" +${CLICKHOUSE_CLIENT} --query "CREATE USER ${TEST_USER} IDENTIFIED WITH plaintext_password BY 'pq_pwd_evol'" +${CLICKHOUSE_CLIENT} --query "GRANT SELECT ON *.* TO ${TEST_USER}" +${CLICKHOUSE_CLIENT} --query "DROP ROW POLICY IF EXISTS ${TEST_POLICY} ON ${TEST_TABLE}" +${CLICKHOUSE_CLIENT} --query "CREATE ROW POLICY ${TEST_POLICY} ON ${TEST_TABLE} FOR SELECT USING renamed_c0 > 5 TO ${TEST_USER}" + +${CLICKHOUSE_CLIENT} --user="${TEST_USER}" --password=pq_pwd_evol --query " + SELECT count() FROM ${TEST_TABLE} PREWHERE renamed_c0 > 10 + SETTINGS input_format_parquet_use_native_reader_v3 = 1, enable_analyzer = 1 +" + +# Cleanup +${CLICKHOUSE_CLIENT} --query "DROP ROW POLICY IF EXISTS ${TEST_POLICY} ON ${TEST_TABLE}" +${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${TEST_USER}" +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${TEST_TABLE}" +rm -rf "${ICEBERG_PATH}" diff --git a/tests/queries/0_stateless/04614_group_array_state_resample_finalize_no_crash.reference b/tests/queries/0_stateless/04614_group_array_state_resample_finalize_no_crash.reference new file mode 100644 index 0000000000000..1b0480cab8af0 --- /dev/null +++ b/tests/queries/0_stateless/04614_group_array_state_resample_finalize_no_crash.reference @@ -0,0 +1,5 @@ +0 1000 0 +1 1000 1 +[[1000],[1000]] +[2000,2000] +[2000] diff --git a/tests/queries/0_stateless/04614_group_array_state_resample_finalize_no_crash.sql b/tests/queries/0_stateless/04614_group_array_state_resample_finalize_no_crash.sql new file mode 100644 index 0000000000000..2fe507188483b --- /dev/null +++ b/tests/queries/0_stateless/04614_group_array_state_resample_finalize_no_crash.sql @@ -0,0 +1,38 @@ +-- Tags: no-parallel +-- no-parallel -- enables a process-global failpoint that fires in every -State finalization + +-- A combinator that transfers several sub-states in one insertResultInto call aliases them into the +-- result column one by one. When the transfer throws part-way through, the aggregator destroys the +-- whole place while the result column still owns the already-transferred sub-states, so they are +-- freed twice. The failpoint throws from the -State transfer once the destination column already +-- holds an aliased state, which is exactly that partial transfer. +-- Each sub-state below holds more than 4096 bytes of groupArray data, so it is allocated outside the +-- arena and the second destroy reaches a real deallocation. + +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; + +-- Plain -State: the transfer is atomic, so this must throw cleanly on every version. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT number % 2 AS k, groupArrayState(number) FROM numbers(2000) GROUP BY k ORDER BY k SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT k, length(finalizeAggregation(s)), finalizeAggregation(s)[1] FROM (SELECT number % 2 AS k, groupArrayState(number) AS s FROM numbers(2000) GROUP BY k) ORDER BY k SETTINGS max_threads = 1; + +-- -Resample over -ForEach, one element per bucket: the throw is in the second bucket, so -Resample +-- has to undo a completed -ForEach child. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT groupArrayStateForEachResample(0, 2, 1)([number], number % 2) FROM numbers(2000) SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT arrayMap(x -> arrayMap(y -> length(finalizeAggregation(y)), x), groupArrayStateForEachResample(0, 2, 1)([number], number % 2)) FROM numbers(2000) SETTINGS max_threads = 1; + +-- -ForEach alone: the throw is in its own loop, with the first element already aliased. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT groupArrayStateForEach([number, number + 1]) FROM numbers(2000) SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT arrayMap(x -> length(finalizeAggregation(x)), groupArrayStateForEach([number, number + 1])) FROM numbers(2000) SETTINGS max_threads = 1; + +-- Exactly one bucket: the -State transfer cannot throw (the column is still empty), so the throw +-- comes from -Resample appending its own offset with the single sub-state already aliased. +SYSTEM ENABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT groupArrayStateResample(0, 1, 1)(number, 0) FROM numbers(2000) SETTINGS max_threads = 1 FORMAT Null; -- { serverError MEMORY_LIMIT_EXCEEDED } +SYSTEM DISABLE FAILPOINT aggregate_function_state_transfer_throw; +SELECT arrayMap(x -> length(finalizeAggregation(x)), groupArrayStateResample(0, 1, 1)(number, 0)) FROM numbers(2000) SETTINGS max_threads = 1; diff --git a/tests/queries/0_stateless/04626_create_or_replace_drop_privilege_preflight.reference b/tests/queries/0_stateless/04626_create_or_replace_drop_privilege_preflight.reference new file mode 100644 index 0000000000000..0b9bc73749de4 --- /dev/null +++ b/tests/queries/0_stateless/04626_create_or_replace_drop_privilege_preflight.reference @@ -0,0 +1,7 @@ +-- [scoped DROP DICTIONARY grant] replacing the dictionary with a view must succeed: +42 +-- [no DROP DICTIONARY grant] the replace must be denied: +ACCESS_DENIED +-- [no DROP DICTIONARY grant] and the denied replace must leave the dictionary intact and no leftovers: +1 +0 diff --git a/tests/queries/0_stateless/04626_create_or_replace_drop_privilege_preflight.sh b/tests/queries/0_stateless/04626_create_or_replace_drop_privilege_preflight.sh new file mode 100755 index 0000000000000..125c26b20a49d --- /dev/null +++ b/tests/queries/0_stateless/04626_create_or_replace_drop_privilege_preflight.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# `CREATE OR REPLACE` drops the replaced table after the swap, under an internal `_tmp_replace_*` name. +# The drop privilege for the replaced table's kind must be enforced before the swap: a denied query +# must leave the replaced table intact instead of failing after the replace is already committed. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +granted="granted_${CLICKHOUSE_TEST_UNIQUE_NAME}" +nogrant="nogrant_${CLICKHOUSE_TEST_UNIQUE_NAME}" +db=${CLICKHOUSE_DATABASE} + +${CLICKHOUSE_CLIENT} --query " +CREATE TABLE src (key String, value UInt64) ENGINE = MergeTree ORDER BY key; +INSERT INTO src VALUES ('k1', 1); +CREATE DICTIONARY dict_granted (key String, value UInt64) PRIMARY KEY key SOURCE(CLICKHOUSE(TABLE 'src')) LAYOUT(DIRECT()); +CREATE DICTIONARY dict_denied (key String, value UInt64) PRIMARY KEY key SOURCE(CLICKHOUSE(TABLE 'src')) LAYOUT(DIRECT()); +DROP USER IF EXISTS ${granted}, ${nogrant}; +CREATE USER ${granted} IDENTIFIED WITH plaintext_password BY '${granted}'; +CREATE USER ${nogrant} IDENTIFIED WITH plaintext_password BY '${nogrant}'; +GRANT SELECT, INSERT, CREATE TABLE, DROP TABLE, CREATE VIEW, DROP VIEW ON ${db}.* TO ${granted}, ${nogrant}; +-- The drop grant scoped to the replaced dictionary's real name must be sufficient. +GRANT DROP DICTIONARY ON ${db}.dict_granted TO ${granted}; +" + +echo "-- [scoped DROP DICTIONARY grant] replacing the dictionary with a view must succeed:" +${CLICKHOUSE_CLIENT} --user "${granted}" --password "${granted}" --query "CREATE OR REPLACE VIEW ${db}.dict_granted AS SELECT 42 AS x" +${CLICKHOUSE_CLIENT} --query "SELECT x FROM dict_granted" + +echo "-- [no DROP DICTIONARY grant] the replace must be denied:" +${CLICKHOUSE_CLIENT} --user "${nogrant}" --password "${nogrant}" --query "CREATE OR REPLACE VIEW ${db}.dict_denied AS SELECT 42 AS x" 2>&1 | grep -Fo ACCESS_DENIED | uniq + +echo "-- [no DROP DICTIONARY grant] and the denied replace must leave the dictionary intact and no leftovers:" +${CLICKHOUSE_CLIENT} --query " +SELECT dictGet(dict_denied, 'value', 'k1'); +SELECT count() FROM system.tables WHERE database = '${db}' AND startsWith(name, '_tmp_replace_'); +" + +${CLICKHOUSE_CLIENT} --query "DROP USER ${granted}, ${nogrant}" diff --git a/tests/queries/0_stateless/04647_no_blocks_marshalling_for_local_shard_plan.reference b/tests/queries/0_stateless/04647_no_blocks_marshalling_for_local_shard_plan.reference new file mode 100644 index 0000000000000..2f9eca3ebd44d --- /dev/null +++ b/tests/queries/0_stateless/04647_no_blocks_marshalling_for_local_shard_plan.reference @@ -0,0 +1,24 @@ +-- local shard plan, several grouping sets: with and without marshalling +1007 10083376534861865471 +1007 10083376534861865471 +-- parallel replicas local plan, several grouping sets: with and without marshalling +1007 16998005863169288206 +1007 16998005863169288206 +-- single grouping set +7 13874850128157539051 +-- initial query (control) +1007 10083376534861865471 +-- Merge table over a local and a distributed table +2000 11251509125014382908 +2000 11251509125014382908 +-- Buffer table whose destination is distributed +1007 10083376534861865471 +1007 10083376534861865471 +-- BlocksMarshalling steps for remote(): total, above ReadFromRemote, ReadFromRemote steps +1 0 1 +-- BlocksMarshalling steps for parallel replicas: total, above the remote step, remote steps +2 0 1 +-- BlocksMarshalling steps for a Buffer table: total, buffer-reading steps +2 2 +-- BlocksMarshalling steps for a serialized plan: total, above ReadFromRemote, ReadFromRemote steps +1 0 1 diff --git a/tests/queries/0_stateless/04647_no_blocks_marshalling_for_local_shard_plan.sh b/tests/queries/0_stateless/04647_no_blocks_marshalling_for_local_shard_plan.sh new file mode 100755 index 0000000000000..68c9f92fd010e --- /dev/null +++ b/tests/queries/0_stateless/04647_no_blocks_marshalling_for_local_shard_plan.sh @@ -0,0 +1,191 @@ +#!/usr/bin/env bash +# Tags: shard + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `--query_kind=secondary_query` makes the server treat the query as a shard-side one, which is the +# only state in which the marshalling gate in `Planner::buildPlanForQueryNode` fires. It applies to +# every statement of a multi-statement invocation, so batching below does not weaken it. +# `--compression 1` is required as well: without a network codec the marshalling callback returns +# the block unchanged, so nothing is wrapped into `ColumnBLOB`. +# `enable_analyzer` is pinned because `BlocksMarshallingStep` is only ever added by the analyzer, +# and CI runs variants with `allow_experimental_analyzer = 0`. +# `serialize_query_plan` is pinned to 0 for the queries that are executed: a plan serialized and +# shipped to a shard legitimately still carries `BlocksMarshalling`, which is not registered in +# `QueryPlanStepRegistry` on master, so executing one fails with `Unknown query plan step: +# BlocksMarshalling` independently of this fix. The `distributed plan` CI variants set +# `serialize_query_plan = 1` in the default profile. +CLIENT="$CLICKHOUSE_CLIENT --enable_analyzer 1 --compression 1 --serialize_query_plan 0 --query_kind=secondary_query" +INITIAL_CLIENT="$CLICKHOUSE_CLIENT --enable_analyzer 1 --compression 1 --serialize_query_plan 0" + +# Statements are grouped into as few client invocations as possible, and the section headers are +# emitted as `SELECT` constants inside those batches instead of by the shell. None of the queries +# here is heavy: the runtime of this test is dominated by starting client processes, which costs +# about five times more under a sanitizer, and the flaky check runs every test many times +# concurrently. Splitting these statements back into one invocation each pushes the test over the +# 180s flaky-check limit on the ASan lane. +MARSHALLING_ON="enable_parallel_blocks_marshalling = 1, prefer_localhost_replica = 1" +MARSHALLING_OFF="enable_parallel_blocks_marshalling = 0, prefer_localhost_replica = 1" +# Mode 2 throws instead of silently falling back to plain local execution, so an unavailable +# cluster fails the test instead of making the parallel replicas cases assert nothing. +PARALLEL_REPLICAS="automatic_parallel_replicas_mode = 0, enable_parallel_replicas = 2, parallel_replicas_local_plan = 1, max_parallel_replicas = 3, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1" + +$INITIAL_CLIENT -q " + CREATE TABLE tab (x UInt64, y UInt64, z UInt64) ENGINE = MergeTree ORDER BY x; + INSERT INTO tab SELECT number % 7, number, number FROM numbers(1000); +" + +# Counts of `BlocksMarshalling` steps in a distributed plan: the total, how many appear above the +# step named in $2 that reads from another replica, and how many such reading steps there are. +# Asserted as counts, not as a plan diff, so unrelated plan formatting changes do not break it. +# Only prints the query, so that several of these can share one client invocation. +# `rowNumberInAllBlocks` restarts for every statement, so batching does not affect the counts. +marshalling_counts_query() { + echo " + SELECT + countIf(explain ILIKE '%BlocksMarshalling%'), + countIf(explain ILIKE '%BlocksMarshalling%' AND rn < remote_rn), + countIf(explain ILIKE '%$2%') + FROM ( + SELECT + explain, + rowNumberInAllBlocks() AS rn, + min(if(explain ILIKE '%$2%', rn, NULL)) OVER () AS remote_rn + FROM ( EXPLAIN distributed = 1 $1 ) + )" +} + +# `GROUPING SETS` with 2+ sets, so `MergingAggregatedTransform::addChunk` type-checks +# `__grouping_set` instead of returning early. A `ColumnBLOB` reaching it used to throw +# `Expected UInt64 column for __grouping_set, got BLOB`. +GROUPING_SETS_QUERY=" + SELECT count(), sum(cityHash64(x, s)) FROM ( + SELECT x, sum(y) AS s FROM remote('127.0.0.{1,2}', currentDatabase(), tab) + GROUP BY GROUPING SETS ((x, z % 1), (intDiv(z, 100), z + 2)) + )" + +PARALLEL_REPLICAS_QUERY=" + SELECT count(), sum(cityHash64(x, s)) FROM ( + SELECT x, sum(y) AS s FROM tab + GROUP BY GROUPING SETS ((x, z % 1), (intDiv(z, 100), z + 2)) + )" + +# All of the shard-side result assertions. A single grouping set takes the early-return branch of +# `addChunk` and never type-checked the column, so it worked even with a `ColumnBLOB`; it is pinned +# so the branch the fix does not target cannot regress either. +$CLIENT -q " + SELECT '-- local shard plan, several grouping sets: with and without marshalling'; + $GROUPING_SETS_QUERY SETTINGS $MARSHALLING_ON; + $GROUPING_SETS_QUERY SETTINGS $MARSHALLING_OFF; + + SELECT '-- parallel replicas local plan, several grouping sets: with and without marshalling'; + $PARALLEL_REPLICAS_QUERY SETTINGS enable_parallel_blocks_marshalling = 1, $PARALLEL_REPLICAS; + $PARALLEL_REPLICAS_QUERY SETTINGS enable_parallel_blocks_marshalling = 0, $PARALLEL_REPLICAS; + + SELECT '-- single grouping set'; + SELECT count(), sum(cityHash64(x, s)) FROM ( + SELECT x, sum(y) AS s FROM remote('127.0.0.{1,2}', currentDatabase(), tab) + GROUP BY GROUPING SETS ((x)) + ) SETTINGS $MARSHALLING_ON; +" + +BUFFER_QUERY=" + SELECT count(), sum(cityHash64(x, s)) FROM ( + SELECT x, sum(y) AS s FROM remote('127.0.0.{1,2}', currentDatabase(), buf_all) + GROUP BY GROUPING SETS ((x, z % 1), (intDiv(z, 100), z + 2)) + )" + +# Everything that must run as an ordinary (non-secondary) query. +# +# CONTROL: an ordinary query never marshals and passes without the fix too, so the fix must not +# change it. Do not mistake it for one of the assertions above. +# +# A `Merge` table builds a child plan per underlying table with its own `SelectQueryOptions`, and +# unites them in this process with no unmarshalling. With a marshalling producer in the set the +# stage of the `MergeTree` child degrades, so it is planned by the analyzer and used to get its own +# marshalling step, which then reached `ExpressionActions::executeOnColumns` as a `ColumnBLOB`. +# +# A `Buffer` table plans the reads of its in-memory buffers with their own `SelectQueryOptions` and +# unites them with the destination plan in this process, again with nothing unmarshalling them. The +# buffers are only planned at all above `FetchColumns`, and a `Buffer` inherits its stage from the +# destination table, so the destination has to be a `Distributed` one. Rows must still be in the +# buffer at read time, hence the large flush thresholds. +# +# The `Merge` and `Buffer` cases run here on purpose rather than through \$CLIENT: their secondary +# context comes from the inner `remote(...)` hop. +$INITIAL_CLIENT -q " + SELECT '-- initial query (control)'; + $GROUPING_SETS_QUERY SETTINGS $MARSHALLING_ON; + + CREATE TABLE mrg_local (x UInt64) ENGINE = MergeTree ORDER BY x; + INSERT INTO mrg_local SELECT number FROM numbers(1000); + CREATE TABLE mrg_dist (x UInt64) ENGINE = Distributed(test_shard_localhost, currentDatabase(), mrg_local); + CREATE TABLE mrg_all (x UInt64) ENGINE = Merge(currentDatabase(), '^mrg_(local|dist)\$'); + + SELECT '-- Merge table over a local and a distributed table'; + SELECT count(), sum(cityHash64(x)) FROM remote('127.0.0.1', currentDatabase(), mrg_all) + SETTINGS enable_parallel_blocks_marshalling = 1, prefer_localhost_replica = 0; + SELECT count(), sum(cityHash64(x)) FROM remote('127.0.0.1', currentDatabase(), mrg_all) + SETTINGS enable_parallel_blocks_marshalling = 0, prefer_localhost_replica = 0; + + CREATE TABLE buf_local (x UInt64, y UInt64, z UInt64) ENGINE = MergeTree ORDER BY x; + INSERT INTO buf_local SELECT number % 7, number, number FROM numbers(500); + CREATE TABLE buf_dist (x UInt64, y UInt64, z UInt64) + ENGINE = Distributed(test_shard_localhost, currentDatabase(), buf_local); + CREATE TABLE buf_all (x UInt64, y UInt64, z UInt64) + ENGINE = Buffer(currentDatabase(), buf_dist, 1, 10000, 10000, 1000000, 1000000, 100000000, 1000000000); + INSERT INTO buf_all SELECT number % 7, number, number FROM numbers(500, 500); + + SELECT '-- Buffer table whose destination is distributed'; + $BUFFER_QUERY SETTINGS enable_parallel_blocks_marshalling = 1, prefer_localhost_replica = 0; + $BUFFER_QUERY SETTINGS enable_parallel_blocks_marshalling = 0, prefer_localhost_replica = 0; +" + +# The plan shape is the invariant being fixed: the locally executed branch must carry no +# `BlocksMarshalling` step, while the branch read from a remote replica must keep it. +# +# The last one asserts that a plan which is serialized and shipped to a shard keeps marshalling: +# the shard sends its blocks back over the network. EXPLAIN only, executing such a plan fails with +# `Unknown query plan step: BlocksMarshalling` both with and without this fix. +$CLIENT -q " + SELECT '-- BlocksMarshalling steps for remote(): total, above ReadFromRemote, ReadFromRemote steps'; + $(marshalling_counts_query " + SELECT x, sum(y) FROM remote('127.0.0.{1,2}', currentDatabase(), tab) + GROUP BY GROUPING SETS ((x, z % 1), (intDiv(z, 100), z + 2)) + SETTINGS $MARSHALLING_ON, serialize_query_plan = 0" "ReadFromRemote"); + + SELECT '-- BlocksMarshalling steps for parallel replicas: total, above the remote step, remote steps'; + $(marshalling_counts_query " + SELECT x, sum(y) FROM tab + GROUP BY GROUPING SETS ((x, z % 1), (intDiv(z, 100), z + 2)) + SETTINGS enable_parallel_blocks_marshalling = 1, $PARALLEL_REPLICAS, serialize_query_plan = 0" "ReadFromRemoteParallelReplicas"); +" + +# The same plan-shape invariant for the `Buffer` carrier, so that the assignment on the buffers +# options is pinned structurally and not only through the result above. Each of the two replicas +# contributes one `BlocksMarshalling` step for the blocks it really sends back; without the fix the +# in-process buffers branch of each of them gets one of its own too, so the total doubles. The count +# of buffer-reading steps is asserted as well, so the query cannot silently stop reading the buffers +# and make the first count pass for the wrong reason. +$INITIAL_CLIENT -q " + SELECT '-- BlocksMarshalling steps for a Buffer table: total, buffer-reading steps'; + SELECT + countIf(explain ILIKE '%BlocksMarshalling%'), + countIf(explain ILIKE '%Read from buffers%' OR explain ILIKE '%ReadFromStorage (Values)%') + FROM ( + EXPLAIN distributed = 1 + SELECT x, sum(y) FROM remote('127.0.0.{1,2}', currentDatabase(), buf_all) + GROUP BY GROUPING SETS ((x, z % 1), (intDiv(z, 100), z + 2)) + SETTINGS enable_parallel_blocks_marshalling = 1, prefer_localhost_replica = 0, serialize_query_plan = 0 + ); +" + +$CLIENT -q " + SELECT '-- BlocksMarshalling steps for a serialized plan: total, above ReadFromRemote, ReadFromRemote steps'; + $(marshalling_counts_query " + SELECT x, sum(y) FROM remote('127.0.0.{1,2}', currentDatabase(), tab) + GROUP BY GROUPING SETS ((x, z % 1), (intDiv(z, 100), z + 2)) + SETTINGS $MARSHALLING_ON, serialize_query_plan = 1" "ReadFromRemote"); +" diff --git a/tests/queries/0_stateless/04651_system_sync_transaction_log_access.reference b/tests/queries/0_stateless/04651_system_sync_transaction_log_access.reference new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/queries/0_stateless/04651_system_sync_transaction_log_access.sql b/tests/queries/0_stateless/04651_system_sync_transaction_log_access.sql new file mode 100644 index 0000000000000..44814f2c966e6 --- /dev/null +++ b/tests/queries/0_stateless/04651_system_sync_transaction_log_access.sql @@ -0,0 +1,20 @@ +-- Tags: no-parallel, zookeeper +-- zookeeper: SYSTEM SYNC TRANSACTION LOG reads the transaction log from Keeper after access is granted. +-- no-parallel: creates a temporary user; avoid concurrent GRANT/DROP races on the same name pattern. + +DROP USER IF EXISTS user_test_04651; +CREATE USER user_test_04651; + +-- Before the fix, local SYSTEM SYNC TRANSACTION LOG never called checkAccess, so a user +-- without SYSTEM SYNC TRANSACTION LOG could still run the command (only the feature flag +-- allow_experimental_transactions was checked). ON CLUSTER already required the privilege. +-- Without the privilege, expect ACCESS_DENIED (checked before the transactions feature flag). +EXECUTE AS user_test_04651 SYSTEM SYNC TRANSACTION LOG; -- { serverError ACCESS_DENIED } + +-- Grant the privilege that ON CLUSTER already required. +GRANT SYSTEM SYNC TRANSACTION LOG ON *.* TO user_test_04651; + +-- With the privilege, the command must get past access control and sync the transaction log. +EXECUTE AS user_test_04651 SYSTEM SYNC TRANSACTION LOG; + +DROP USER user_test_04651; diff --git a/tests/queries/0_stateless/04652_nullable_tuple_element_subcolumn_of_dynamic_variant.reference b/tests/queries/0_stateless/04652_nullable_tuple_element_subcolumn_of_dynamic_variant.reference new file mode 100644 index 0000000000000..8414b98d4c2a1 --- /dev/null +++ b/tests/queries/0_stateless/04652_nullable_tuple_element_subcolumn_of_dynamic_variant.reference @@ -0,0 +1,69 @@ +Nullable(UInt32) +1 1 +2 \N +3 \N +1 s +2 s +3 +1 0 +2 1 +3 1 +1 +1 +1 1 +2 \N +3 \N +1 1 +2 \N +3 \N +Nullable(UInt32) +1 1 +2 \N +3 \N +2 +\N +\N +\N +LowCardinality(Nullable(String)) \N +LowCardinality(Nullable(String)) x +LowCardinality(Nullable(String)) \N +LowCardinality(Nullable(String)) \N +\N +\N +1 +\N +\N +\N +\N +1 +Nullable(UInt32) 5 +Nullable(UInt32) \N +Nullable(UInt32) \N +Nullable(UInt32) \N +Nullable(UInt32) \N +UInt32 0 +UInt32 1 +UInt32 0 +UInt32 0 +UInt32 0 +[] +[] +[1] +[] +[] +{} +{} +{} +{'k':1} +{} +5 \N +\N \N +\N \N +\N \N +\N 1 +Nullable(UInt32) 1 +1 +Nullable(UInt32) +1 1 +2 \N +3 \N diff --git a/tests/queries/0_stateless/04652_nullable_tuple_element_subcolumn_of_dynamic_variant.sql b/tests/queries/0_stateless/04652_nullable_tuple_element_subcolumn_of_dynamic_variant.sql new file mode 100644 index 0000000000000..0c7ca9368ad59 --- /dev/null +++ b/tests/queries/0_stateless/04652_nullable_tuple_element_subcolumn_of_dynamic_variant.sql @@ -0,0 +1,120 @@ +-- Reading a subcolumn whose own type is Nullable out of a Variant/Dynamic column stored in MergeTree. + +SET enable_variant_type = 1; + +DROP TABLE IF EXISTS t_dyn; +DROP TABLE IF EXISTS t_dyn_wide; +DROP TABLE IF EXISTS t_var; +DROP TABLE IF EXISTS t_mem; +DROP TABLE IF EXISTS t_json; +DROP TABLE IF EXISTS t_json_var; + +-- Part thresholds are pinned everywhere below so that the compact and wide readers, which resolve +-- the subcolumn through different code, are both covered on every run: t_dyn_wide is the only wide +-- table, all the others are compact regardless of what the runner randomizes. + +-- The requested subcolumn is a Nullable tuple element. +CREATE TABLE t_dyn (id UInt64, value Dynamic) ENGINE = MergeTree ORDER BY id + SETTINGS min_bytes_for_wide_part = 1000000000, min_rows_for_wide_part = 1000000000; +INSERT INTO t_dyn VALUES (1, CAST(tuple(CAST(1, 'Nullable(UInt32)'), 's'), 'Tuple(a Nullable(UInt32), b String)')); +INSERT INTO t_dyn VALUES (2, CAST(tuple(CAST(NULL, 'Nullable(UInt32)'), 's'), 'Tuple(a Nullable(UInt32), b String)')); +INSERT INTO t_dyn VALUES (3, 'not a tuple'); + +SELECT toTypeName(value.`Tuple(a Nullable(UInt32), b String)`.a) FROM t_dyn LIMIT 1; +-- Element value, element NULL, and a row of another variant. +SELECT id, value.`Tuple(a Nullable(UInt32), b String)`.a FROM t_dyn ORDER BY id; +SELECT id, value.`Tuple(a Nullable(UInt32), b String)`.b FROM t_dyn ORDER BY id; +-- The null map subcolumn of the extracted subcolumn. +SELECT id, value.`Tuple(a Nullable(UInt32), b String)`.a.null FROM t_dyn ORDER BY id; +SELECT id FROM t_dyn PREWHERE value.`Tuple(a Nullable(UInt32), b String)`.a = 1; +SELECT sum(value.`Tuple(a Nullable(UInt32), b String)`.a) FROM t_dyn; + +-- Same in a wide part. +CREATE TABLE t_dyn_wide (id UInt64, value Dynamic) ENGINE = MergeTree ORDER BY id + SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0; +INSERT INTO t_dyn_wide SELECT * FROM t_dyn; +SELECT id, value.`Tuple(a Nullable(UInt32), b String)`.a FROM t_dyn_wide ORDER BY id; + +-- Same values from a Memory table, as an oracle for the two queries above. +CREATE TABLE t_mem (id UInt64, value Dynamic) ENGINE = Memory; +INSERT INTO t_mem SELECT * FROM t_dyn; +SELECT id, value.`Tuple(a Nullable(UInt32), b String)`.a FROM t_mem ORDER BY id; + +-- Variant shares the same serialization, so it must be fixed too. +CREATE TABLE t_var (id UInt64, value Variant(Tuple(a Nullable(UInt32), b String), String)) ENGINE = MergeTree ORDER BY id + SETTINGS min_bytes_for_wide_part = 1000000000, min_rows_for_wide_part = 1000000000; +INSERT INTO t_var VALUES (1, CAST(tuple(CAST(1, 'Nullable(UInt32)'), 's'), 'Tuple(a Nullable(UInt32), b String)')); +INSERT INTO t_var VALUES (2, CAST(tuple(CAST(NULL, 'Nullable(UInt32)'), 's'), 'Tuple(a Nullable(UInt32), b String)')); +INSERT INTO t_var VALUES (3, 'not a tuple'); +SELECT toTypeName(value.`Tuple(a Nullable(UInt32), b String)`.a) FROM t_var LIMIT 1; +SELECT id, value.`Tuple(a Nullable(UInt32), b String)`.a FROM t_var ORDER BY id; + +-- Other shapes of an intrinsically nullable requested subcolumn. +DROP TABLE IF EXISTS t_shapes; +CREATE TABLE t_shapes (id UInt64, value Dynamic) ENGINE = MergeTree ORDER BY id + SETTINGS min_bytes_for_wide_part = 1000000000, min_rows_for_wide_part = 1000000000; +INSERT INTO t_shapes VALUES (1, CAST(tuple(CAST(1, 'Nullable(UInt32)'), CAST(2, 'Nullable(UInt32)')), 'Tuple(a Nullable(UInt32), b Nullable(UInt32))')); +INSERT INTO t_shapes VALUES (2, CAST(tuple(CAST('x', 'LowCardinality(Nullable(String))'), 's'), 'Tuple(a LowCardinality(Nullable(String)), b String)')); +INSERT INTO t_shapes VALUES (3, CAST(tuple(tuple(CAST(1, 'Nullable(UInt32)'))), 'Tuple(a Tuple(x Nullable(UInt32)))')); +INSERT INTO t_shapes VALUES (4, CAST(tuple(CAST(1, 'Nullable(UInt32)'), 's'), 'Tuple(Nullable(UInt32), String)')); +SELECT value.`Tuple(a Nullable(UInt32), b Nullable(UInt32))`.b FROM t_shapes ORDER BY id; +SELECT toTypeName(value.`Tuple(a LowCardinality(Nullable(String)), b String)`.a), + value.`Tuple(a LowCardinality(Nullable(String)), b String)`.a FROM t_shapes ORDER BY id; +SELECT value.`Tuple(a Tuple(x Nullable(UInt32)))`.a.x FROM t_shapes ORDER BY id; +SELECT value.`Tuple(Nullable(UInt32), String)`.1 FROM t_shapes ORDER BY id; + +-- Controls. The nullability added by the extraction must still be removed: +-- here the on-disk element is UInt32 while the subcolumn is exposed as Nullable(UInt32). +DROP TABLE IF EXISTS t_controls; +CREATE TABLE t_controls (id UInt64, value Dynamic) ENGINE = MergeTree ORDER BY id + SETTINGS min_bytes_for_wide_part = 1000000000, min_rows_for_wide_part = 1000000000; +INSERT INTO t_controls VALUES (1, CAST(5, 'UInt32')); +INSERT INTO t_controls VALUES (2, CAST(tuple(CAST(1, 'UInt32'), 's'), 'Tuple(a UInt32, b String)')); +INSERT INTO t_controls VALUES (3, CAST(tuple([CAST(1, 'Nullable(UInt32)')]), 'Tuple(a Array(Nullable(UInt32)))')); +INSERT INTO t_controls VALUES (4, CAST(tuple(map('k', CAST(1, 'Nullable(UInt32)'))), 'Tuple(a Map(String, Nullable(UInt32)))')); +-- Row 5 makes Tuple(a Nullable(UInt32), b String) a real alternative of this column, so the paired +-- query below resolves it as a Variant element instead of falling back to the shared variant. +INSERT INTO t_controls VALUES (5, CAST(tuple(CAST(1, 'Nullable(UInt32)'), 's'), 'Tuple(a Nullable(UInt32), b String)')); +SELECT toTypeName(value.UInt32), value.UInt32 FROM t_controls ORDER BY id; +SELECT toTypeName(value.`Tuple(a UInt32, b String)`.a), value.`Tuple(a UInt32, b String)`.a FROM t_controls ORDER BY id; +SELECT value.`Tuple(a Array(Nullable(UInt32)))`.a FROM t_controls ORDER BY id; +SELECT value.`Tuple(a Map(String, Nullable(UInt32)))`.a FROM t_controls ORDER BY id; +-- Reads an intrinsically nullable and an extraction-wrapped subcolumn of the same column in one +-- query, so both serializations are built and used for the same read. +SELECT value.UInt32, value.`Tuple(a Nullable(UInt32), b String)`.a FROM t_controls ORDER BY id; + +-- The shared variant branch always reads into a Nullable column, so its unwrap stays unconditional. +DROP TABLE IF EXISTS t_shared; +CREATE TABLE t_shared (id UInt64, value Dynamic(max_types = 0)) ENGINE = MergeTree ORDER BY id + SETTINGS min_bytes_for_wide_part = 1000000000, min_rows_for_wide_part = 1000000000; +INSERT INTO t_shared VALUES (1, CAST(tuple(CAST(1, 'Nullable(UInt32)'), 's'), 'Tuple(a Nullable(UInt32), b String)')); +SELECT toTypeName(value.`Tuple(a Nullable(UInt32), b String)`.a), + value.`Tuple(a Nullable(UInt32), b String)`.a FROM t_shared; + +-- A JSON dynamic path with a non-nullable type hint: the nullability is added by the extraction, +-- so this is a control for the same JSON route the carrier below uses. +CREATE TABLE t_json (id UInt64, value JSON) ENGINE = MergeTree ORDER BY id + SETTINGS min_bytes_for_wide_part = 1000000000, min_rows_for_wide_part = 1000000000; +INSERT INTO t_json VALUES (1, '{"k": 1}'); +SELECT value.k.:Int64 FROM t_json; + +-- A JSON typed path whose type is a Variant carries the bug too: the tuple element requested +-- through it is intrinsically Nullable. +CREATE TABLE t_json_var (id UInt64, value JSON(t Variant(Tuple(a Nullable(UInt32), b String), String))) + ENGINE = MergeTree ORDER BY id + SETTINGS min_bytes_for_wide_part = 1000000000, min_rows_for_wide_part = 1000000000; +INSERT INTO t_json_var VALUES (1, '{"t": {"a": 1, "b": "s"}}'); +INSERT INTO t_json_var VALUES (2, '{"t": {"a": null, "b": "s"}}'); +INSERT INTO t_json_var VALUES (3, '{"t": "not a tuple"}'); +SELECT toTypeName(value.t.`Tuple(a Nullable(UInt32), b String)`.a) FROM t_json_var LIMIT 1; +SELECT id, value.t.`Tuple(a Nullable(UInt32), b String)`.a FROM t_json_var ORDER BY id; + +DROP TABLE t_dyn; +DROP TABLE t_dyn_wide; +DROP TABLE t_var; +DROP TABLE t_mem; +DROP TABLE t_json; +DROP TABLE t_json_var; +DROP TABLE t_shapes; +DROP TABLE t_controls; +DROP TABLE t_shared; diff --git a/tests/queries/0_stateless/04653_nullable_tuple_element_subcolumn_nullable_tuple_setting.reference b/tests/queries/0_stateless/04653_nullable_tuple_element_subcolumn_nullable_tuple_setting.reference new file mode 100644 index 0000000000000..568880a1a64e5 --- /dev/null +++ b/tests/queries/0_stateless/04653_nullable_tuple_element_subcolumn_nullable_tuple_setting.reference @@ -0,0 +1,9 @@ +Nullable(Tuple(a Nullable(UInt32), b String)) +Nullable(UInt32) +1 1 +2 \N +3 \N +1 1 +2 \N +1 \N +2 5 diff --git a/tests/queries/0_stateless/04653_nullable_tuple_element_subcolumn_nullable_tuple_setting.sh b/tests/queries/0_stateless/04653_nullable_tuple_element_subcolumn_nullable_tuple_setting.sh new file mode 100755 index 0000000000000..882567218fc5f --- /dev/null +++ b/tests/queries/0_stateless/04653_nullable_tuple_element_subcolumn_nullable_tuple_setting.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# allow_nullable_tuple_in_extracted_subcolumns is read from the global context, so it has to be +# passed on the command line - a session level SET does not reach the extraction. + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# With the setting on, the whole tuple alternative is itself exposed as Nullable, while the requested +# tuple element keeps its own Nullable. Both must be readable. +${CLICKHOUSE_LOCAL} --allow_nullable_tuple_in_extracted_subcolumns=1 --enable_variant_type=1 --query " +CREATE TABLE t_var (id UInt64, value Variant(Tuple(a Nullable(UInt32), b String), String)) ENGINE = MergeTree ORDER BY id; +INSERT INTO t_var VALUES (1, CAST(tuple(CAST(1, 'Nullable(UInt32)'), 's'), 'Tuple(a Nullable(UInt32), b String)')); +INSERT INTO t_var VALUES (2, CAST(tuple(CAST(NULL, 'Nullable(UInt32)'), 's'), 'Tuple(a Nullable(UInt32), b String)')); +INSERT INTO t_var VALUES (3, 'not a tuple'); +SELECT toTypeName(value.\`Tuple(a Nullable(UInt32), b String)\`) FROM t_var LIMIT 1; +SELECT toTypeName(value.\`Tuple(a Nullable(UInt32), b String)\`.a) FROM t_var LIMIT 1; +SELECT id, value.\`Tuple(a Nullable(UInt32), b String)\`.a FROM t_var ORDER BY id; + +CREATE TABLE t_dyn (id UInt64, value Dynamic) ENGINE = MergeTree ORDER BY id; +INSERT INTO t_dyn VALUES (1, CAST(tuple(CAST(1, 'Nullable(UInt32)'), 's'), 'Tuple(a Nullable(UInt32), b String)')); +INSERT INTO t_dyn VALUES (2, CAST(5, 'UInt32')); +SELECT id, value.\`Tuple(a Nullable(UInt32), b String)\`.a FROM t_dyn ORDER BY id; +SELECT id, value.UInt32 FROM t_dyn ORDER BY id; +" diff --git a/tests/queries/0_stateless/04654_delta_lake_predicate_untranslatable_under_not.reference b/tests/queries/0_stateless/04654_delta_lake_predicate_untranslatable_under_not.reference new file mode 100644 index 0000000000000..4cec17c923575 --- /dev/null +++ b/tests/queries/0_stateless/04654_delta_lake_predicate_untranslatable_under_not.reference @@ -0,0 +1,55 @@ +-- wrong before the fix: the untranslatable child was dropped and the NOT inverted +NOT (c0 NOT BETWEEN c0 AND 100): ok +NOT (c0 > 100 OR c0 < 0): ok +NOT (c0 < c0): ok +NOT (c0 >= c0 AND c0 < 100): ok +-- wrong before the fix: the conjunction was truncated, so a narrower predicate was pushed +NOT (c0 >= 100 AND c0 < c0): ok +NOT (c0 < c0 AND c0 >= 100): ok +NOT (c0 >= 100 AND c0 < c0) AND c0 >= 0: ok +NOT (c0 < 100 AND c0 < c0): ok +-- wrong before the fix for any operand type, not just plain integers +NOT (toNullable(c0) < toNullable(c0)): ok +NOT (toString(c0) < toString(c0) AND c0 >= 100): ok +NOT ([c0] = [c0] AND c0 >= 100): ok +-- the Unknown predicate name must not be derived from the filter, which may hold non-UTF-8 bytes +NOT (c0 < c0 AND non-utf8 literal): ok +-- unchanged: predicates that already translated fully +c0 >= 100: ok +NOT (c0 < 100): ok +NOT (c0 > 100): ok +NOT (c0 > 100 AND c0 > 100): ok +NOT (c0 >= 100 AND c0 >= 100): ok +NOT (NOT (c0 >= 100)): ok +c0 >= 100::Int16: ok +NOT (c0 >= 100::Int16): ok +NOT (c0 >= 100::Int64): ok +-- unchanged: dropping an untranslatable child stays correct in monotone position +c0 < c0: ok +c0 >= c0: ok +c0 > 200 OR c0 < 0: ok +c0 >= 100 AND c0 < c0: ok +c0 >= 100 AND (c0 > 1 OR c0 < 0): ok +c0 >= 100 AND c0 != 105: ok +c0 < 50 AND NOT (c0 >= 100 AND c0 < c0): ok +NOT (NOT (c0 < c0)): ok +c0 NOT BETWEEN c0 AND 100: ok +-- file pruning must be preserved: these skip one of the two files +c0 >= 100: ok +NOT (c0 < 100): ok +NOT (c0 >= 100 AND c0 >= 100): ok +NOT (NOT (c0 >= 100)): ok +c0 >= 100 AND c0 < c0: ok +c0 >= 100 AND (c0 > 1 OR c0 < 0): ok +c0 >= 100 AND c0 != 105: ok +c0 >= 100::Int16: ok +NOT (c0 >= 100::Int16): ok +-- and these legitimately read both files +NOT (c0 < c0): ok +NOT (c0 > 100): ok +c0 < c0: ok +NOT (c0 >= 100 AND c0 < c0): ok +-- a visitor exception is the same class: it must not skip files either +failpoint NOT (c0 >= 100): ok +-- but delta_lake_throw_on_engine_predicate_error still reports it +1 diff --git a/tests/queries/0_stateless/04654_delta_lake_predicate_untranslatable_under_not.sh b/tests/queries/0_stateless/04654_delta_lake_predicate_untranslatable_under_not.sh new file mode 100755 index 0000000000000..94fcb5cb06ed4 --- /dev/null +++ b/tests/queries/0_stateless/04654_delta_lake_predicate_untranslatable_under_not.sh @@ -0,0 +1,157 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-msan +# Tag no-fasttest: delta-kernel pulls in extra dependencies. +# Tag no-msan: delta-kernel-rs (Rust) is not built under MSan, so DeltaLakeLocal is absent. + +# Regression test for https://github.com/ClickHouse/ClickHouse/issues/87390 +# A sub-expression the delta-kernel predicate translator cannot handle was reported to the +# kernel as "the iterator is exhausted", which truncates the enclosing conjunction instead of +# dropping one child. In monotone position that is harmless, but under NOT it changes the +# answer: an empty conjunction normalizes to TRUE, so NOT(TRUE) skipped every data file, and a +# partially consumed conjunction produced a predicate narrower than the truth. Both returned +# too few rows with no error. The translator now emits an explicit `Unknown` predicate, which +# the kernel treats as "cannot decide" in any polarity. +# +# Every answer is asserted against the same query over the same bytes read as plain Parquet, so +# the expected values are derived rather than hand-copied. File counts are asserted too: losing +# pruning would be a silent performance regression that a correct-answers test cannot see. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +ROOT="${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}_delta_not" +trap 'rm -rf "${ROOT}" 2>/dev/null' EXIT +rm -rf "${ROOT}" +mkdir -p "${ROOT}/t/_delta_log" + +# Two data files with disjoint ranges, written by plain file() so no experimental Delta write +# path is involved. ClickHouse's own Delta writer emits stats containing only numRecords, and a +# statsless table can never prune, which would make every pruning assertion below vacuous. So +# the transaction log is hand-written with real per-column stats. +${CLICKHOUSE_LOCAL} --query " + INSERT INTO FUNCTION file('${ROOT}/t/lo.parquet', Parquet, 'c0 Int8') SELECT toInt8(number) FROM numbers(50); + INSERT INTO FUNCTION file('${ROOT}/t/hi.parquet', Parquet, 'c0 Int8') SELECT toInt8(100 + number) FROM numbers(27); +" + +cat > "${ROOT}/t/_delta_log/00000000000000000000.json" <= c0 are untranslatable because a comparison needs one side to be a literal; +# OR has no case in the translator at all. BETWEEN expands to a conjunction containing c0 >= c0, +# which is the shape reported in the issue. +check "NOT (c0 NOT BETWEEN c0 AND 100)" +check "NOT (c0 > 100 OR c0 < 0)" +check "NOT (c0 < c0)" +# A bare NOT (c0 >= c0) would be useless here: c0 >= c0 always holds, so the correct answer is 0 +# and the wrong one is 0 too. Pairing it with a translatable conjunct makes it discriminating. +check "NOT (c0 >= c0 AND c0 < 100)" + +echo "-- wrong before the fix: the conjunction was truncated, so a narrower predicate was pushed" +# Only the child after the untranslatable one is lost, so the answer depended on operand order: +# these two returned 50 and 0 respectively before the fix. +check "NOT (c0 >= 100 AND c0 < c0)" +check "NOT (c0 < c0 AND c0 >= 100)" +check "NOT (c0 >= 100 AND c0 < c0) AND c0 >= 0" +check "NOT (c0 < 100 AND c0 < c0)" + +echo "-- wrong before the fix for any operand type, not just plain integers" +# The defect is in how the translator reports an untranslatable node, so it is independent of the +# operand type: Nullable, String and Array operands all reach the same path. +check "NOT (toNullable(c0) < toNullable(c0))" +check "NOT (toString(c0) < toString(c0) AND c0 >= 100)" +check "NOT ([c0] = [c0] AND c0 >= 100)" + +echo "-- the Unknown predicate name must not be derived from the filter, which may hold non-UTF-8 bytes" +check "NOT (c0 < c0 AND toString(c0) > unhex('fffe'))" "NOT (c0 < c0 AND non-utf8 literal)" + +echo "-- unchanged: predicates that already translated fully" +check "c0 >= 100" +check "NOT (c0 < 100)" +check "NOT (c0 > 100)" +check "NOT (c0 > 100 AND c0 > 100)" +check "NOT (c0 >= 100 AND c0 >= 100)" +check "NOT (NOT (c0 >= 100))" +check "c0 >= 100::Int16" +check "NOT (c0 >= 100::Int16)" +check "NOT (c0 >= 100::Int64)" + +echo "-- unchanged: dropping an untranslatable child stays correct in monotone position" +check "c0 < c0" +check "c0 >= c0" +check "c0 > 200 OR c0 < 0" +check "c0 >= 100 AND c0 < c0" +check "c0 >= 100 AND (c0 > 1 OR c0 < 0)" +check "c0 >= 100 AND c0 != 105" +check "c0 < 50 AND NOT (c0 >= 100 AND c0 < c0)" +check "NOT (NOT (c0 < c0))" +check "c0 NOT BETWEEN c0 AND 100" + +echo "-- file pruning must be preserved: these skip one of the two files" +check_files "c0 >= 100" 1 +check_files "NOT (c0 < 100)" 1 +check_files "NOT (c0 >= 100 AND c0 >= 100)" 1 +check_files "NOT (NOT (c0 >= 100))" 1 +check_files "c0 >= 100 AND c0 < c0" 1 +check_files "c0 >= 100 AND (c0 > 1 OR c0 < 0)" 1 +check_files "c0 >= 100 AND c0 != 105" 1 +check_files "c0 >= 100::Int16" 1 +check_files "NOT (c0 >= 100::Int16)" 1 + +echo "-- and these legitimately read both files" +check_files "NOT (c0 < c0)" 2 +check_files "NOT (c0 > 100)" 2 +check_files "c0 < c0" 2 +check_files "NOT (c0 >= 100 AND c0 < c0)" 2 + +echo "-- a visitor exception is the same class: it must not skip files either" +# The failpoint makes the literal visitor throw. It is ONCE, so it self-disarms after one hit; +# a separate invocation keeps that hit on the intended query. +${CLICKHOUSE_LOCAL} --query " + SYSTEM ENABLE FAILPOINT delta_kernel_fail_literal_visitor; + SELECT 'failpoint NOT (c0 >= 100): ' || if(count() = 50, 'ok', 'got ' || toString(count())) FROM deltaLakeLocal('${ROOT}/t') WHERE NOT (c0 >= 100); +" + +echo "-- but delta_lake_throw_on_engine_predicate_error still reports it" +${CLICKHOUSE_LOCAL} --query " + SYSTEM ENABLE FAILPOINT delta_kernel_fail_literal_visitor; + SELECT count() FROM deltaLakeLocal('${ROOT}/t') WHERE NOT (c0 >= 100) SETTINGS delta_lake_throw_on_engine_predicate_error = 1; +" 2>&1 | grep -c "FAULT_INJECTED" diff --git a/tests/queries/0_stateless/04658_deduplication_mv_row_drift_partitioned.reference b/tests/queries/0_stateless/04658_deduplication_mv_row_drift_partitioned.reference new file mode 100644 index 0000000000000..ced195aff7fd1 --- /dev/null +++ b/tests/queries/0_stateless/04658_deduplication_mv_row_drift_partitioned.reference @@ -0,0 +1,4 @@ +400 100 2 +800 100 2 +200 2 +200 2 diff --git a/tests/queries/0_stateless/04658_deduplication_mv_row_drift_partitioned.sh b/tests/queries/0_stateless/04658_deduplication_mv_row_drift_partitioned.sh new file mode 100755 index 0000000000000..d755328e22c14 --- /dev/null +++ b/tests/queries/0_stateless/04658_deduplication_mv_row_drift_partitioned.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Tags: no-random-settings, no-random-merge-tree-settings +# The scenarios pin the deduplication path exactly (the squash thresholds, the insert/thread +# counts), so settings randomization is disabled, as in 04621_deduplication_alias_hop_partitioned_target. +# +# The production crash from a plain materialized view - NO `Alias` engine anywhere. A row-count +# changing inner query (`GROUP BY`) feeds a PARTITIONED deduplicating target. The target sink splits +# the view-output block by partition and DeduplicationInfo::filterToPartition attributes each token's +# source-row range to the partitions via the scatter selector. But the view changed the row count, so +# the tokens describe the source rows while the selector describes the smaller view-output block: +# there is no source-row -> partition mapping. filterToPartition must keep every token in every +# partition (the target may still deduplicate a repeated token) instead of reading out of the +# selector's bounds. +# See https://github.com/ClickHouse/clickhouse-core-incidents/issues/2006 + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +SETTINGS="--insert_deduplicate=1 --deduplicate_blocks_in_dependent_materialized_views=1 --parallel_view_processing=1 --max_threads=1 --max_insert_threads=1" + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS mv_drift_mv" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS mv_drift_dst" +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS mv_drift_src" + +# Only dst deduplicates, and it is partitioned, so the sink splits every insert by partition before +# deduplicating. The view's GROUP BY reduces the row count between src and dst. +$CLICKHOUSE_CLIENT -q "CREATE TABLE mv_drift_src (x UInt64) ENGINE = MergeTree ORDER BY tuple() SETTINGS non_replicated_deduplication_window = 0" +$CLICKHOUSE_CLIENT -q "CREATE TABLE mv_drift_dst (x UInt64) ENGINE = MergeTree PARTITION BY x % 2 ORDER BY tuple() SETTINGS non_replicated_deduplication_window = 100000" +$CLICKHOUSE_CLIENT -q "CREATE MATERIALIZED VIEW mv_drift_mv TO mv_drift_dst AS SELECT x FROM mv_drift_src GROUP BY x" + +# A single data-fed insert carries one deduplication token, so the partition split keeps the whole +# info for every partition (single-token fast path) and only the cached data hash is used: the +# repeated insert must deduplicate in both partitions of dst. +for _ in $(seq 1 4); do seq 1 100; done | $CLICKHOUSE_CLIENT $SETTINGS -q "INSERT INTO mv_drift_src FORMAT TSV" +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM mv_drift_src), (SELECT count() FROM mv_drift_dst), (SELECT count(DISTINCT _partition_id) FROM mv_drift_dst)" + +for _ in $(seq 1 4); do seq 1 100; done | $CLICKHOUSE_CLIENT $SETTINGS -q "INSERT INTO mv_drift_src FORMAT TSV" +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM mv_drift_src), (SELECT count() FROM mv_drift_dst), (SELECT count(DISTINCT _partition_id) FROM mv_drift_dst)" + +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE mv_drift_src" +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE mv_drift_dst" + +# Two deduplication tokens in one sync insert: the source-side squashing (min_insert_block_size_rows, +# with max_insert_block_size as the parser cap) re-blocks the 800 input rows into two 400-row source +# blocks, each carrying its own token, and the view-input squashing concatenates them into one block +# before the row-count-changing GROUP BY. The partitioned sink then sees two tokens over a block that +# no longer matches the tokens' source rows. filterToPartition keeps both tokens in both partitions +# (pre-fix it walked the source-row ranges over the smaller partition selector: an out-of-bounds +# read), so the insert fills dst and the repeated insert is deduplicated. +SPLIT_SETTINGS="$SETTINGS --async_insert=0 --max_insert_block_size=400 --min_insert_block_size_rows=400 --min_insert_block_size_bytes=0 --min_insert_block_size_rows_for_materialized_views=1000000" + +{ for _ in $(seq 1 4); do seq 1 100; done; for _ in $(seq 1 4); do seq 101 200; done; } | $CLICKHOUSE_CLIENT $SPLIT_SETTINGS -q "INSERT INTO mv_drift_src FORMAT TSV" +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM mv_drift_dst), (SELECT count(DISTINCT _partition_id) FROM mv_drift_dst)" + +{ for _ in $(seq 1 4); do seq 1 100; done; for _ in $(seq 1 4); do seq 101 200; done; } | $CLICKHOUSE_CLIENT $SPLIT_SETTINGS -q "INSERT INTO mv_drift_src FORMAT TSV" +$CLICKHOUSE_CLIENT -q "SELECT (SELECT count() FROM mv_drift_dst), (SELECT count(DISTINCT _partition_id) FROM mv_drift_dst)" + +$CLICKHOUSE_CLIENT -q "DROP TABLE mv_drift_mv" +$CLICKHOUSE_CLIENT -q "DROP TABLE mv_drift_dst" +$CLICKHOUSE_CLIENT -q "DROP TABLE mv_drift_src" diff --git a/tests/queries/0_stateless/04739_iceberg_insert_partial_statistics_bounds.reference b/tests/queries/0_stateless/04739_iceberg_insert_partial_statistics_bounds.reference new file mode 100644 index 0000000000000..415eb3c1f59a8 --- /dev/null +++ b/tests/queries/0_stateless/04739_iceberg_insert_partial_statistics_bounds.reference @@ -0,0 +1,8 @@ +--- A: key Int32 + arr Array(Int32) +lower_ids=[1] upper_ids=[1] key_lower=[0] key_upper=[4] +--- B: key Int32 + opt Nullable(Int32) all NULL +lower_ids=[1] upper_ids=[1] key_lower=[0] key_upper=[4] +--- C: key Int32 + val String (control, all supported) +lower_ids=[1,2] upper_ids=[1,2] key_lower=[0] key_upper=[4] +--- D: key Int32 + f Nullable(Float64) with values +lower_ids=[1] upper_ids=[1] key_lower=[0] key_upper=[4] diff --git a/tests/queries/0_stateless/04739_iceberg_insert_partial_statistics_bounds.sh b/tests/queries/0_stateless/04739_iceberg_insert_partial_statistics_bounds.sh new file mode 100755 index 0000000000000..8acdb1edb91bc --- /dev/null +++ b/tests/queries/0_stateless/04739_iceberg_insert_partial_statistics_bounds.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CUR_DIR"/../shell_config.sh + +PREFIX="t_${CLICKHOUSE_DATABASE}_${RANDOM}" +ROOT="${USER_FILES_PATH}/${PREFIX}" + +cleanup() +{ + for suffix in a b c d; do + ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${PREFIX}_${suffix}" + done + rm -rf "${ROOT}" +} +trap cleanup EXIT + +# Prints the field ids present in lower_bounds / upper_bounds, plus the decoded Int32 bounds of +# field id 1 (the `key` column, always Int32 and always serializable here). +report() +{ + local table_dir="$1" + for manifest in $(find "${table_dir}/metadata" -maxdepth 1 -name '*.avro' \ + -not -name 'snap-*.avro' -type f | sort); do + ${CLICKHOUSE_CLIENT} --query " + SELECT + 'lower_ids=' || toString(arraySort(arrayMap(x -> x.1, tupleElement(data_file, 'lower_bounds')))), + 'upper_ids=' || toString(arraySort(arrayMap(x -> x.1, tupleElement(data_file, 'upper_bounds')))), + 'key_lower=' || toString(arrayMap(x -> reinterpretAsInt32(x.2), + arrayFilter(x -> x.1 = 1, tupleElement(data_file, 'lower_bounds')))), + 'key_upper=' || toString(arrayMap(x -> reinterpretAsInt32(x.2), + arrayFilter(x -> x.1 = 1, tupleElement(data_file, 'upper_bounds')))) + FROM file('${manifest}', Avro) + ORDER BY 1, 2, 3, 4 + " + done +} + +# Case A: an unsupported column type (Array(Int32)) must not suppress the bounds of `key`. +echo '--- A: key Int32 + arr Array(Int32)' +${CLICKHOUSE_CLIENT} --query " + CREATE TABLE ${PREFIX}_a (key Int32, arr Array(Int32)) + ENGINE = IcebergLocal('${ROOT}/a/') +" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query \ + "INSERT INTO ${PREFIX}_a SELECT number::Int32, [number, number] FROM numbers(5)" +report "${ROOT}/a" + +# Case B: an entirely-NULL Nullable column yields a Null bound, which must not suppress `key` either. +echo '--- B: key Int32 + opt Nullable(Int32) all NULL' +${CLICKHOUSE_CLIENT} --query " + CREATE TABLE ${PREFIX}_b (key Int32, opt Nullable(Int32)) + ENGINE = IcebergLocal('${ROOT}/b/') +" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query \ + "INSERT INTO ${PREFIX}_b SELECT number::Int32, NULL FROM numbers(5)" +report "${ROOT}/b" + +# Case C: control. All columns serializable, so both field ids keep their bounds exactly as before. +echo '--- C: key Int32 + val String (control, all supported)' +${CLICKHOUSE_CLIENT} --query " + CREATE TABLE ${PREFIX}_c (key Int32, val String) + ENGINE = IcebergLocal('${ROOT}/c/') +" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query \ + "INSERT INTO ${PREFIX}_c SELECT number::Int32, 'val-' || toString(number) FROM numbers(5)" +report "${ROOT}/c" + +# Case D: wrapper composition with a non-NULL value. Float64 is unsupported for bounds even though +# the byte dumper handles it, so the filter must gate on the bounds predicate and skip only `f`. +echo '--- D: key Int32 + f Nullable(Float64) with values' +${CLICKHOUSE_CLIENT} --query " + CREATE TABLE ${PREFIX}_d (key Int32, f Nullable(Float64)) + ENGINE = IcebergLocal('${ROOT}/d/') +" +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --query \ + "INSERT INTO ${PREFIX}_d SELECT number::Int32, number + 0.5 FROM numbers(5)" +report "${ROOT}/d" diff --git a/tests/queries/0_stateless/04750_database_backup_multiple_databases_same_backup.reference b/tests/queries/0_stateless/04750_database_backup_multiple_databases_same_backup.reference new file mode 100644 index 0000000000000..1e419f351efe3 --- /dev/null +++ b/tests/queries/0_stateless/04750_database_backup_multiple_databases_same_backup.reference @@ -0,0 +1,13 @@ +two databases from one backup +1000 499500 +1000 499500 +-- +recreated under the same name +1000 499500 +1000 499500 +-- +recreated under a different name +1000 499500 +-- +rename is not supported +Backup: RENAME DATABASE is not supported. (NOT_IMPLEMENTED) diff --git a/tests/queries/0_stateless/04750_database_backup_multiple_databases_same_backup.sh b/tests/queries/0_stateless/04750_database_backup_multiple_databases_same_backup.sh new file mode 100755 index 0000000000000..18e32d410e8f6 --- /dev/null +++ b/tests/queries/0_stateless/04750_database_backup_multiple_databases_same_backup.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Tags: no-encrypted-storage + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +SOURCE_DATABASE_NAME=${CLICKHOUSE_TEST_UNIQUE_NAME}_source +FIRST_DATABASE_NAME=${CLICKHOUSE_TEST_UNIQUE_NAME}_first +SECOND_DATABASE_NAME=${CLICKHOUSE_TEST_UNIQUE_NAME}_second +BACKUP_PATH=${CLICKHOUSE_TEST_UNIQUE_NAME}_backup + +$CLICKHOUSE_CLIENT -q """ +DROP DATABASE IF EXISTS $SOURCE_DATABASE_NAME; +CREATE DATABASE $SOURCE_DATABASE_NAME; + +CREATE TABLE $SOURCE_DATABASE_NAME.test_table (id UInt64, value String) ENGINE = MergeTree ORDER BY id; +INSERT INTO $SOURCE_DATABASE_NAME.test_table SELECT number, number FROM numbers(1000); + +BACKUP DATABASE $SOURCE_DATABASE_NAME TO Disk('backups', '$BACKUP_PATH') FORMAT Null; + +SELECT 'two databases from one backup'; + +DROP DATABASE IF EXISTS $FIRST_DATABASE_NAME; +DROP DATABASE IF EXISTS $SECOND_DATABASE_NAME; +CREATE DATABASE $FIRST_DATABASE_NAME ENGINE = Backup('$SOURCE_DATABASE_NAME', Disk('backups', '$BACKUP_PATH')); +CREATE DATABASE $SECOND_DATABASE_NAME ENGINE = Backup('$SOURCE_DATABASE_NAME', Disk('backups', '$BACKUP_PATH')); + +SELECT count(), sum(id) FROM $FIRST_DATABASE_NAME.test_table; +SELECT count(), sum(id) FROM $SECOND_DATABASE_NAME.test_table; + +DROP DATABASE $SECOND_DATABASE_NAME; +DROP DATABASE $FIRST_DATABASE_NAME; + +SELECT '--'; +SELECT 'recreated under the same name'; + +CREATE DATABASE $FIRST_DATABASE_NAME ENGINE = Backup('$SOURCE_DATABASE_NAME', Disk('backups', '$BACKUP_PATH')); +SELECT count(), sum(id) FROM $FIRST_DATABASE_NAME.test_table; +DROP DATABASE $FIRST_DATABASE_NAME; +CREATE DATABASE $FIRST_DATABASE_NAME ENGINE = Backup('$SOURCE_DATABASE_NAME', Disk('backups', '$BACKUP_PATH')); +SELECT count(), sum(id) FROM $FIRST_DATABASE_NAME.test_table; + +SELECT '--'; +SELECT 'recreated under a different name'; + +DROP DATABASE $FIRST_DATABASE_NAME; +CREATE DATABASE $SECOND_DATABASE_NAME ENGINE = Backup('$SOURCE_DATABASE_NAME', Disk('backups', '$BACKUP_PATH')); +SELECT count(), sum(id) FROM $SECOND_DATABASE_NAME.test_table; +""" + +echo '--' +echo 'rename is not supported' +$CLICKHOUSE_CLIENT -q "RENAME DATABASE $SECOND_DATABASE_NAME TO ${SECOND_DATABASE_NAME}_renamed" 2>&1 \ + | grep -oF 'Backup: RENAME DATABASE is not supported. (NOT_IMPLEMENTED)' | head -n 1 + +$CLICKHOUSE_CLIENT -q """ +DROP DATABASE $SECOND_DATABASE_NAME; +DROP DATABASE $SOURCE_DATABASE_NAME; +""" diff --git a/tests/queries/0_stateless/04762_datalake_concurrent_metadata_update.reference b/tests/queries/0_stateless/04762_datalake_concurrent_metadata_update.reference new file mode 100644 index 0000000000000..a75a25f9e1f6a --- /dev/null +++ b/tests/queries/0_stateless/04762_datalake_concurrent_metadata_update.reference @@ -0,0 +1,4 @@ +1 +1 +48 48 +1 diff --git a/tests/queries/0_stateless/04762_datalake_concurrent_metadata_update.sh b/tests/queries/0_stateless/04762_datalake_concurrent_metadata_update.sh new file mode 100755 index 0000000000000..fa99d6edd2af3 --- /dev/null +++ b/tests/queries/0_stateless/04762_datalake_concurrent_metadata_update.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-msan + +# Concurrent readers of the same DeltaLake table while its metadata is republished. +# A data race is not visible in query output, so a regression shows up as a sanitizer +# report rather than as a diff against the reference. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +DELTA_DIR="${USER_FILES_PATH}/${CLICKHOUSE_DATABASE}_delta_concurrent" +TABLE="t_${CLICKHOUSE_DATABASE}" +OUT_DIR="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}_streams" +trap 'rm -rf "${DELTA_DIR:?}" "${OUT_DIR:?}"; $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS ${TABLE}"' EXIT + +rm -rf "${DELTA_DIR:?}" +mkdir -p "$DELTA_DIR" +cp -r "$CUR_DIR"/data_delta_lake/lakehouses/spark_catalog/test/t0/. "$DELTA_DIR/" + +CREATE="DROP TABLE IF EXISTS ${TABLE}; + CREATE TABLE ${TABLE} ENGINE = DeltaLakeLocal('${DELTA_DIR}')" +$CLICKHOUSE_CLIENT --allow_experimental_delta_kernel_rs=0 --multiquery "$CREATE" + +# total_rows is served only when the delta kernel is enabled. +$CLICKHOUSE_CLIENT --allow_experimental_delta_kernel_rs=0 \ + -q "SELECT total_rows IS NULL FROM system.tables WHERE database = currentDatabase() AND name = '${TABLE}'" +$CLICKHOUSE_CLIENT --allow_experimental_delta_kernel_rs=1 \ + -q "SELECT total_rows IS NOT NULL FROM system.tables WHERE database = currentDatabase() AND name = '${TABLE}'" + +# The probe above cached delta-kernel metadata, whose `supportsUpdate` is true, so further +# queries refresh it in place instead of reassigning the pointer. Re-creating under the +# kernel-disabled setting caches metadata that has to be replaced instead. +$CLICKHOUSE_CLIENT --allow_experimental_delta_kernel_rs=0 --multiquery "$CREATE" + +# The kernel-enabled readers below then replace that pointer concurrently while reading it. +# One long-lived client per stream keeps the queries back to back. Each iteration ends with a +# marker: the race is invisible in query output, so without counting the queries a stream +# really issued, one dying early would still match the reference. +# The kernel-disabled stream must not run a read pipeline: `createFileIterator` reads the metadata +# twice, so it can pin no snapshot version from the object the first read saw and then call +# `iterate` on a replacement that requires one. DESCRIBE reaches `update` without that iterator. +STREAMS=4 +QUERIES=12 +READS="SELECT total_rows, total_bytes FROM system.tables WHERE database = currentDatabase() FORMAT Null; SELECT 1 FORMAT TSVRaw;" +COUNTS="DESCRIBE TABLE ${TABLE} FORMAT Null; SELECT 1 FORMAT TSVRaw;" +rm -rf "${OUT_DIR:?}" +mkdir -p "$OUT_DIR" +pids=() +for i in $(seq 1 $STREAMS); do + yes "$READS" | head -n $QUERIES | + $CLICKHOUSE_CLIENT --allow_experimental_delta_kernel_rs=1 --multiquery > "$OUT_DIR/reads-$i" 2>/dev/null & + pids+=($!) + yes "$COUNTS" | head -n $QUERIES | + $CLICKHOUSE_CLIENT --allow_experimental_delta_kernel_rs=0 --multiquery > "$OUT_DIR/counts-$i" 2>/dev/null & + pids+=($!) +done +# Bare `wait` reports success once every child is reaped, whatever they exited with. +for pid in "${pids[@]}"; do + wait "$pid" || echo "concurrent stream $pid failed with status $?" +done + +echo "$(cat "$OUT_DIR"/reads-* | wc -l) $(cat "$OUT_DIR"/counts-* | wc -l)" + +$CLICKHOUSE_CLIENT --allow_experimental_delta_kernel_rs=1 \ + -q "SELECT total_rows > 0 FROM system.tables WHERE database = currentDatabase() AND name = '${TABLE}'" diff --git a/tests/queries/0_stateless/04831_replicated_keeper_path_macro_attach.reference b/tests/queries/0_stateless/04831_replicated_keeper_path_macro_attach.reference new file mode 100644 index 0000000000000..5199306181a4f --- /dev/null +++ b/tests/queries/0_stateless/04831_replicated_keeper_path_macro_attach.reference @@ -0,0 +1,9 @@ +BAD_ARGUMENTS +BAD_ARGUMENTS +BAD_ARGUMENTS +1 +ATTACHED +1 +1 +RESTARTED +1 diff --git a/tests/queries/0_stateless/04831_replicated_keeper_path_macro_attach.sh b/tests/queries/0_stateless/04831_replicated_keeper_path_macro_attach.sh new file mode 100755 index 0000000000000..12fdaca2c3f9e --- /dev/null +++ b/tests/queries/0_stateless/04831_replicated_keeper_path_macro_attach.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash +# Tags: zookeeper, no-replicated-database, no-ordinary-database, no-shared-merge-tree +# Rows needing a per-copy unique database name or UUID, which a .sql file cannot interpolate. + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +ZK="/clickhouse/04831/${CLICKHOUSE_DATABASE}" +# A database name is a carrier in its own right, so use one whose own name is path-unsafe. +UNSAFE_DB="${CLICKHOUSE_DATABASE}/d" + +# A full-definition ATTACH warns on stderr, which says nothing about the checks under test. +CLIENT="${CLICKHOUSE_CLIENT} --server_logs_file=/dev/null" + +# Report the error name once, whether or not the client repeats the message. +expect_bad_arguments() { grep -q -F 'BAD_ARGUMENTS' && echo BAD_ARGUMENTS || echo UNEXPECTED; } + +${CLIENT} -q "DROP DATABASE IF EXISTS \`${UNSAFE_DB}\` SYNC" +${CLIENT} -q "CREATE DATABASE \`${UNSAFE_DB}\`" + +# Only {database} substitutes here, so this row fails on the database branch alone. +${CLIENT} -q "CREATE TABLE \`${UNSAFE_DB}\`.t (c0 Int) ENGINE = ReplicatedMergeTree('${ZK}/{database}/fixed', 'r1') ORDER BY c0" 2>&1 | + expect_bad_arguments + +${CLIENT} -q "DROP DATABASE \`${UNSAFE_DB}\` SYNC" + +# The same branch, reached by a database name that is illegal as a path COMPONENT instead of by +# carrying a '/': here the check on the substituted value has nothing to object to. +CTL_DB=$(printf '%s_g\001h' "${CLICKHOUSE_DATABASE}") +${CLIENT} -q "DROP DATABASE IF EXISTS \`${CTL_DB}\` SYNC" +${CLIENT} -q "CREATE DATABASE \`${CTL_DB}\`" +${CLIENT} -q "CREATE TABLE \`${CTL_DB}\`.t (c0 Int) ENGINE = ReplicatedMergeTree('${ZK}/{database}/fixed', 'r1c') ORDER BY c0" 2>&1 | + expect_bad_arguments +${CLIENT} -q "DROP DATABASE \`${CTL_DB}\` SYNC" + +# A full definition supplied by the user is judged like a CREATE, unlike the short form above. +U1=$(${CLIENT} -q "SELECT generateUUIDv4()") +${CLIENT} -q "ATTACH TABLE \`h/replicas/i\` UUID '${U1}' (c0 Int) ENGINE = ReplicatedMergeTree('${ZK}/{database}/{table}', 'r2') ORDER BY c0" 2>&1 | + expect_bad_arguments + +U2=$(${CLIENT} -q "SELECT generateUUIDv4()") +${CLIENT} -q "ATTACH TABLE t_full_def UUID '${U2}' (c0 Int) ENGINE = ReplicatedMergeTree('${ZK}/{database}/{table}', 'r3') ORDER BY c0" 2>/dev/null +${CLIENT} -q "SELECT count() FROM system.replicas WHERE database = currentDatabase() AND table = 't_full_def'" +${CLIENT} -q "DROP TABLE t_full_def" + +# A table whose stored path still carries {database} keeps loading after the database is renamed to a +# path-unsafe name: re-reading a definition from metadata must not re-judge it. A configured macro +# supplies the {database}, so unlike the short ATTACH above the substitution does survive into metadata. +LEGACY_DB="${CLICKHOUSE_DATABASE}_legacy" +${CLIENT} -q "DROP DATABASE IF EXISTS \`${LEGACY_DB}/d\` SYNC" +${CLIENT} -q "DROP DATABASE IF EXISTS \`${LEGACY_DB}\` SYNC" +${CLIENT} -q "CREATE DATABASE \`${LEGACY_DB}\`" +${CLIENT} -q "CREATE TABLE \`${LEGACY_DB}\`.t (c0 Int) ENGINE = ReplicatedMergeTree('{default_path_test}04831legacy', 'r4') ORDER BY c0" +${CLIENT} -q "RENAME DATABASE \`${LEGACY_DB}\` TO \`${LEGACY_DB}/d\`" +${CLIENT} -q "DETACH TABLE \`${LEGACY_DB}/d\`.t" +${CLIENT} -q "ATTACH TABLE \`${LEGACY_DB}/d\`.t" 2>&1 | grep -q -F 'BAD_ARGUMENTS' && echo REJECTED || echo ATTACHED +${CLIENT} -q "SELECT count() FROM system.tables WHERE database = '${LEGACY_DB}/d' AND name = 't'" +# Re-resolve the path under the original name before dropping: the stored path re-expands {database}, +# so the table now points at a different znode tree than the one the CREATE made, and dropping it +# under the new name would leave the original tree behind. This is pre-existing behaviour of a stored +# {database}, unrelated to the checks, and it is why the table is created in its own database. +${CLIENT} -q "RENAME DATABASE \`${LEGACY_DB}/d\` TO \`${LEGACY_DB}\`" +${CLIENT} -q "DETACH TABLE \`${LEGACY_DB}\`.t" +${CLIENT} -q "ATTACH TABLE \`${LEGACY_DB}\`.t" +${CLIENT} -q "DROP DATABASE \`${LEGACY_DB}\` SYNC" + +# SYSTEM RESTART REPLICA re-attaches from stored metadata, so it must not re-judge the path either. +# On a rejection the table is left permanently detached, so the count below is the real assertion. +RESTART_DB="${CLICKHOUSE_DATABASE}_restart" +${CLIENT} -q "DROP DATABASE IF EXISTS \`${RESTART_DB}/d\` SYNC" +${CLIENT} -q "DROP DATABASE IF EXISTS \`${RESTART_DB}\` SYNC" +${CLIENT} -q "CREATE DATABASE \`${RESTART_DB}\`" +${CLIENT} -q "CREATE TABLE \`${RESTART_DB}\`.t (c0 Int) ENGINE = ReplicatedMergeTree('{default_path_test}04831restart', 'r5') ORDER BY c0" +# A direct {database} is unfolded before the definition is stored, leaving nothing to re-substitute, +# so assert the retention this row depends on instead of assuming it. +${CLIENT} -q "SELECT create_table_query LIKE '%{default_path_test}%' FROM system.tables WHERE database = '${RESTART_DB}' AND name = 't'" +${CLIENT} -q "RENAME DATABASE \`${RESTART_DB}\` TO \`${RESTART_DB}/d\`" +${CLIENT} -q "SYSTEM RESTART REPLICA \`${RESTART_DB}/d\`.t" 2>&1 | grep -q -F 'BAD_ARGUMENTS' && echo REJECTED || echo RESTARTED +${CLIENT} -q "SELECT count() FROM system.tables WHERE database = '${RESTART_DB}/d' AND name = 't'" +${CLIENT} -q "RENAME DATABASE \`${RESTART_DB}/d\` TO \`${RESTART_DB}\`" +${CLIENT} -q "DETACH TABLE \`${RESTART_DB}\`.t" +${CLIENT} -q "ATTACH TABLE \`${RESTART_DB}\`.t" +${CLIENT} -q "DROP DATABASE \`${RESTART_DB}\` SYNC" diff --git a/tests/queries/0_stateless/04832_replicated_keeper_path_macro_substitution.reference b/tests/queries/0_stateless/04832_replicated_keeper_path_macro_substitution.reference new file mode 100644 index 0000000000000..7080556549b8f --- /dev/null +++ b/tests/queries/0_stateless/04832_replicated_keeper_path_macro_substitution.reference @@ -0,0 +1,3 @@ +table_. +. +/clickhouse/04832/{db}/a/b/c diff --git a/tests/queries/0_stateless/04832_replicated_keeper_path_macro_substitution.sql b/tests/queries/0_stateless/04832_replicated_keeper_path_macro_substitution.sql new file mode 100644 index 0000000000000..858173c0b61f3 --- /dev/null +++ b/tests/queries/0_stateless/04832_replicated_keeper_path_macro_substitution.sql @@ -0,0 +1,89 @@ +-- Tags: zookeeper, no-replicated-database, no-ordinary-database, no-shared-merge-tree + +SET send_logs_level = 'fatal'; +SET database_replicated_allow_replicated_engine_arguments = 1; + +-- A {database}/{table} substitution must stay a single ZooKeeper path component. +-- Every path below is scoped by {database} so parallel copies of this test cannot collide. + +-- '/' in the substituted value adds path components. +CREATE TABLE `a/b` (c0 Int) ENGINE = ReplicatedMergeTree('/clickhouse/04832/{database}/{table}', 'r1') ORDER BY c0; -- { serverError BAD_ARGUMENTS } +-- The replica name becomes a znode name under `/replicas/`, so it is a carrier too. +CREATE TABLE `c/d` (c0 Int) ENGINE = ReplicatedMergeTree('/clickhouse/04832/{database}/x', '{table}') ORDER BY c0; -- { serverError BAD_ARGUMENTS } + +-- '.' and '..' are not valid znode names. This is checked on the expanded path, so the substituted +-- value only has to be illegal once the surrounding template has been applied. +CREATE TABLE `.` (c0 Int) ENGINE = ReplicatedMergeTree('/clickhouse/04832/{database}/{table}', 'r2') ORDER BY c0; -- { serverError BAD_ARGUMENTS } +CREATE TABLE `..` (c0 Int) ENGINE = ReplicatedMergeTree('/clickhouse/04832/{database}/{table}', 'r3') ORDER BY c0; -- { serverError BAD_ARGUMENTS } + +-- A configured macro whose value contains {database}/{table} is expanded in the second pass, which +-- must be validated too. +CREATE TABLE `{default_path_test}n1` (c0 Int) ENGINE = ReplicatedMergeTree('{table}', 'r4') ORDER BY c0; -- { serverError BAD_ARGUMENTS } +CREATE TABLE `{default_path_test}n2/replicas/p` (c0 Int) ENGINE = ReplicatedMergeTree('{table}', 'r5') ORDER BY c0; -- { serverError BAD_ARGUMENTS } + +-- Here a configured macro supplies the '/' and the table name supplies the illegal component, so the +-- two appear in different expansion passes. +CREATE TABLE `.` (c0 Int) ENGINE = ReplicatedMergeTree('{default_path_test}{table}', 'r6') ORDER BY c0; -- { serverError BAD_ARGUMENTS } +-- Both macros here are configured ones, so {table} reaches the output only in the second pass: this +-- row is the only one that depends on the substitution being recorded there. The value carries +-- neither '/' nor a brace, so the checks on the value cannot be what rejects it. +CREATE TABLE `i\x01j` (c0 Int) ENGINE = ReplicatedMergeTree('{default_path_test}{default_name_test}', 'r6a') ORDER BY c0; -- { serverError BAD_ARGUMENTS } + +-- A closing brace on its own is enough, because it can complete a brace opened by the template. +CREATE TABLE `e}f` (c0 Int) ENGINE = ReplicatedMergeTree('/clickhouse/04832/{database}/{table}', 'r6b') ORDER BY c0; -- { serverError BAD_ARGUMENTS } + +-- Control characters are not valid znode names either. U+0085 is a C1 control, encoded as C2 85. +CREATE TABLE `g\x01h` (c0 Int) ENGINE = ReplicatedMergeTree('/clickhouse/04832/{database}/{table}', 'r6c') ORDER BY c0; -- { serverError BAD_ARGUMENTS } +CREATE TABLE `g\xC2\x85h` (c0 Int) ENGINE = ReplicatedMergeTree('/clickhouse/04832/{database}/{table}', 'r6d') ORDER BY c0; -- { serverError BAD_ARGUMENTS } +-- Both ends of each forbidden range: U+001F and U+007F are single bytes, U+0080 and U+009F need the +-- pair test. +CREATE TABLE `g\x1Fh` (c0 Int) ENGINE = ReplicatedMergeTree('/clickhouse/04832/{database}/{table}', 'r6d0') ORDER BY c0; -- { serverError BAD_ARGUMENTS } +CREATE TABLE `g\x7Fh` (c0 Int) ENGINE = ReplicatedMergeTree('/clickhouse/04832/{database}/{table}', 'r6d1') ORDER BY c0; -- { serverError BAD_ARGUMENTS } +CREATE TABLE `g\xC2\x80h` (c0 Int) ENGINE = ReplicatedMergeTree('/clickhouse/04832/{database}/{table}', 'r6d2') ORDER BY c0; -- { serverError BAD_ARGUMENTS } +CREATE TABLE `g\xC2\x9Fh` (c0 Int) ENGINE = ReplicatedMergeTree('/clickhouse/04832/{database}/{table}', 'r6d3') ORDER BY c0; -- { serverError BAD_ARGUMENTS } +-- A name whose bytes merely look like the above is unaffected: U+00BF is C2 BF, outside the C1 range. +CREATE TABLE `g\xC2\xBFh` (c0 Int) ENGINE = ReplicatedMergeTree('/clickhouse/04832/{database}/{table}', 'r6e') ORDER BY c0; +CREATE TABLE `таблица_🚀_表` (c0 Int) ENGINE = ReplicatedMergeTree('/clickhouse/04832/{database}/{table}', 'r6f') ORDER BY c0; + +-- Ordinary and dotted names keep working: the rule is not a character-level ban on '.'. +CREATE TABLE t_plain (c0 Int) ENGINE = ReplicatedMergeTree('/clickhouse/04832/{database}/{table}', 'r7') ORDER BY c0; +CREATE TABLE `my.table` (c0 Int) ENGINE = ReplicatedMergeTree('/clickhouse/04832/{database}/{table}', 'r8') ORDER BY c0; + +-- An embedded substitution still yields a legal component, so it must be accepted. +CREATE TABLE `.` (c0 Int) ENGINE = ReplicatedMergeTree('{default_path_test}embedded', '{default_name_test}') ORDER BY c0; +SELECT splitByChar('/', replica_path)[-1] FROM system.replicas WHERE database = currentDatabase() AND table = '.'; + +-- The replica name is a znode name too, so an illegal component there is rejected the same way. This +-- value has no '/', so only the component check on the replica name can reject it. +CREATE TABLE `..` (c0 Int) ENGINE = ReplicatedMergeTree('/clickhouse/04832/{database}/x', '{table}') ORDER BY c0; -- { serverError BAD_ARGUMENTS } +-- An explicitly written value is judged on its own merits even when the path substitutes: the same '.' +-- that is rejected above when it comes from {table} is accepted here. +CREATE TABLE t_explicit_replica (c0 Int) ENGINE = ReplicatedMergeTree('/clickhouse/04832/{database}/{table}', '.') ORDER BY c0; +SELECT replica_name FROM system.replicas WHERE database = currentDatabase() AND table = 't_explicit_replica'; + +-- The replica name reaches the output only in the second pass here, via a configured macro, while the +-- path is fully literal. So the substitution has to be recorded in that pass for this to be rejected: +-- 'table_' prefixes the value, which keeps a '.' legal, and the control byte is what survives it. +CREATE TABLE `g\x01h` (c0 Int) ENGINE = ReplicatedMergeTree('/clickhouse/04832/literal/x', '{default_name_test}') ORDER BY c0; -- { serverError BAD_ARGUMENTS } + +-- A substituted replica name pins the path to the table, so renaming is refused. The path here is +-- literal, so only the replica name can be what makes the restriction apply. +CREATE TABLE t_renamable (c0 Int) ENGINE = ReplicatedMergeTree('/clickhouse/04832/{database}/rn', '{default_name_test}') ORDER BY c0; +RENAME TABLE t_renamable TO t_renamed; -- { serverError NOT_IMPLEMENTED } + +-- Paths written out in full are untouched, including the nesting an operator may rely on. +CREATE TABLE t_nested_path (c0 Int) ENGINE = ReplicatedMergeTree('/clickhouse/04832/{database}/a/b/c', 'r9') ORDER BY c0; +SELECT replaceOne(zookeeper_path, currentDatabase(), '{db}') FROM system.replicas WHERE database = currentDatabase() AND table = 't_nested_path'; + +-- A definition read back from metadata is never re-judged, so an existing table keeps loading. +DETACH TABLE t_plain; +ATTACH TABLE t_plain; + +DROP TABLE t_plain; +DROP TABLE `my.table`; +DROP TABLE `.`; +DROP TABLE `g\xC2\xBFh`; +DROP TABLE `таблица_🚀_表`; +DROP TABLE t_explicit_replica; +DROP TABLE t_renamable; +DROP TABLE t_nested_path; diff --git a/tests/queries/0_stateless/04846_materialized_view_target_is_materialized_view.reference b/tests/queries/0_stateless/04846_materialized_view_target_is_materialized_view.reference new file mode 100644 index 0000000000000..4575a3379dfe1 --- /dev/null +++ b/tests/queries/0_stateless/04846_materialized_view_target_is_materialized_view.reference @@ -0,0 +1,2 @@ +insert into the source table still succeeds 2 +table target still works 3 diff --git a/tests/queries/0_stateless/04846_materialized_view_target_is_materialized_view.sql b/tests/queries/0_stateless/04846_materialized_view_target_is_materialized_view.sql new file mode 100644 index 0000000000000..34778e59ab40a --- /dev/null +++ b/tests/queries/0_stateless/04846_materialized_view_target_is_materialized_view.sql @@ -0,0 +1,41 @@ +CREATE TABLE src (a UInt16) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE tgt (a UInt16) ENGINE = MergeTree ORDER BY tuple(); + +CREATE MATERIALIZED VIEW v_table_target TO tgt AS SELECT a FROM src; +CREATE MATERIALIZED VIEW v_view_target TO v_table_target AS SELECT a FROM src; + +-- Inserting into a view whose target is another view is refused on both insert routes. +INSERT INTO v_view_target VALUES (1); -- { serverError NOT_IMPLEMENTED } +INSERT INTO v_view_target SELECT 1; -- { serverError NOT_IMPLEMENTED } + +-- On master this reaches the same hop through the population insert and is refused there, but this +-- branch still rejects `TO` together with `POPULATE` while parsing, so the client fails before the +-- query is ever sent and the hop is never reached. Parse errors are always client errors. +INSERT INTO src VALUES (2); +CREATE MATERIALIZED VIEW v_populate TO v_table_target POPULATE AS SELECT a FROM src; -- { clientError SYNTAX_ERROR } + +-- A refreshable view is a materialized view too, so it is refused as a target on the same edge. +-- It refreshes into a table of its own, so its initial refresh cannot disturb the counts below. +CREATE TABLE tgt_refreshable (a UInt16) ENGINE = MergeTree ORDER BY tuple(); +CREATE MATERIALIZED VIEW v_refreshable REFRESH EVERY 1 YEAR APPEND TO tgt_refreshable AS SELECT a FROM src; +CREATE MATERIALIZED VIEW v_refreshable_target TO v_refreshable AS SELECT a FROM src; +INSERT INTO v_refreshable_target VALUES (5); -- { serverError NOT_IMPLEMENTED } + +-- Reaching the same hop as a dependent view keeps the pre-existing behaviour: the insert addresses +-- `src`, the hop is skipped, and the rows that would have gone through `v_view_target` are dropped. +INSERT INTO src VALUES (3); +SELECT 'insert into the source table still succeeds', count() FROM src ORDER BY ALL; + +-- A target that is an ordinary table keeps working, on the direct and the dependent route. +INSERT INTO v_table_target VALUES (4); +SELECT 'table target still works', count() FROM tgt; + +-- `v_populate` is never created here, since the statement above is rejected while it is parsed. +DROP VIEW IF EXISTS v_populate; +DROP VIEW v_refreshable_target; +DROP VIEW v_refreshable; +DROP VIEW v_view_target; +DROP VIEW v_table_target; +DROP TABLE tgt_refreshable; +DROP TABLE tgt; +DROP TABLE src; diff --git a/tests/queries/0_stateless/04847_ttl_where_json_subcolumn.reference b/tests/queries/0_stateless/04847_ttl_where_json_subcolumn.reference new file mode 100644 index 0000000000000..e7d6081b6479c --- /dev/null +++ b/tests/queries/0_stateless/04847_ttl_where_json_subcolumn.reference @@ -0,0 +1,4 @@ +2 +3 +2 +3 diff --git a/tests/queries/0_stateless/04847_ttl_where_json_subcolumn.sql b/tests/queries/0_stateless/04847_ttl_where_json_subcolumn.sql new file mode 100644 index 0000000000000..b73de246a4b77 --- /dev/null +++ b/tests/queries/0_stateless/04847_ttl_where_json_subcolumn.sql @@ -0,0 +1,32 @@ +-- Regression test: a TTL `WHERE` referencing a subcolumn (e.g. of a `JSON` column) used to +-- throw a logical error `original.has_value()` at CREATE, because the TTL analysis reported +-- the subcolumn in its required source columns and the mapping back to the table columns +-- asserted that every reported name is a top-level column. + +DROP TABLE IF EXISTS t_ttl_where_json_subcolumn; + +CREATE TABLE t_ttl_where_json_subcolumn +( + id UInt32, + d DateTime, + j JSON(ts UInt32) +) +ENGINE = MergeTree +ORDER BY id +TTL d + INTERVAL 1 DAY WHERE j.ts > 0 +SETTINGS min_bytes_for_wide_part = 0, materialize_ttl_recalculate_only = 0; + +INSERT INTO t_ttl_where_json_subcolumn VALUES (1, now() - INTERVAL 10 DAY, '{"ts" : 1}'), (2, now() - INTERVAL 10 DAY, '{"ts" : 0}'), (3, now(), '{"ts" : 1}'); + +OPTIMIZE TABLE t_ttl_where_json_subcolumn FINAL; + +-- Row 1 is expired and matches the WHERE on the subcolumn; rows 2 and 3 must survive. +SELECT id FROM t_ttl_where_json_subcolumn ORDER BY id; + +-- Rebuilding the TTL expression from the stored source columns (which include the subcolumn) +-- must also work, e.g. for ALTER ... MODIFY TTL and the subsequent materialization. +ALTER TABLE t_ttl_where_json_subcolumn MODIFY TTL d + INTERVAL 2 DAY WHERE j.ts > 42 SETTINGS mutations_sync = 2; + +SELECT id FROM t_ttl_where_json_subcolumn ORDER BY id; + +DROP TABLE t_ttl_where_json_subcolumn; diff --git a/tests/queries/0_stateless/04848_compressed_decompressed_size_bound.reference b/tests/queries/0_stateless/04848_compressed_decompressed_size_bound.reference new file mode 100644 index 0000000000000..94303fa45f29b --- /dev/null +++ b/tests/queries/0_stateless/04848_compressed_decompressed_size_bound.reference @@ -0,0 +1,11 @@ +-- a valid frame still executes (proves the arms below fail for the intended reason) +1 +-- a codec that stores data uncompressed must not lie about the uncompressed size +1 +-- nor may it understate the uncompressed size +1 +-- engines keep working on ordinary data +1000 100000 +1000 100000 +1 +1 diff --git a/tests/queries/0_stateless/04848_compressed_decompressed_size_bound.sh b/tests/queries/0_stateless/04848_compressed_decompressed_size_bound.sh new file mode 100755 index 0000000000000..d146e65f34fb8 --- /dev/null +++ b/tests/queries/0_stateless/04848_compressed_decompressed_size_bound.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +set -o pipefail + +# Wire layout of a compressed frame: +# [16B checksum][1B method][4B LE size_compressed][4B LE size_decompressed][payload] +# size_compressed counts the 9-byte header. The checksum is an unkeyed CityHash128 of everything +# after it, so a peer can compute it: these frames are accepted at default settings, and +# http_native_compression_disable_checksumming_on_decompress is deliberately NOT used. +# +# checksum-for-compressed-block prints CityHash128 of every single-bit mutation of its input, so +# feeding it the body with bit 0 flipped yields CityHash128(body) on the line labelled "0, 0". The +# wire order is low64 then high64, each little-endian, i.e. the reverse of the printed hex. +emit() { # $1 = frame body as hex, checksum excluded + local checksum + checksum=$(python3 -c " +import sys +b = bytearray.fromhex(sys.argv[1]); b[0] ^= 1 +sys.stdout.buffer.write(bytes(b)) +" "$1" | $CLICKHOUSE_BINARY checksum-for-compressed-block | awk -F'\t' '$2 == "0, 0" { print $1; exit }') + python3 -c " +import sys +sys.stdout.buffer.write(bytearray.fromhex(sys.argv[1])[::-1] + bytearray.fromhex(sys.argv[2])) +" "$checksum" "$1" +} + +# Method bytes are from CompressionInfo.h: NONE is 0x02 = 2. +frame() { # $1 = method byte (decimal), $2 = size_decompressed, $3 = payload + emit "$(python3 -c " +import struct, sys +payload = sys.argv[3].encode() +sys.stdout.buffer.write(bytes([int(sys.argv[1])]) + struct.pack('&1 | grep -c '(8) does not match size_decompressed (999)' + +# The comparison is for inequality, so an arm that only ever declares more than the body pins one side +# of it: narrowed to "body shorter than declared", the frame above would still be refused. This one +# declares less instead, and without the check it is not refused at all, it executes its real body. +echo '-- nor may it understate the uncompressed size' +frame 2 3 'SELECT 1' | post 2>&1 | grep -c '(8) does not match size_decompressed (3)' + +# No-regression control: only a NONE-coded frame takes the shortcut, so the column, the marks and +# the primary key are all stored uncompressed here. StripeLog is absent because it compresses its +# whole stream with the default codec and ignores the column codec, so its reads never take it. +echo '-- engines keep working on ordinary data' +${CLICKHOUSE_CLIENT} --query " + DROP TABLE IF EXISTS t_log_bound; + DROP TABLE IF EXISTS t_mt_bound; + + CREATE TABLE t_log_bound (s String CODEC(NONE)) ENGINE = Log; + CREATE TABLE t_mt_bound (k UInt64, s String CODEC(NONE), INDEX idx_s s TYPE minmax GRANULARITY 1) + ENGINE = MergeTree ORDER BY k + SETTINGS index_granularity = 8, compress_marks = 1, compress_primary_key = 1, + min_bytes_for_wide_part = 0, + marks_compression_codec = 'NONE', primary_key_compression_codec = 'NONE'; + + INSERT INTO t_log_bound SELECT repeat('a', 100) FROM numbers(1000); + INSERT INTO t_mt_bound SELECT number, repeat('a', 100) FROM numbers(1000); + + SELECT count(), sum(length(s)) FROM t_log_bound; + SELECT count(), sum(length(s)) FROM t_mt_bound WHERE s LIKE '%a%'; + SELECT count() FROM t_mt_bound WHERE k = 42; + CHECK TABLE t_mt_bound SETTINGS check_query_single_value_result = 1; + + DROP TABLE t_log_bound; + DROP TABLE t_mt_bound; +" diff --git a/tests/queries/0_stateless/04853_attach_as_replicated_keeper_path_macro.reference b/tests/queries/0_stateless/04853_attach_as_replicated_keeper_path_macro.reference new file mode 100644 index 0000000000000..d91901a6aeaf4 --- /dev/null +++ b/tests/queries/0_stateless/04853_attach_as_replicated_keeper_path_macro.reference @@ -0,0 +1,8 @@ +safe_name_converts ReplicatedMergeTree 2 +definition_not_rewritten 1 +as_not_replicated_allows_unsafe_name MergeTree +stored_macro_armed 1 +short_attach OK +short_attach_count 1 +restart_replica OK +restart_replica_count 1 diff --git a/tests/queries/0_stateless/04853_attach_as_replicated_keeper_path_macro.sh b/tests/queries/0_stateless/04853_attach_as_replicated_keeper_path_macro.sh new file mode 100755 index 0000000000000..60ade65ffbc3a --- /dev/null +++ b/tests/queries/0_stateless/04853_attach_as_replicated_keeper_path_macro.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Tags: zookeeper, no-replicated-database, no-ordinary-database, no-shared-merge-tree +# Rows needing a per-copy unique database name, which a .sql file cannot interpolate. + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# The rejection this test guards fires only on a name-based `default_replica_path`, which the test +# config does not set (it is {uuid}-based, and a {uuid} is never path-unsafe). So the treatment row +# lives in tests/integration/test_modify_engine_on_restart/test_unsafe_name.py, which configures +# that setting, and the rows here pin the exemptions the rejection must not disturb. +CLIENT="${CLICKHOUSE_CLIENT} --server_logs_file=/dev/null" + +# Converting a table with a path-safe name still works. Guards the check against rejecting the +# ordinary case: the shipped {uuid} template resolves through the same code path. +${CLIENT} -q "CREATE TABLE c1 (c0 Int) ENGINE = MergeTree ORDER BY c0" +${CLIENT} -q "INSERT INTO c1 VALUES (1), (2)" +${CLIENT} -q "DETACH TABLE c1" +${CLIENT} -q "ATTACH TABLE c1 AS REPLICATED" +${CLIENT} -q "SELECT 'safe_name_converts', engine, (SELECT count() FROM c1) FROM system.tables WHERE database = currentDatabase() AND name = 'c1'" + +# The check only reads the definition, it must not rewrite the engine arguments. The stored +# template stays unexpanded, which is what keeps a metadata file copyable between replicas. +${CLIENT} -q "SELECT 'definition_not_rewritten', create_table_query LIKE '%{uuid}%' FROM system.tables WHERE database = currentDatabase() AND name = 'c1'" +${CLIENT} -q "DROP TABLE c1" + +# The opposite direction strips the path arguments instead of minting one, so an unsafe name is +# irrelevant to it. The path is given explicitly here, so the CREATE itself is legal. +${CLIENT} -q "CREATE TABLE \`c3/unsafe\` (c0 Int) ENGINE = ReplicatedMergeTree('/clickhouse/04853/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/c3', 'r1') ORDER BY c0" +${CLIENT} -q "DETACH TABLE \`c3/unsafe\`" +${CLIENT} -q "ATTACH TABLE \`c3/unsafe\` AS NOT REPLICATED" +${CLIENT} -q "SELECT 'as_not_replicated_allows_unsafe_name', engine FROM system.tables WHERE database = currentDatabase() AND name = 'c3/unsafe'" +${CLIENT} -q "DROP TABLE \`c3/unsafe\`" + +# A table whose STORED path re-expands to a path-unsafe value keeps loading, both through a +# short ATTACH and through SYSTEM RESTART REPLICA. Two arming details: +# * only a CONFIGURED macro survives into metadata unexpanded; a direct {database} is unfolded at +# CREATE, leaving nothing to re-substitute. +# * the DATABASE is renamed rather than the table, because RenamingRestrictions refuses to rename +# a table whose path carries an implicit macro. +# Report the exemption from the client's exit status, not from a grep for one error code: the table +# is registered in the catalog before `startup()` runs on both routes, so a failure there leaves the +# table present and the count assertions below cannot see it. +run_exempt() { + local label=$1 query=$2 out rc + out=$(${CLIENT} -q "$query" 2>&1) + rc=$? + if [ "$rc" = "0" ]; then + echo "$label OK" + else + echo "$label FAILED rc=$rc: $out" + fi +} + +LEGACY_DB="${CLICKHOUSE_DATABASE}_legacy" +${CLIENT} -q "DROP DATABASE IF EXISTS \`${LEGACY_DB}/d\` SYNC" +${CLIENT} -q "DROP DATABASE IF EXISTS \`${LEGACY_DB}\` SYNC" +${CLIENT} -q "CREATE DATABASE \`${LEGACY_DB}\`" +${CLIENT} -q "CREATE TABLE \`${LEGACY_DB}\`.t (c0 Int) ENGINE = ReplicatedMergeTree('{default_path_test}04853legacy', 'r2') ORDER BY c0" +${CLIENT} -q "SELECT 'stored_macro_armed', create_table_query LIKE '%{default_path_test}%' FROM system.tables WHERE database = '${LEGACY_DB}' AND name = 't'" +${CLIENT} -q "RENAME DATABASE \`${LEGACY_DB}\` TO \`${LEGACY_DB}/d\`" +${CLIENT} -q "DETACH TABLE \`${LEGACY_DB}/d\`.t" +run_exempt short_attach "ATTACH TABLE \`${LEGACY_DB}/d\`.t" +${CLIENT} -q "SELECT 'short_attach_count', count() FROM system.tables WHERE database = '${LEGACY_DB}/d' AND name = 't'" +run_exempt restart_replica "SYSTEM RESTART REPLICA \`${LEGACY_DB}/d\`.t" +${CLIENT} -q "SELECT 'restart_replica_count', count() FROM system.tables WHERE database = '${LEGACY_DB}/d' AND name = 't'" +# Re-resolve the path under the original name before dropping: the stored path re-expands {database}, +# so under the new name the table points at a different znode tree than the CREATE made, and dropping +# it there would leave the original tree behind. +${CLIENT} -q "RENAME DATABASE \`${LEGACY_DB}/d\` TO \`${LEGACY_DB}\`" +${CLIENT} -q "DETACH TABLE \`${LEGACY_DB}\`.t" +${CLIENT} -q "ATTACH TABLE \`${LEGACY_DB}\`.t" +${CLIENT} -q "DROP DATABASE \`${LEGACY_DB}\` SYNC" diff --git a/tests/queries/0_stateless/04870_mysql_bit_column_oversized_value.reference b/tests/queries/0_stateless/04870_mysql_bit_column_oversized_value.reference new file mode 100644 index 0000000000000..a8319147f8482 --- /dev/null +++ b/tests/queries/0_stateless/04870_mysql_bit_column_oversized_value.reference @@ -0,0 +1,6 @@ +--- two bytes --- +258 +--- eight bytes --- +72623859790382856 +--- hundred bytes --- +OK: the oversized value was rejected diff --git a/tests/queries/0_stateless/04870_mysql_bit_column_oversized_value.sh b/tests/queries/0_stateless/04870_mysql_bit_column_oversized_value.sh new file mode 100755 index 0000000000000..4624065db0bc4 --- /dev/null +++ b/tests/queries/0_stateless/04870_mysql_bit_column_oversized_value.sh @@ -0,0 +1,271 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: the fast test is built with `ENABLE_LIBRARIES=0`, so it has no MySQL client. + +# The value of a `MYSQL_TYPE_BIT` column used to be copied into an eight-byte stack local with +# `memcpy` using the length from the wire, without checking it. A `BIT` column holds at most +# 64 bits, but nothing bounded what the server actually sent, so a malicious or broken MySQL +# endpoint could overflow that local. The length is now validated. +# +# This starts a fake MySQL server that answers with a `BIT` column and returns, depending on the +# remote table name, a value of two, eight, or a hundred bytes. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +STUB="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}_mysql_stub.py" + +cat > "$STUB" <<'PYEOF' +#!/usr/bin/env python3 +"""A fake MySQL server that answers every `SELECT` with a single `BIT` column. + +The size of the returned value is chosen by the remote table name mentioned in the query, +so that one server can serve all the cases of the test. +""" + +import socket +import struct +import sys +import threading + +PORT = int(sys.argv[1]) + +# The value returned for a query, by the remote table name it mentions. +VALUES = { + "bit_two": bytes([0x01, 0x02]), + "bit_eight": bytes([0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08]), + "bit_oversized": b"A" * 100, +} + +CLIENT_LONG_PASSWORD = 0x00000001 +CLIENT_LONG_FLAG = 0x00000004 +CLIENT_CONNECT_WITH_DB = 0x00000008 +CLIENT_PROTOCOL_41 = 0x00000200 +CLIENT_TRANSACTIONS = 0x00002000 +CLIENT_SECURE_CONNECTION = 0x00008000 +CLIENT_PLUGIN_AUTH = 0x00080000 + +# Deliberately without `CLIENT_SSL`, `CLIENT_COMPRESS` and `CLIENT_DEPRECATE_EOF`, to keep the +# client on the plain, classic path. +CAPABILITIES = ( + CLIENT_LONG_PASSWORD + | CLIENT_LONG_FLAG + | CLIENT_CONNECT_WITH_DB + | CLIENT_PROTOCOL_41 + | CLIENT_TRANSACTIONS + | CLIENT_SECURE_CONNECTION + | CLIENT_PLUGIN_AUTH +) + +SERVER_STATUS_AUTOCOMMIT = 0x0002 + +MYSQL_TYPE_BIT = 0x10 +CHARSET_BINARY = 63 + +COM_QUIT = 0x01 +COM_QUERY = 0x03 + + +def lenenc_int(value): + if value < 0xFB: + return bytes([value]) + if value < 1 << 16: + return b"\xfc" + struct.pack("> 16) & 0xFFFF) + payload += bytes([21]) # Length of the whole scramble, including the terminating zero. + payload += b"\x00" * 10 # Reserved. + payload += bytes(range(9, 21)) + b"\x00" # Second part of the scramble. + payload += b"mysql_native_password\x00" + return bytes(payload) + + +def ok_packet(): + return ( + b"\x00" + + lenenc_int(0) # Affected rows. + + lenenc_int(0) # Last insert id. + + struct.pack("/dev/null; wait ${STUB_PID} 2>/dev/null; rm -f "${STUB}"' EXIT + +# Wait for the fake server to start listening. +for _ in $(seq 1 100); do + if python3 -c " +import socket, sys +sys.exit(0 if socket.socket().connect_ex(('127.0.0.1', ${PORT})) == 0 else 1) +" 2>/dev/null; then + break + fi + sleep 0.1 +done + +${CLICKHOUSE_CLIENT} -q " +DROP TABLE IF EXISTS bit_two; +DROP TABLE IF EXISTS bit_eight; +DROP TABLE IF EXISTS bit_oversized; +CREATE TABLE bit_two (x UInt64) ENGINE = MySQL('127.0.0.1:${PORT}', 'db', 'bit_two', 'user', 'password'); +CREATE TABLE bit_eight (x UInt64) ENGINE = MySQL('127.0.0.1:${PORT}', 'db', 'bit_eight', 'user', 'password'); +CREATE TABLE bit_oversized (x UInt64) ENGINE = MySQL('127.0.0.1:${PORT}', 'db', 'bit_oversized', 'user', 'password'); +" + +# A `BIT` value is sent most significant byte first, and shorter values than eight bytes are normal. +echo "--- two bytes ---" +${CLICKHOUSE_CLIENT} -q "SELECT x FROM bit_two" + +echo "--- eight bytes ---" +${CLICKHOUSE_CLIENT} -q "SELECT x FROM bit_eight" + +# More than eight bytes cannot be a `BIT` value and must be rejected instead of overflowing. +# Before the fix this printed 4702111234474983745, the first eight of the hundred `A` bytes, +# having written the remaining ninety two past the end of the destination. +echo "--- hundred bytes ---" +ERROR=$(${CLICKHOUSE_CLIENT} -q "SELECT x FROM bit_oversized" 2>&1) +if grep -qF "INCORRECT_DATA" <<< "$ERROR" && grep -qE "MySQL sent 100 bytes" <<< "$ERROR" +then + echo "OK: the oversized value was rejected" +else + echo "FAIL: the oversized value was not rejected, got: $ERROR" +fi diff --git a/tests/queries/0_stateless/04905_lwu_drop_race_table_lock.reference b/tests/queries/0_stateless/04905_lwu_drop_race_table_lock.reference new file mode 100644 index 0000000000000..31745adb0b13f --- /dev/null +++ b/tests/queries/0_stateless/04905_lwu_drop_race_table_lock.reference @@ -0,0 +1,3 @@ +update: update=ok drop=ok +alter: update=ok drop=ok +alias: update=ok drop=ok diff --git a/tests/queries/0_stateless/04905_lwu_drop_race_table_lock.sh b/tests/queries/0_stateless/04905_lwu_drop_race_table_lock.sh new file mode 100755 index 0000000000000..dca044d4eb873 --- /dev/null +++ b/tests/queries/0_stateless/04905_lwu_drop_race_table_lock.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-replicated-database, no-shared-merge-tree +# Tag no-fasttest: relies on a failpoint (libfiu). +# Tag no-parallel: the test waits on a server-global pauseable failpoint, so a concurrent copy's +# sink could satisfy this copy's wait and this copy's resume could release that sink. +# FailPointInjection::{enable,disable}FailPoint take only a name, so it cannot be scoped. +# Tag no-replicated-database: the test needs its own Memory database, which a Replicated database +# run cannot host, and DROP must reach the synchronous exclusive-lock path. +# Tag no-shared-merge-tree: the failpoint this test parks on is in the ReplicatedMergeTree sink, +# which a SharedMergeTree table does not go through. + +set -e + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +FP=rmt_pause_before_commit_local_part + +# The failpoint is server-global, so leaving it enabled would park the sink of every later test. +# This has to survive the test failing or being killed part-way through. +function cleanup() +{ + ${CLICKHOUSE_CLIENT} --query "SYSTEM DISABLE FAILPOINT ${FP}" < /dev/null 2>/dev/null || true +} +trap cleanup EXIT + +# A Memory database has no UUID, so DROP takes the table's exclusive lock and removes the data +# inline instead of deferring it to the background cleanup an Atomic database uses. +${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${CLICKHOUSE_DATABASE_1}" < /dev/null +${CLICKHOUSE_CLIENT} --query "CREATE DATABASE ${CLICKHOUSE_DATABASE_1} ENGINE = Memory" < /dev/null + +${CLICKHOUSE_CLIENT} --query "SYSTEM DISABLE FAILPOINT ${FP}" < /dev/null 2>/dev/null || true + +# Every DROP the test's logic depends on pins ignore_drop_queries_probability: the stress runner +# injects 0.2 and clickhouse-client --fake-drop (upgrade check) injects 1, and for a storage that +# keeps data on disk the injection returns success without dropping anything. +function setup_table() +{ + ${CLICKHOUSE_CLIENT} --query " + DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE_1}.t SYNC SETTINGS ignore_drop_queries_probability = 0; + + CREATE TABLE ${CLICKHOUSE_DATABASE_1}.t (id UInt64, c2 String) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/04905_$1/', '1') + ORDER BY id + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1; + + INSERT INTO ${CLICKHOUSE_DATABASE_1}.t SELECT number, 'a' FROM numbers(2); + " < /dev/null +} + +# The DROP has to land between the patch-part rename and the commit. The sink parks there and stays +# parked until this function resumes it, so the window is held open instead of being a timed one the +# drop could miss. +function race_with_drop() +{ + local arm=$1 + shift + local qid="${CLICKHOUSE_DATABASE_1}_${arm}_${RANDOM}${RANDOM}" + local drop_qid="${qid}_drop" + + ${CLICKHOUSE_CLIENT} --query "SYSTEM ENABLE FAILPOINT ${FP}" < /dev/null + + # Everything below reads the armed state as evidence, so an arm that is not actually armed would + # make the rest of the checks vacuous rather than failing. + if [ "$(${CLICKHOUSE_CLIENT} --query " + SELECT enabled FROM system.fail_points WHERE name = '${FP}' + SETTINGS enable_parallel_replicas = 0" < /dev/null 2>/dev/null)" != 1 ]; then + echo "$arm: the commit hook was not armed" + return + fi + + ${CLICKHOUSE_CLIENT} --query_id "$qid" "$@" < /dev/null > /dev/null 2>&1 & + local updater=$! + + # Returns once the sink has parked, so the update still holds the lock it took for the pipeline. + # An update that never reaches the hook would otherwise wait here forever. + # shellcheck disable=SC2086 # CLICKHOUSE_CLIENT carries arguments and must word-split + if ! timeout 60 ${CLICKHOUSE_CLIENT} --query "SYSTEM WAIT FAILPOINT ${FP} PAUSE" < /dev/null; then + ${CLICKHOUSE_CLIENT} --query "SYSTEM DISABLE FAILPOINT ${FP}" < /dev/null 2>/dev/null || true + wait "$updater" 2>/dev/null || true + echo "$arm: the sink never reached the commit hook" + return + fi + + # The wait above returns at once when nothing is parked, so it does not by itself establish that + # the sink reached the hook. The failpoint is one-shot, so it reports itself disabled only after + # something fired it. + if [ "$(${CLICKHOUSE_CLIENT} --query " + SELECT enabled FROM system.fail_points WHERE name = '${FP}' + SETTINGS enable_parallel_replicas = 0" < /dev/null 2>/dev/null)" != 0 ]; then + ${CLICKHOUSE_CLIENT} --query "SYSTEM DISABLE FAILPOINT ${FP}" < /dev/null 2>/dev/null || true + wait "$updater" 2>/dev/null || true + echo "$arm: the sink never parked at the commit hook" + return + fi + + ${CLICKHOUSE_CLIENT} --query_id "$drop_qid" --query " + DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE_1}.t SYNC + SETTINGS ignore_drop_queries_probability = 0" < /dev/null > /dev/null 2>&1 & + local dropper=$! + + # The sink is still parked, so the drop reaches the table lock and blocks on it while the + # window is held open. + sleep 2 + + ${CLICKHOUSE_CLIENT} --query "SYSTEM DISABLE FAILPOINT ${FP}" < /dev/null + + local update=ok + wait "$updater" 2>/dev/null || update=failed + + local drop=ok + wait "$dropper" 2>/dev/null || drop=failed + + # The lock wait is charged to the statement that blocked, so a non-zero value here is this + # drop's own wait on this table and no other writer can supply it. A drop that reached the lock + # only after the commit released it reports zero and never covered the race. + local waited + waited=$(${CLICKHOUSE_CLIENT} --query " + SYSTEM FLUSH LOGS query_log; + SELECT ProfileEvents['RWLockWritersWaitMilliseconds'] FROM system.query_log + WHERE query_id = '${drop_qid}' AND type = 'QueryFinish' AND event_date >= yesterday() + AND current_database = currentDatabase() + ORDER BY event_time_microseconds DESC LIMIT 1 + SETTINGS max_rows_to_read = 0, enable_parallel_replicas = 0" < /dev/null 2>/dev/null) || waited="" + case "$waited" in + '' | *[!0-9]*) waited=0 ;; + esac + + if [ "$waited" -eq 0 ]; then + echo "$arm: the drop did not wait for the table lock" + return + fi + + # A drop that reported success without removing the table took no lock and proved nothing. + if [ "$drop" = ok ] && [ "$(${CLICKHOUSE_CLIENT} --query " + EXISTS ${CLICKHOUSE_DATABASE_1}.t + SETTINGS enable_parallel_replicas = 0" < /dev/null 2>/dev/null)" != 0 ]; then + drop=ignored + fi + + echo "$arm: update=$update drop=$drop" +} + +setup_table update +race_with_drop update --enable_lightweight_update 1 \ + --query "UPDATE ${CLICKHOUSE_DATABASE_1}.t SET c2 = 'xx' WHERE id = 1" + +setup_table alter +race_with_drop alter --enable_lightweight_update 1 --alter_update_mode 'lightweight_force' \ + --query "ALTER TABLE ${CLICKHOUSE_DATABASE_1}.t UPDATE c2 = 'xx' WHERE id = 1" + +# An Alias table resolves a different storage, so the update has to hold the target's lock as well. +setup_table alias +${CLICKHOUSE_CLIENT} --allow_experimental_alias_table_engine 1 --query " + DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE_1}.a SYNC SETTINGS ignore_drop_queries_probability = 0; + CREATE TABLE ${CLICKHOUSE_DATABASE_1}.a ENGINE = Alias('${CLICKHOUSE_DATABASE_1}', 't'); +" < /dev/null +race_with_drop alias --allow_experimental_alias_table_engine 1 --enable_lightweight_update 1 \ + --query "UPDATE ${CLICKHOUSE_DATABASE_1}.a SET c2 = 'xx' WHERE id = 1" + +${CLICKHOUSE_CLIENT} --query "DROP DATABASE ${CLICKHOUSE_DATABASE_1}" < /dev/null diff --git a/tests/queries/0_stateless/04927_substreams_cache_key_collision.reference b/tests/queries/0_stateless/04927_substreams_cache_key_collision.reference new file mode 100644 index 0000000000000..ba91c81af2da0 --- /dev/null +++ b/tests/queries/0_stateless/04927_substreams_cache_key_collision.reference @@ -0,0 +1,184 @@ +[(100),(200)] ('abc',99) +[(300)] ('de',98) +[] ('',97) +4 +Row 1: +────── +c1: ([(100),(200)],1) +c2: ([('abc')],2) +c3: ([('abc')],3) +c4: ([([10,20])],4) +c5: ([(1),(NULL)],5) +c6: ([[(10),(20)]],6) +c7: ([[(10),(20)]],7) +c8: ([(1),NULL],8) +c9: ({'k':[(1),(2)]},9) +c10: (('abc',99),10) +c11: (([10,20],99),11) +c12: ((5,7),12) +Row 1: +────── +c1.x: [(100),(200)] +c2.x: [('abc')] +c3.x: [('abc')] +c4.x: [([10,20])] +c5.x: [(1),(NULL)] +c6.x: [[(10),(20)]] +c7.x: [[(10),(20)]] +c8.x: [(1),NULL] +c9.x: {'k':[(1),(2)]} +c10.x: ('abc',99) +c11.x: ([10,20],99) +c12.x: (5,7) +Row 1: +────── +c1: ([(100),(200)],1) +c2: ([('abc')],2) +c3: ([('abc')],3) +c4: ([([10,20])],4) +c5: ([(1),(NULL)],5) +c6: ([[(10),(20)]],6) +c7: ([[(10),(20)]],7) +c8: ([(1),NULL],8) +c9: ({'k':[(1),(2)]},9) +c10: (('abc',99),10) +c11: (([10,20],99),11) +c12: ((5,7),12) +Row 1: +────── +c1.x: [(100),(200)] +c2.x: [('abc')] +c3.x: [('abc')] +c4.x: [([10,20])] +c5.x: [(1),(NULL)] +c6.x: [[(10),(20)]] +c7.x: [[(10),(20)]] +c8.x: [(1),NULL] +c9.x: {'k':[(1),(2)]} +c10.x: ('abc',99) +c11.x: ([10,20],99) +c12.x: (5,7) +Row 1: +────── +c1: ('{"object_shared_data":{"0":{"size0":1}}}',1) +c2: ('{"a":[1,2],"a":{"size0":7}}',2) +c3: ('{"a":1,"a":{"null":7}}',3) +c4: ('{"a":"abc","a":{"size":7}}',4) +c5: ('{"a":1,"a":{"Int64":7}}',5) +c6: ('{"a":1,"a":{"Int64":7}}',6) +c7: ('{"a":{"b":0},"a":{"b":1}}',7) +Row 1: +────── +c1.x: { + "object_shared_data": { + "0": { + "size0": 1 + } + } +} +c2.x: { + "a": [1,2], + "a": { + "size0": 7 + } +} +c3.x: { + "a": 1, + "a": { + "null": 7 + } +} +c4.x: { + "a": "abc", + "a": { + "size": 7 + } +} +c5.x: { + "a": 1, + "a": { + "Int64": 7 + } +} +c6.x: { + "a": 1, + "a": { + "Int64": 7 + } +} +c7.x: { + "a": { + "b": 0 + }, + "a": { + "b": 1 + } +} +Row 1: +────── +c1: ('{"object_shared_data":{"0":{"size0":1}}}',1) +c2: ('{"a":[1,2],"a":{"size0":7}}',2) +c3: ('{"a":1,"a":{"null":7}}',3) +c4: ('{"a":"abc","a":{"size":7}}',4) +c5: ('{"a":1,"a":{"Int64":7}}',5) +c6: ('{"a":1,"a":{"Int64":7}}',6) +c7: ('{"a":{"b":0},"a":{"b":1}}',7) +Row 1: +────── +c1.x: { + "object_shared_data": { + "0": { + "size0": 1 + } + } +} +c2.x: { + "a": [1,2], + "a": { + "size0": 7 + } +} +c3.x: { + "a": 1, + "a": { + "null": 7 + } +} +c4.x: { + "a": "abc", + "a": { + "size": 7 + } +} +c5.x: { + "a": 1, + "a": { + "Int64": 7 + } +} +c6.x: { + "a": 1, + "a": { + "Int64": 7 + } +} +c7.x: { + "a": { + "b": 0 + }, + "a": { + "b": 1 + } +} +12495667661173940433 30 14365225858895003680 +5579088841599825588 20 3931234904921333272 +11007181315820846960 20 16997049742924259240 3194560342723893250 +9842334541689766390 20 3194560342723893250 +1328710051673613252 0 11053694533033933970 +9067169503920147426 190 +1 +9469320946673274046 2 13032090342589669733 2 2268159345449755554 10 +[(100),(200)] ('abc',99) +[(300)] ('de',98) +[(100),(200)] ('abc',99) +[(300)] ('de',98) diff --git a/tests/queries/0_stateless/04927_substreams_cache_key_collision.sql b/tests/queries/0_stateless/04927_substreams_cache_key_collision.sql new file mode 100644 index 0000000000000..bd31cc606f7f9 --- /dev/null +++ b/tests/queries/0_stateless/04927_substreams_cache_key_collision.sql @@ -0,0 +1,167 @@ +-- Two streams of one column whose files differ (`c.size0` and `c%2Esize0` for Array(Tuple(`size0` UInt64))) +-- used to share a substreams cache slot, so one of them got the other's column. The data on disk is correct. +-- The wide reader uses the cache for every read, the compact reader only for subcolumn reads. + +SET allow_experimental_nullable_tuple_type = 1; + +DROP TABLE IF EXISTS t_cache_key; + +CREATE TABLE t_cache_key (c Array(Tuple(`size0` UInt64)), d Tuple(`a` String, `a.size` UInt64)) +ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; +INSERT INTO t_cache_key VALUES ([(100), (200)], ('abc', 99)), ([(300)], ('de', 98)), ([], ('', 97)); +SELECT c, d FROM t_cache_key; + +-- A merge reads the parts the same way. +INSERT INTO t_cache_key VALUES ([(400)], ('f', 96)); +OPTIMIZE TABLE t_cache_key FINAL; +SELECT count() FROM t_cache_key; + +DROP TABLE t_cache_key; + +DROP TABLE IF EXISTS t_cache_wide; +DROP TABLE IF EXISTS t_cache_compact; + +-- Every column is wrapped in a Tuple: the compact reader allocates a substreams cache only for subcolumn +-- reads of a part that has substream marks, hence `write_marks_for_substreams_in_compact_parts` below. +CREATE TABLE t_cache_wide +( + -- A Tuple element named like the array sizes of an enclosing Array. + c1 Tuple(x Array(Tuple(`size0` UInt64)), y UInt8), + c2 Tuple(x Array(Tuple(`size0` String)), y UInt8), + c3 Tuple(x Array(Tuple(`size0` LowCardinality(String))), y UInt8), + c4 Tuple(x Array(Tuple(`size0` Array(UInt64))), y UInt8), + c5 Tuple(x Array(Tuple(`size0` Nullable(UInt64))), y UInt8), + c6 Tuple(x Array(Array(Tuple(`size0` UInt64))), y UInt8), + c7 Tuple(x Array(Array(Tuple(`size1` UInt64))), y UInt8), + c8 Tuple(x Array(Nullable(Tuple(`size0` Int64))), y UInt8), + c9 Tuple(x Map(String, Array(Tuple(`size1` UInt64))), y UInt8), + -- A Tuple element named like an automatic subcolumn of a sibling element. + c10 Tuple(x Tuple(`a` String, `a.size` UInt64), y UInt8), + c11 Tuple(x Tuple(`a` Array(UInt64), `a.size0` UInt64), y UInt8), + c12 Tuple(x Tuple(`a` Nullable(UInt64), `a.null` UInt8), y UInt8) +) +ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; + +CREATE TABLE t_cache_compact +( + c1 Tuple(x Array(Tuple(`size0` UInt64)), y UInt8), + c2 Tuple(x Array(Tuple(`size0` String)), y UInt8), + c3 Tuple(x Array(Tuple(`size0` LowCardinality(String))), y UInt8), + c4 Tuple(x Array(Tuple(`size0` Array(UInt64))), y UInt8), + c5 Tuple(x Array(Tuple(`size0` Nullable(UInt64))), y UInt8), + c6 Tuple(x Array(Array(Tuple(`size0` UInt64))), y UInt8), + c7 Tuple(x Array(Array(Tuple(`size1` UInt64))), y UInt8), + c8 Tuple(x Array(Nullable(Tuple(`size0` Int64))), y UInt8), + c9 Tuple(x Map(String, Array(Tuple(`size1` UInt64))), y UInt8), + c10 Tuple(x Tuple(`a` String, `a.size` UInt64), y UInt8), + c11 Tuple(x Tuple(`a` Array(UInt64), `a.size0` UInt64), y UInt8), + c12 Tuple(x Tuple(`a` Nullable(UInt64), `a.null` UInt8), y UInt8) +) +ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = '10G', min_rows_for_wide_part = 1000000000, write_marks_for_substreams_in_compact_parts = 1; + +INSERT INTO t_cache_wide SELECT ([tuple(100), tuple(200)], 1), ([tuple('abc')], 2), ([tuple('abc')], 3), ([tuple([10, 20])], 4), ([tuple(1), tuple(NULL)], 5), ([[tuple(10), tuple(20)]], 6), ([[tuple(10), tuple(20)]], 7), ([tuple(1), NULL], 8), (map('k', [tuple(1), tuple(2)]), 9), (('abc', 99), 10), (([10, 20], 99), 11), ((5, 7), 12); +INSERT INTO t_cache_compact SELECT ([tuple(100), tuple(200)], 1), ([tuple('abc')], 2), ([tuple('abc')], 3), ([tuple([10, 20])], 4), ([tuple(1), tuple(NULL)], 5), ([[tuple(10), tuple(20)]], 6), ([[tuple(10), tuple(20)]], 7), ([tuple(1), NULL], 8), (map('k', [tuple(1), tuple(2)]), 9), (('abc', 99), 10), (([10, 20], 99), 11), ((5, 7), 12); + +SELECT * FROM t_cache_wide FORMAT Vertical; +SELECT c1.x, c2.x, c3.x, c4.x, c5.x, c6.x, c7.x, c8.x, c9.x, c10.x, c11.x, c12.x FROM t_cache_wide FORMAT Vertical; +SELECT * FROM t_cache_compact FORMAT Vertical; +SELECT c1.x, c2.x, c3.x, c4.x, c5.x, c6.x, c7.x, c8.x, c9.x, c10.x, c11.x, c12.x FROM t_cache_compact FORMAT Vertical; + +DROP TABLE t_cache_wide; +DROP TABLE t_cache_compact; + +-- The same with JSON typed paths named like a substream of a sibling path or like an internal stream. +DROP TABLE IF EXISTS t_cache_json_wide; +DROP TABLE IF EXISTS t_cache_json_compact; + +CREATE TABLE t_cache_json_wide +( + c1 Tuple(x JSON(`object_shared_data.0.size0` Int64), y UInt8), + c2 Tuple(x JSON(`a` Array(Int64), `a.size0` Int64), y UInt8), + c3 Tuple(x JSON(`a` Nullable(Int64), `a.null` Int64), y UInt8), + c4 Tuple(x JSON(`a` String, `a.size` Int64), y UInt8), + c5 Tuple(x JSON(`a` Dynamic, `a.Int64` Int64), y UInt8), + c6 Tuple(x JSON(`a` Variant(Int64, String), `a.Int64` Int64), y UInt8), + c7 Tuple(x JSON(`a` JSON(`b` Int64), `a.b` Int64), y UInt8) +) +ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; + +CREATE TABLE t_cache_json_compact +( + c1 Tuple(x JSON(`object_shared_data.0.size0` Int64), y UInt8), + c2 Tuple(x JSON(`a` Array(Int64), `a.size0` Int64), y UInt8), + c3 Tuple(x JSON(`a` Nullable(Int64), `a.null` Int64), y UInt8), + c4 Tuple(x JSON(`a` String, `a.size` Int64), y UInt8), + c5 Tuple(x JSON(`a` Dynamic, `a.Int64` Int64), y UInt8), + c6 Tuple(x JSON(`a` Variant(Int64, String), `a.Int64` Int64), y UInt8), + c7 Tuple(x JSON(`a` JSON(`b` Int64), `a.b` Int64), y UInt8) +) +ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = '10G', min_rows_for_wide_part = 1000000000, write_marks_for_substreams_in_compact_parts = 1; + +INSERT INTO t_cache_json_wide VALUES (('{"object_shared_data.0.size0" : 1}', 1), ('{"a" : [1, 2], "a.size0" : 7}', 2), ('{"a" : 1, "a.null" : 7}', 3), ('{"a" : "abc", "a.size" : 7}', 4), ('{"a" : 1, "a.Int64" : 7}', 5), ('{"a" : 1, "a.Int64" : 7}', 6), ('{"a" : {"b" : 1}}', 7)); +INSERT INTO t_cache_json_compact VALUES (('{"object_shared_data.0.size0" : 1}', 1), ('{"a" : [1, 2], "a.size0" : 7}', 2), ('{"a" : 1, "a.null" : 7}', 3), ('{"a" : "abc", "a.size" : 7}', 4), ('{"a" : 1, "a.Int64" : 7}', 5), ('{"a" : 1, "a.Int64" : 7}', 6), ('{"a" : {"b" : 1}}', 7)); + +SELECT * FROM t_cache_json_wide FORMAT Vertical; +SELECT c1.x, c2.x, c3.x, c4.x, c5.x, c6.x, c7.x FROM t_cache_json_wide FORMAT Vertical; +SELECT * FROM t_cache_json_compact FORMAT Vertical; +SELECT c1.x, c2.x, c3.x, c4.x, c5.x, c6.x, c7.x FROM t_cache_json_compact FORMAT Vertical; + +DROP TABLE t_cache_json_wide; +DROP TABLE t_cache_json_compact; + +-- Whole column and subcolumn reads must keep the same key: the wide reader keys streams by file name, so +-- without sharing the second read continues from where the first stopped. Needs several granules. +DROP TABLE IF EXISTS t_cache_sharing; + +SET flatten_nested = 0; + +CREATE TABLE t_cache_sharing +( + i UInt64, + arr Array(Nullable(UInt64)), + arr2 Array(Array(String)), + m Map(String, UInt64), + n Nested(a UInt64, b String), + s Nullable(String), + j JSON(`p` Int64) +) +ENGINE = MergeTree ORDER BY i SETTINGS min_bytes_for_wide_part = 0, index_granularity = 3; + +INSERT INTO t_cache_sharing SELECT number, range(number % 4), [[toString(number)]], map('k', number), [(number, toString(number))], toString(number), '{"p" : ' || toString(number) || '}' FROM numbers(20); + +SELECT sum(cityHash64(arr)), sum(arr.size0), sum(cityHash64(arr.null)) FROM t_cache_sharing; +SELECT sum(cityHash64(arr2)), sum(arr2.size0), sum(cityHash64(arr2.size1)) FROM t_cache_sharing; +SELECT sum(cityHash64(m)), sum(m.size0), sum(cityHash64(m.keys)), sum(cityHash64(m.values)) FROM t_cache_sharing; +SELECT sum(cityHash64(n)), sum(n.size0), sum(cityHash64(n.a)) FROM t_cache_sharing; +SELECT sum(cityHash64(s)), sum(s.null), sum(cityHash64(s.size)) FROM t_cache_sharing; +SELECT sum(cityHash64(j)), sum(j.p) FROM t_cache_sharing; + +DROP TABLE t_cache_sharing; + +-- Sparse elements are a separate substream, which the key must keep distinct from the column itself. +DROP TABLE IF EXISTS t_cache_sparse; + +CREATE TABLE t_cache_sparse (i UInt64, s String, arr Array(UInt64), t Tuple(a UInt64, b String)) +ENGINE = MergeTree ORDER BY i +SETTINGS min_bytes_for_wide_part = 0, ratio_of_defaults_for_sparse_serialization = 0.1, index_granularity = 3; + +INSERT INTO t_cache_sparse SELECT number, if(number % 10 = 0, 'x', ''), if(number % 10 = 0, [number], []), (if(number % 10 = 0, number, 0), '') FROM numbers(20); + +SELECT sum(serialization_kind = 'Sparse') > 0 FROM system.parts_columns WHERE database = currentDatabase() AND table = 't_cache_sparse' AND active; +SELECT sum(cityHash64(s)), sum(s.size), sum(cityHash64(arr)), sum(arr.size0), sum(cityHash64(t)), sum(t.a) FROM t_cache_sparse; + +DROP TABLE t_cache_sparse; + +-- StorageLog and StorageTinyLog use the same cache. +DROP TABLE IF EXISTS t_cache_log; +DROP TABLE IF EXISTS t_cache_tiny_log; + +CREATE TABLE t_cache_log (c Array(Tuple(`size0` UInt64)), d Tuple(`a` String, `a.size` UInt64)) ENGINE = Log; +CREATE TABLE t_cache_tiny_log (c Array(Tuple(`size0` UInt64)), d Tuple(`a` String, `a.size` UInt64)) ENGINE = TinyLog; +INSERT INTO t_cache_log VALUES ([(100), (200)], ('abc', 99)), ([(300)], ('de', 98)); +INSERT INTO t_cache_tiny_log VALUES ([(100), (200)], ('abc', 99)), ([(300)], ('de', 98)); +SELECT c, d FROM t_cache_log ORDER BY c; +SELECT c, d FROM t_cache_tiny_log ORDER BY c; + +DROP TABLE t_cache_log; +DROP TABLE t_cache_tiny_log; diff --git a/tests/queries/0_stateless/04930_jit_compiled_if_date_to_datetime.reference b/tests/queries/0_stateless/04930_jit_compiled_if_date_to_datetime.reference new file mode 100644 index 0000000000000..18edb0c19e526 --- /dev/null +++ b/tests/queries/0_stateless/04930_jit_compiled_if_date_to_datetime.reference @@ -0,0 +1,17 @@ +Date32 + DateTime64(0) 1 +Date32 + DateTime64(3) 1 +Date32 + DateTime64(6) 1 +Date32 + DateTime64(9) 1 +Date + DateTime64(3) 1 +Date32 + DateTime 1 +Date + DateTime 1 +Date32 + Date 1 +branch order swapped 1 +date as a column 1 +multiIf 1 +CASE 1 +Nullable(Date32) + DateTime64(3) 1 +post-1970 date 1 +absolute value ['1900-01-01 00:00:00.000','1970-01-01 00:00:00.000'] +mixed-unit shape is not compiled 1 +same-unit shape is still compiled 1 diff --git a/tests/queries/0_stateless/04930_jit_compiled_if_date_to_datetime.sql b/tests/queries/0_stateless/04930_jit_compiled_if_date_to_datetime.sql new file mode 100644 index 0000000000000..4882d4347a38b --- /dev/null +++ b/tests/queries/0_stateless/04930_jit_compiled_if_date_to_datetime.sql @@ -0,0 +1,72 @@ +-- Tags: no-fasttest, no-msan +-- no-fasttest: the Fast test build has no embedded compiler (ENABLE_EMBEDDED_COMPILER defaults to +-- ENABLE_LIBRARIES, which that build sets to 0), so nothing is ever JIT-compiled and the +-- "same-unit shape is still compiled" assertion cannot hold. +-- no-msan: the MSan build also disables the embedded compiler (contrib/llvm-project-cmake/CMakeLists.txt), +-- so the same assertion cannot hold there. + +SET session_timezone = 'UTC'; +SET min_count_to_compile_expression = 0; + +SELECT 'Date32 + DateTime64(0)', (SELECT groupArray(if(number % 2 = 0, toDate32('1900-01-01'), toDateTime64('1970-01-01', 0))) FROM numbers(2) SETTINGS compile_expressions = 1) + = (SELECT groupArray(if(number % 2 = 0, toDate32('1900-01-01'), toDateTime64('1970-01-01', 0))) FROM numbers(2) SETTINGS compile_expressions = 0); + +SELECT 'Date32 + DateTime64(3)', (SELECT groupArray(if(number % 2 = 0, toDate32('1900-01-01'), toDateTime64('1970-01-01', 3))) FROM numbers(2) SETTINGS compile_expressions = 1) + = (SELECT groupArray(if(number % 2 = 0, toDate32('1900-01-01'), toDateTime64('1970-01-01', 3))) FROM numbers(2) SETTINGS compile_expressions = 0); + +SELECT 'Date32 + DateTime64(6)', (SELECT groupArray(if(number % 2 = 0, toDate32('1900-01-01'), toDateTime64('1970-01-01', 6))) FROM numbers(2) SETTINGS compile_expressions = 1) + = (SELECT groupArray(if(number % 2 = 0, toDate32('1900-01-01'), toDateTime64('1970-01-01', 6))) FROM numbers(2) SETTINGS compile_expressions = 0); + +SELECT 'Date32 + DateTime64(9)', (SELECT groupArray(if(number % 2 = 0, toDate32('1900-01-01'), toDateTime64('1970-01-01', 9))) FROM numbers(2) SETTINGS compile_expressions = 1) + = (SELECT groupArray(if(number % 2 = 0, toDate32('1900-01-01'), toDateTime64('1970-01-01', 9))) FROM numbers(2) SETTINGS compile_expressions = 0); + +SELECT 'Date + DateTime64(3)', (SELECT groupArray(if(number % 2 = 0, toDate('2000-01-01'), toDateTime64('1970-01-01', 3))) FROM numbers(2) SETTINGS compile_expressions = 1) + = (SELECT groupArray(if(number % 2 = 0, toDate('2000-01-01'), toDateTime64('1970-01-01', 3))) FROM numbers(2) SETTINGS compile_expressions = 0); + +SELECT 'Date32 + DateTime', (SELECT groupArray(if(number % 2 = 0, toDate32('1900-01-01'), toDateTime('1970-01-01'))) FROM numbers(2) SETTINGS compile_expressions = 1) + = (SELECT groupArray(if(number % 2 = 0, toDate32('1900-01-01'), toDateTime('1970-01-01'))) FROM numbers(2) SETTINGS compile_expressions = 0); + +SELECT 'Date + DateTime', (SELECT groupArray(if(number % 2 = 0, toDate('2000-01-01'), toDateTime('1970-01-01'))) FROM numbers(2) SETTINGS compile_expressions = 1) + = (SELECT groupArray(if(number % 2 = 0, toDate('2000-01-01'), toDateTime('1970-01-01'))) FROM numbers(2) SETTINGS compile_expressions = 0); + +SELECT 'Date32 + Date', (SELECT groupArray(if(number % 2 = 0, toDate32('1900-01-01'), toDate('1970-01-01'))) FROM numbers(2) SETTINGS compile_expressions = 1) + = (SELECT groupArray(if(number % 2 = 0, toDate32('1900-01-01'), toDate('1970-01-01'))) FROM numbers(2) SETTINGS compile_expressions = 0); + +SELECT 'branch order swapped', (SELECT groupArray(if(number % 2 = 0, toDateTime64('1970-01-01', 3), toDate32('1900-01-01'))) FROM numbers(2) SETTINGS compile_expressions = 1) + = (SELECT groupArray(if(number % 2 = 0, toDateTime64('1970-01-01', 3), toDate32('1900-01-01'))) FROM numbers(2) SETTINGS compile_expressions = 0); + +SELECT 'date as a column', (SELECT groupArray(if(number % 2 = 0, materialize(toDate32('1900-01-01')), toDateTime64('1970-01-01', 3))) FROM numbers(2) SETTINGS compile_expressions = 1) + = (SELECT groupArray(if(number % 2 = 0, materialize(toDate32('1900-01-01')), toDateTime64('1970-01-01', 3))) FROM numbers(2) SETTINGS compile_expressions = 0); + +SELECT 'multiIf', (SELECT groupArray(multiIf(number % 3 = 0, toDate32('1900-01-01'), number % 3 = 1, toDateTime64('1980-01-01', 3), toDateTime64('1990-01-01', 3))) FROM numbers(3) SETTINGS compile_expressions = 1) + = (SELECT groupArray(multiIf(number % 3 = 0, toDate32('1900-01-01'), number % 3 = 1, toDateTime64('1980-01-01', 3), toDateTime64('1990-01-01', 3))) FROM numbers(3) SETTINGS compile_expressions = 0); + +SELECT 'CASE', (SELECT groupArray(CASE WHEN number % 2 = 0 THEN toDate32('1900-01-01') ELSE toDateTime64('1970-01-01', 3) END) FROM numbers(2) SETTINGS compile_expressions = 1) + = (SELECT groupArray(CASE WHEN number % 2 = 0 THEN toDate32('1900-01-01') ELSE toDateTime64('1970-01-01', 3) END) FROM numbers(2) SETTINGS compile_expressions = 0); + +SELECT 'Nullable(Date32) + DateTime64(3)', (SELECT groupArray(if(number % 2 = 0, toNullable(toDate32('1900-01-01')), toDateTime64('1970-01-01', 3))) FROM numbers(2) SETTINGS compile_expressions = 1) + = (SELECT groupArray(if(number % 2 = 0, toNullable(toDate32('1900-01-01')), toDateTime64('1970-01-01', 3))) FROM numbers(2) SETTINGS compile_expressions = 0); + +SELECT 'post-1970 date', (SELECT groupArray(if(number % 2 = 0, toDate32('2000-01-01'), toDateTime64('1970-01-01', 3))) FROM numbers(2) SETTINGS compile_expressions = 1) + = (SELECT groupArray(if(number % 2 = 0, toDate32('2000-01-01'), toDateTime64('1970-01-01', 3))) FROM numbers(2) SETTINGS compile_expressions = 0); + +SELECT 'absolute value', groupArray(if(number % 2 = 0, toDate32('1900-01-01'), toDateTime64('1970-01-01', 3))) FROM numbers(2) SETTINGS compile_expressions = 1; + +-- The rows above compare compiled against interpreted, so they would all pass if compilation stopped +-- happening. These two pin which shapes compile. CompiledFunctionExecute counts executions of an +-- already-compiled node, so it does not care whether the global compiled cache was already warm. +SELECT groupArray(if(number % 2 = 0, toDate32('1900-01-01'), toDateTime64('1970-01-01', 3))) FROM numbers(2) + SETTINGS compile_expressions = 1, log_comment = '04930_mixed_unit' FORMAT Null; + +SELECT groupArray(if(number % 2 = 0, toDate32('1900-01-01'), toDate('1970-01-01'))) FROM numbers(2) + SETTINGS compile_expressions = 1, log_comment = '04930_same_unit' FORMAT Null; + +SYSTEM FLUSH LOGS query_log; + +SELECT 'mixed-unit shape is not compiled', ProfileEvents['CompiledFunctionExecute'] = 0 FROM system.query_log + WHERE current_database = currentDatabase() AND log_comment = '04930_mixed_unit' AND type = 'QueryFinish' + ORDER BY event_time_microseconds DESC LIMIT 1; + +SELECT 'same-unit shape is still compiled', ProfileEvents['CompiledFunctionExecute'] > 0 FROM system.query_log + WHERE current_database = currentDatabase() AND log_comment = '04930_same_unit' AND type = 'QueryFinish' + ORDER BY event_time_microseconds DESC LIMIT 1; diff --git a/tests/queries/0_stateless/04936_detached_serialization_from_client_native_protocol.python b/tests/queries/0_stateless/04936_detached_serialization_from_client_native_protocol.python new file mode 100755 index 0000000000000..cd620dfafefab --- /dev/null +++ b/tests/queries/0_stateless/04936_detached_serialization_from_client_native_protocol.python @@ -0,0 +1,315 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Regression test for a post-auth type confusion over the native TCP protocol. +# +# The serialization kind of every column of a block is chosen by the peer that sends the +# block. `Kind::DETACHED` is an internal transport representation for parallel blocks +# marshalling: it produces a `ColumnBLOB` holding raw peer-supplied bytes while the block +# keeps the declared type of the column. A client that selects it for an external table +# therefore puts a column that is not a `ColumnMap` into a block declared as `Map`, and +# every later `assert_cast` (a plain `static_cast` in release builds) +# reinterprets attacker-controlled bytes. +# +# `NativeReader` must accept `DETACHED` only from a peer whose pipeline unmarshalls blobs, +# which never includes a client, and must reject kind stacks that no writer can produce. +# All scenarios below expect INCORRECT_DATA (code 117) and a server that stays alive. +# +# The client speaks the raw native protocol pinned to revision 54482 +# (DBMS_MIN_REVISION_WITH_REPLICATED_SERIALIZATION), so the set of exchanged, +# revision-gated fields is fixed and does not drift. + +import os +import socket +import struct +import uuid + +CLICKHOUSE_HOST = os.environ.get("CLICKHOUSE_HOST", "127.0.0.1") +CLICKHOUSE_PORT = int(os.environ.get("CLICKHOUSE_PORT_TCP", "9000")) +CLICKHOUSE_DATABASE = os.environ.get("CLICKHOUSE_DATABASE", "default") +CLIENT_NAME = "detached serialization poc" + +REVISION = 54482 + +# KindStackBinarySerializationType +KIND_STACK_DEFAULT = 0 +KIND_STACK_DETACHED = 2 +KIND_STACK_DETACHED_OVER_SPARSE = 3 +KIND_STACK_COMBINATION = 5 + +# ISerialization::Kind +KIND_DEFAULT = 0 +KIND_SPARSE = 1 +KIND_DETACHED = 2 +KIND_REPLICATED = 3 + + +def writeVarUInt(x, ba): + for _ in range(0, 9): + byte = x & 0x7F + if x > 0x7F: + byte |= 0x80 + ba.append(byte) + x >>= 7 + if x == 0: + return + + +def writeStringBinary(s, ba): + b = bytes(s, "utf-8") if isinstance(s, str) else s + writeVarUInt(len(b), ba) + ba.extend(b) + + +def readStrict(s, size=1): + res = bytearray() + while size: + cur = s.recv(size) + if not cur: + raise EOFError("Connection closed by server") + size -= len(cur) + res.extend(cur) + return res + + +def readUInt(s, size=1): + res = readStrict(s, size) + val = 0 + for i in range(len(res)): + val += res[i] << (i * 8) + return val + + +def readUInt32(s): + return readUInt(s, 4) + + +def readVarUInt(s): + x = 0 + for i in range(9): + byte = readStrict(s)[0] + x |= (byte & 0x7F) << (7 * i) + if not byte & 0x80: + return x + return x + + +def readStringBinary(s): + size = readVarUInt(s) + return readStrict(s, size).decode("utf-8") + + +def sendHello(s): + ba = bytearray() + writeVarUInt(0, ba) # Hello + writeStringBinary(CLIENT_NAME, ba) + writeVarUInt(24, ba) # major + writeVarUInt(9, ba) # minor + writeVarUInt(REVISION, ba) + writeStringBinary(CLICKHOUSE_DATABASE, ba) # database + writeStringBinary("default", ba) # user + writeStringBinary("", ba) # password + s.sendall(ba) + + +def receiveHello(s): + assert readVarUInt(s) == 0 # Hello + readStringBinary(s) # server name + readVarUInt(s) # major + readVarUInt(s) # minor + readVarUInt(s) # revision + readVarUInt(s) # parallel replicas protocol version (>= 54471) + readStringBinary(s) # timezone (>= 54058) + readStringBinary(s) # display name (>= 54372) + readVarUInt(s) # version patch (>= 54401) + readStringBinary(s) # proto_send chunked (>= 54470) + readStringBinary(s) # proto_recv chunked (>= 54470) + for _ in range(readVarUInt(s)): # password complexity rules (>= 54461) + readStringBinary(s) + readStringBinary(s) + readStrict(s, 8) # nonce, UInt64 (>= 54462) + # Server settings in STRINGS_WITH_FLAGS format: (name, flags, value)* terminated + # by an empty name (>= 54474). + while True: + if readStringBinary(s) == "": # setting name + break + readVarUInt(s) # flags + readStringBinary(s) # value + readVarUInt(s) # query plan serialization version (>= 54477) + readVarUInt(s) # cluster function protocol version (>= 54479) + + +def sendAddendum(s): + ba = bytearray() + writeStringBinary("", ba) # quota key (>= 54458) + writeStringBinary("notchunked", ba) # proto_send chunked (>= 54470) + writeStringBinary("notchunked", ba) # proto_recv chunked (>= 54470) + writeVarUInt(0, ba) # parallel replicas protocol version (>= 54471) + s.sendall(ba) + + +def serializeClientInfo(ba, query_id): + ba.append(1) # INITIAL_QUERY + writeStringBinary("default", ba) # initial_user + writeStringBinary(query_id, ba) # initial_query_id + writeStringBinary("127.0.0.1:9000", ba) # initial_address + ba.extend([0] * 8) # initial_query_start_time_microseconds (>= 54449) + ba.append(1) # interface = TCP + writeStringBinary("os_user", ba) + writeStringBinary("client_hostname", ba) + writeStringBinary(CLIENT_NAME, ba) + writeVarUInt(24, ba) # client major + writeVarUInt(9, ba) # client minor + writeVarUInt(REVISION, ba) # client tcp protocol version + writeStringBinary("", ba) # quota key (>= 54060) + writeVarUInt(0, ba) # distributed_depth (>= 54448) + writeVarUInt(1, ba) # client version patch (>= 54401) + ba.append(0) # opentelemetry: no trace id (>= 54442) + writeVarUInt(0, ba) # parallel replicas: collaborate_with_initiator (>= 54453) + writeVarUInt(0, ba) # parallel replicas: obsolete count + writeVarUInt(0, ba) # parallel replicas: number_of_current_replica + writeVarUInt(0, ba) # script query number (>= 54475) + writeVarUInt(0, ba) # script line number (>= 54475) + ba.append(0) # jwt: none (>= 54476) + + +def sendQuery(s, query): + ba = bytearray() + query_id = uuid.uuid4().hex + writeVarUInt(1, ba) # Query + writeStringBinary(query_id, ba) + serializeClientInfo(ba, query_id) + writeStringBinary("", ba) # empty per-query settings + writeStringBinary("", ba) # interserver externally granted roles (>= 54472) + writeStringBinary("", ba) # interserver secret (>= 54441) + writeVarUInt(2, ba) # stage = Complete + ba.append(0) # no compression + writeStringBinary(query, ba) + writeStringBinary("", ba) # query parameters terminator (>= 54459) + s.sendall(ba) + + +def serializeBlockInfo(ba): + writeVarUInt(1, ba) # field 1 + ba.append(0) # is_overflows = false + writeVarUInt(2, ba) # field 2 + ba.extend(struct.pack(" inside mapConcat. + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.settimeout(30) + s.connect((CLICKHOUSE_HOST, CLICKHOUSE_PORT)) + sendHello(s) + receiveHello(s) + sendAddendum(s) + sendQuery(s, "SELECT mapConcat(x) FROM e") + s.sendall(block) + expectIncorrectData(s, must_contain) + print("{}: caught expected exception INCORRECT_DATA".format(name)) + + +def main(): + # The reported vector: a Map column whose runtime representation becomes a ColumnBLOB. + runScenario( + "detached", + externalTableBlock("Map(UInt64, UInt64)", bytes([KIND_STACK_DETACHED])), + "Unexpected serialization kind Detached", + ) + + # The same, reached through the dedicated Detached-over-Sparse encoding. + runScenario( + "detached over sparse", + externalTableBlock("UInt64", bytes([KIND_STACK_DETACHED_OVER_SPARSE])), + "Unexpected serialization kind Detached", + ) + + # The same, reached through the generic kind stack encoding. + runScenario( + "detached in a combination", + externalTableBlock("Map(UInt64, UInt64)", combinationKindStack([KIND_DEFAULT, KIND_DETACHED])), + "Unexpected serialization kind Detached", + ) + + # Kind stacks are per tuple element, so a blob can also be requested for a nested column. + tuple_block = externalTableBlock("Tuple(UInt64, UInt64)", bytes([KIND_STACK_DEFAULT])) + tuple_block += bytes([KIND_STACK_DETACHED]) # element 0 + tuple_block += bytes([KIND_STACK_DEFAULT]) # element 1 + runScenario("detached tuple element", tuple_block, "Unexpected serialization kind Detached") + + # A repeated kind nests a column inside its own layout: a ColumnSparse whose values column is + # itself sparse never materializes back to a full column of the declared type. + runScenario( + "repeated kind", + externalTableBlock("UInt64", combinationKindStack([KIND_DEFAULT, KIND_SPARSE, KIND_SPARSE])), + "is out of order", + ) + + # Sparse can sit inside Replicated but not the other way round, and ColumnSparse over + # ColumnReplicated does not materialize back to a full column of the declared type either. + runScenario( + "kinds in the wrong order", + externalTableBlock("UInt64", combinationKindStack([KIND_DEFAULT, KIND_REPLICATED, KIND_SPARSE])), + "is out of order", + ) + + # Every kind stack starts with Default. + runScenario( + "kind stack without default", + externalTableBlock("UInt64", combinationKindStack([KIND_SPARSE])), + "must start with Default", + ) + + # A huge kind stack must be rejected before it is materialized into nested serializations. + huge = bytearray() + huge.append(KIND_STACK_COMBINATION) + writeVarUInt(1000000, huge) + runScenario("too many kinds", externalTableBlock("UInt64", bytes(huge)), "Too many serialization kinds") + + +if __name__ == "__main__": + main() diff --git a/tests/queries/0_stateless/04936_detached_serialization_from_client_native_protocol.reference b/tests/queries/0_stateless/04936_detached_serialization_from_client_native_protocol.reference new file mode 100644 index 0000000000000..f362e10f25139 --- /dev/null +++ b/tests/queries/0_stateless/04936_detached_serialization_from_client_native_protocol.reference @@ -0,0 +1,9 @@ +detached: caught expected exception INCORRECT_DATA +detached over sparse: caught expected exception INCORRECT_DATA +detached in a combination: caught expected exception INCORRECT_DATA +detached tuple element: caught expected exception INCORRECT_DATA +repeated kind: caught expected exception INCORRECT_DATA +kinds in the wrong order: caught expected exception INCORRECT_DATA +kind stack without default: caught expected exception INCORRECT_DATA +too many kinds: caught expected exception INCORRECT_DATA +server alive 45 diff --git a/tests/queries/0_stateless/04936_detached_serialization_from_client_native_protocol.sh b/tests/queries/0_stateless/04936_detached_serialization_from_client_native_protocol.sh new file mode 100755 index 0000000000000..55eae4a886457 --- /dev/null +++ b/tests/queries/0_stateless/04936_detached_serialization_from_client_native_protocol.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# We should have correct env vars from shell_config.sh to run this test +python3 "$CUR_DIR"/04936_detached_serialization_from_client_native_protocol.python + +# The kinds a client may legitimately select still work, and the server is still alive. +$CLICKHOUSE_CLIENT -q "SELECT 'server alive', sum(number) FROM numbers(10)" diff --git a/tests/queries/0_stateless/04945_loop_over_table_expression_without_columns.reference b/tests/queries/0_stateless/04945_loop_over_table_expression_without_columns.reference new file mode 100644 index 0000000000000..cff62484e38d4 --- /dev/null +++ b/tests/queries/0_stateless/04945_loop_over_table_expression_without_columns.reference @@ -0,0 +1,13 @@ +-- loop() over an alias whose target was dropped +-- reading the dangling alias directly still names the missing target +-- the dangling alias remains enumerable in the system tables +loop_no_columns_alias Alias +0 +-- structure inference over the dangling alias stays empty and does not throw +-- a parameterized view has no columns by design, with and without an alias +-- a healthy alias keeps working +1 +1 +1 +1 +c0 Int32 diff --git a/tests/queries/0_stateless/04945_loop_over_table_expression_without_columns.sql b/tests/queries/0_stateless/04945_loop_over_table_expression_without_columns.sql new file mode 100644 index 0000000000000..208eb7ecb9728 --- /dev/null +++ b/tests/queries/0_stateless/04945_loop_over_table_expression_without_columns.sql @@ -0,0 +1,51 @@ +-- Tags: no-old-analyzer +-- no-old-analyzer: the guard lives in the planner, so the old analyzer returns UNKNOWN_TABLE here. + +-- The `Alias` engine is still experimental on this branch, unlike on master where the setting is +-- obsolete and the guard is gone, so every alias test here has to enable it. +SET allow_experimental_alias_table_engine = 1; + +DROP TABLE IF EXISTS loop_no_columns_target; +DROP TABLE IF EXISTS loop_no_columns_alias; +DROP TABLE IF EXISTS loop_no_columns_base; +DROP VIEW IF EXISTS loop_no_columns_parameterized; +DROP TABLE IF EXISTS loop_no_columns_alias_to_view; + +CREATE TABLE loop_no_columns_target (c0 Int32) ENGINE = Memory; +CREATE TABLE loop_no_columns_alias ENGINE = Alias(currentDatabase(), loop_no_columns_target); +DROP TABLE loop_no_columns_target; + +SELECT '-- loop() over an alias whose target was dropped'; +SELECT count() FROM loop(currentDatabase(), 'loop_no_columns_alias'); -- { serverError UNSUPPORTED_METHOD } +SELECT 1 FROM loop(currentDatabase(), 'loop_no_columns_alias'); -- { serverError UNSUPPORTED_METHOD } + +SELECT '-- reading the dangling alias directly still names the missing target'; +SELECT count() FROM loop_no_columns_alias; -- { serverError UNKNOWN_TABLE } +DESCRIBE TABLE loop_no_columns_alias; -- { serverError UNKNOWN_TABLE } + +SELECT '-- the dangling alias remains enumerable in the system tables'; +SELECT name, engine FROM system.tables WHERE database = currentDatabase() AND name = 'loop_no_columns_alias'; +SELECT count() FROM system.columns WHERE database = currentDatabase() AND table = 'loop_no_columns_alias'; + +SELECT '-- structure inference over the dangling alias stays empty and does not throw'; +DESCRIBE TABLE loop(currentDatabase(), 'loop_no_columns_alias'); + +SELECT '-- a parameterized view has no columns by design, with and without an alias'; +CREATE TABLE loop_no_columns_base (a Int32) ENGINE = Memory; +CREATE VIEW loop_no_columns_parameterized AS SELECT * FROM loop_no_columns_base WHERE a = {p:Int32}; +SELECT count() FROM loop(currentDatabase(), 'loop_no_columns_parameterized'); -- { serverError UNSUPPORTED_METHOD } +CREATE TABLE loop_no_columns_alias_to_view ENGINE = Alias(currentDatabase(), loop_no_columns_parameterized); +SELECT count() FROM loop(currentDatabase(), 'loop_no_columns_alias_to_view'); -- { serverError UNSUPPORTED_METHOD } +DESCRIBE TABLE loop(currentDatabase(), 'loop_no_columns_parameterized'); + +SELECT '-- a healthy alias keeps working'; +CREATE TABLE loop_no_columns_target (c0 Int32) ENGINE = Memory; +INSERT INTO loop_no_columns_target VALUES (1), (2), (3); +SELECT 1 FROM loop(currentDatabase(), 'loop_no_columns_alias') LIMIT 4; +DESCRIBE TABLE loop(currentDatabase(), 'loop_no_columns_alias'); + +DROP TABLE loop_no_columns_alias_to_view; +DROP VIEW loop_no_columns_parameterized; +DROP TABLE loop_no_columns_base; +DROP TABLE loop_no_columns_alias; +DROP TABLE loop_no_columns_target; diff --git a/tests/queries/0_stateless/04945_loop_over_table_expression_without_columns_access.reference b/tests/queries/0_stateless/04945_loop_over_table_expression_without_columns_access.reference new file mode 100644 index 0000000000000..bfde573d9dcd3 --- /dev/null +++ b/tests/queries/0_stateless/04945_loop_over_table_expression_without_columns_access.reference @@ -0,0 +1,9 @@ +Without SELECT on the loop table +ACCESS_DENIED +With SELECT on the loop table +UNSUPPORTED_METHOD +With SELECT on a live alias but not on its target +ACCESS_DENIED +With SELECT on a live alias and on its target +1 +1 diff --git a/tests/queries/0_stateless/04945_loop_over_table_expression_without_columns_access.sh b/tests/queries/0_stateless/04945_loop_over_table_expression_without_columns_access.sh new file mode 100755 index 0000000000000..8df6141d453d7 --- /dev/null +++ b/tests/queries/0_stateless/04945_loop_over_table_expression_without_columns_access.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Tags: no-old-analyzer + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +. "$CUR_DIR"/../shell_config.sh + +username="user_${CLICKHOUSE_TEST_UNIQUE_NAME}" + +# The `Alias` engine is still experimental on this branch, unlike on master where the setting is +# obsolete and the guard is gone, so every alias test here has to enable it. +${CLICKHOUSE_CLIENT} -m --query " + SET allow_experimental_alias_table_engine = 1; + + DROP USER IF EXISTS ${username}; + DROP TABLE IF EXISTS loop_access_target; + DROP TABLE IF EXISTS loop_access_alias; + DROP TABLE IF EXISTS loop_access_live_target; + DROP TABLE IF EXISTS loop_access_live_alias; + + CREATE TABLE loop_access_target (c0 Int32) ENGINE = Memory; + CREATE TABLE loop_access_alias ENGINE = Alias(currentDatabase(), loop_access_target); + DROP TABLE loop_access_target; + + CREATE TABLE loop_access_live_target (c0 Int32) ENGINE = Memory; + INSERT INTO loop_access_live_target SELECT number FROM numbers(3); + CREATE TABLE loop_access_live_alias ENGINE = Alias(currentDatabase(), loop_access_live_target); + + CREATE USER ${username} NOT IDENTIFIED; + GRANT CREATE TEMPORARY TABLE ON *.* TO ${username}; +" + +echo "Without SELECT on the loop table" +${CLICKHOUSE_CLIENT} --user="${username}" --query \ + "SELECT count() FROM loop(currentDatabase(), 'loop_access_alias');" 2>&1 | + grep -o "ACCESS_DENIED" | uniq + +${CLICKHOUSE_CLIENT} --query "GRANT SELECT ON loop_access_alias TO ${username};" + +echo "With SELECT on the loop table" +${CLICKHOUSE_CLIENT} --user="${username}" --query \ + "SELECT count() FROM loop(currentDatabase(), 'loop_access_alias');" 2>&1 | + grep -o "UNSUPPORTED_METHOD" | uniq + +echo "With SELECT on a live alias but not on its target" +${CLICKHOUSE_CLIENT} --query "GRANT SELECT ON loop_access_live_alias TO ${username};" +${CLICKHOUSE_CLIENT} --user="${username}" --query \ + "SELECT 1 FROM loop(currentDatabase(), 'loop_access_live_alias') LIMIT 2;" 2>&1 | + grep -o "ACCESS_DENIED" | uniq + +echo "With SELECT on a live alias and on its target" +${CLICKHOUSE_CLIENT} --query "GRANT SELECT ON loop_access_live_target TO ${username};" +${CLICKHOUSE_CLIENT} --user="${username}" --query \ + "SELECT 1 FROM loop(currentDatabase(), 'loop_access_live_alias') LIMIT 2;" + +${CLICKHOUSE_CLIENT} -m --query " + DROP USER IF EXISTS ${username}; + DROP TABLE IF EXISTS loop_access_alias; + DROP TABLE IF EXISTS loop_access_live_alias; + DROP TABLE IF EXISTS loop_access_live_target; +" diff --git a/tests/queries/0_stateless/05019_ttl_tuple_datetime_overflow.reference b/tests/queries/0_stateless/05019_ttl_tuple_datetime_overflow.reference new file mode 100644 index 0000000000000..0cfbf08886fca --- /dev/null +++ b/tests/queries/0_stateless/05019_ttl_tuple_datetime_overflow.reference @@ -0,0 +1 @@ +2 diff --git a/tests/queries/0_stateless/05019_ttl_tuple_datetime_overflow.sql b/tests/queries/0_stateless/05019_ttl_tuple_datetime_overflow.sql new file mode 100644 index 0000000000000..4a436ebac411c --- /dev/null +++ b/tests/queries/0_stateless/05019_ttl_tuple_datetime_overflow.sql @@ -0,0 +1,17 @@ +-- Tuple-element TTL source columns are represented by their enclosing tuple during +-- syntax analysis. The nested `DateTime` element must still be widened before the +-- expression is rebuilt, otherwise this interval wraps in 32 bits and drops the part. +DROP TABLE IF EXISTS t_ttl_tuple_datetime_overflow; +CREATE TABLE t_ttl_tuple_datetime_overflow +( + t Tuple(id UInt64, ts DateTime), + value UInt64 +) +ENGINE = MergeTree +ORDER BY value; + +INSERT INTO t_ttl_tuple_datetime_overflow VALUES ((1, '2034-01-01 00:00:00'), 10), ((2, '2034-06-15 12:00:00'), 20); +SET mutations_sync = 2; +ALTER TABLE t_ttl_tuple_datetime_overflow MODIFY TTL t.ts + INTERVAL 100 YEAR; +SELECT count() FROM t_ttl_tuple_datetime_overflow; +DROP TABLE t_ttl_tuple_datetime_overflow; diff --git a/tests/queries/0_stateless/05023_ttl_array_datetime_overflow.reference b/tests/queries/0_stateless/05023_ttl_array_datetime_overflow.reference new file mode 100644 index 0000000000000..0cfbf08886fca --- /dev/null +++ b/tests/queries/0_stateless/05023_ttl_array_datetime_overflow.reference @@ -0,0 +1 @@ +2 diff --git a/tests/queries/0_stateless/05023_ttl_array_datetime_overflow.sql b/tests/queries/0_stateless/05023_ttl_array_datetime_overflow.sql new file mode 100644 index 0000000000000..54d15ff6be44b --- /dev/null +++ b/tests/queries/0_stateless/05023_ttl_array_datetime_overflow.sql @@ -0,0 +1,16 @@ +-- An `Array(DateTime)` source is represented by the enclosing array during syntax +-- analysis. Widen its nested timestamp before evaluating the interval so it cannot wrap. +DROP TABLE IF EXISTS t_ttl_array_datetime_overflow; +CREATE TABLE t_ttl_array_datetime_overflow +( + arr Array(DateTime), + value UInt64 +) +ENGINE = MergeTree +ORDER BY value; + +INSERT INTO t_ttl_array_datetime_overflow VALUES (['2034-01-01 00:00:00'], 10), (['2034-06-15 12:00:00'], 20); +SET mutations_sync = 2; +ALTER TABLE t_ttl_array_datetime_overflow MODIFY TTL arr[1] + INTERVAL 100 YEAR; +SELECT count() FROM t_ttl_array_datetime_overflow; +DROP TABLE t_ttl_array_datetime_overflow; diff --git a/tests/queries/0_stateless/05043_merge_read_in_order_prefix_within_sorting_key.reference b/tests/queries/0_stateless/05043_merge_read_in_order_prefix_within_sorting_key.reference new file mode 100644 index 0000000000000..b6f5127d5c551 --- /dev/null +++ b/tests/queries/0_stateless/05043_merge_read_in_order_prefix_within_sorting_key.reference @@ -0,0 +1,8 @@ +1 +0 +0 0 0 +0 0 0 +1 1 1 +0 0 0 +0 0 0 +1 1 1 diff --git a/tests/queries/0_stateless/05043_merge_read_in_order_prefix_within_sorting_key.sql b/tests/queries/0_stateless/05043_merge_read_in_order_prefix_within_sorting_key.sql new file mode 100644 index 0000000000000..cb1b7ad48862c --- /dev/null +++ b/tests/queries/0_stateless/05043_merge_read_in_order_prefix_within_sorting_key.sql @@ -0,0 +1,38 @@ +-- Tags: no-random-merge-tree-settings + +DROP TABLE IF EXISTS t_read_in_order_prefix; +DROP TABLE IF EXISTS m_read_in_order_prefix; + +CREATE TABLE t_read_in_order_prefix (a UInt64, b UInt64, c UInt64) + ENGINE = MergeTree ORDER BY (a, b, c) SETTINGS index_granularity = 8; + +SYSTEM STOP MERGES t_read_in_order_prefix; + +-- Two parts, so the preliminary merge path is taken with the threshold below. +INSERT INTO t_read_in_order_prefix SELECT number, number, number FROM numbers(500); +INSERT INTO t_read_in_order_prefix SELECT number, number, number FROM numbers(500); + +CREATE TABLE m_read_in_order_prefix (a UInt64, b UInt64, c UInt64) + ENGINE = Merge(currentDatabase(), '^t_read_in_order_prefix$'); + +-- Read-in-order is still chosen for a well-formed query over a Merge table, and the +-- preliminary merge path that consumes the key prefix is the one exercised. +SELECT count() > 0 FROM ( + EXPLAIN actions = 1 SELECT a, b, c FROM m_read_in_order_prefix ORDER BY a, b, c + SETTINGS optimize_read_in_order = 1, read_in_order_two_level_merge_threshold = 1 +) WHERE explain ILIKE '%ReadType: InOrder%'; + +-- Control: the assertion above is not vacuous. +SELECT count() > 0 FROM ( + EXPLAIN actions = 1 SELECT a, b, c FROM m_read_in_order_prefix ORDER BY a, b, c + SETTINGS optimize_read_in_order = 0 +) WHERE explain ILIKE '%ReadType: InOrder%'; + +-- Results are unchanged with and without the optimization. +SELECT a, b, c FROM m_read_in_order_prefix ORDER BY a, b, c LIMIT 3 + SETTINGS optimize_read_in_order = 1, read_in_order_two_level_merge_threshold = 1; +SELECT a, b, c FROM m_read_in_order_prefix ORDER BY a, b, c LIMIT 3 + SETTINGS optimize_read_in_order = 0; + +DROP TABLE m_read_in_order_prefix; +DROP TABLE t_read_in_order_prefix; diff --git a/tests/queries/0_stateless/05054_nullable_key_todate_monotonicity.reference b/tests/queries/0_stateless/05054_nullable_key_todate_monotonicity.reference new file mode 100644 index 0000000000000..19219d2f6180d --- /dev/null +++ b/tests/queries/0_stateless/05054_nullable_key_todate_monotonicity.reference @@ -0,0 +1,8 @@ +Granules: 2/10 +Granules: 2/10 +Granules: 2/10 +3 15 +3 6 +6 105 +6 105 +9 120 diff --git a/tests/queries/0_stateless/05054_nullable_key_todate_monotonicity.sql b/tests/queries/0_stateless/05054_nullable_key_todate_monotonicity.sql new file mode 100644 index 0000000000000..aae7082ce17b0 --- /dev/null +++ b/tests/queries/0_stateless/05054_nullable_key_todate_monotonicity.sql @@ -0,0 +1,33 @@ +-- The monotonicity of a conversion does not depend on the `Nullable` wrapper of its argument, so a +-- `toDate`/`toDateTime` predicate over a `Nullable` key must still be usable for index analysis. + +SET session_timezone = 'UTC'; + +DROP TABLE IF EXISTS t_nullable_key; +CREATE TABLE t_nullable_key (x Nullable(DateTime64(6)), y Int64) +ENGINE = MergeTree ORDER BY x SETTINGS index_granularity = 4, allow_nullable_key = 1; + +INSERT INTO t_nullable_key +SELECT if(number % 7 = 0, NULL, toDateTime64('2026-03-01 00:00:00', 6) + INTERVAL number * 6 HOUR), number +FROM numbers(40); + +SELECT extract(explain, 'Granules: \\d+/\\d+') AS granules FROM ( + EXPLAIN indexes = 1 SELECT count() FROM t_nullable_key WHERE toDate(x) = toDate('2026-03-02') +) WHERE granules != ''; + +SELECT extract(explain, 'Granules: \\d+/\\d+') AS granules FROM ( + EXPLAIN indexes = 1 SELECT count() FROM t_nullable_key WHERE toDate32(x) = toDate32('2026-03-02') +) WHERE granules != ''; + +SELECT extract(explain, 'Granules: \\d+/\\d+') AS granules FROM ( + EXPLAIN indexes = 1 SELECT count() FROM t_nullable_key WHERE toDateTime(x) = toDateTime('2026-03-02 06:00:00') +) WHERE granules != ''; + +-- The rows the index analysis keeps must be the rows the predicate selects, `NULL`s included. +SELECT count(), sum(y) FROM t_nullable_key WHERE toDate(x) = toDate('2026-03-02'); +SELECT count(), sum(y) FROM t_nullable_key WHERE toDate(x) < toDate('2026-03-02'); +SELECT count(), sum(y) FROM t_nullable_key WHERE toDate(x) IS NULL; +SELECT count(), sum(y) FROM t_nullable_key WHERE toDate(x) IN (toDate('2026-03-02'), toDate('2026-03-08')); +SELECT count(), sum(y) FROM t_nullable_key WHERE isNull(x) OR toDate(x) = toDate('2026-03-02'); + +DROP TABLE t_nullable_key; diff --git a/tests/queries/0_stateless/05054_postgresql_protocol_message_size.reference b/tests/queries/0_stateless/05054_postgresql_protocol_message_size.reference new file mode 100644 index 0000000000000..ec35dc29b3b47 --- /dev/null +++ b/tests/queries/0_stateless/05054_postgresql_protocol_message_size.reference @@ -0,0 +1,6 @@ +SASL mechanism longer than the message: error response +password without a terminator: error response +well-formed exchange: query result received +query with a smuggled tail: error response +query shorter than declared: error response +oversized sync: error response diff --git a/tests/queries/0_stateless/05054_postgresql_protocol_message_size.sh b/tests/queries/0_stateless/05054_postgresql_protocol_message_size.sh new file mode 100755 index 0000000000000..e5fd34c2db3ff --- /dev/null +++ b/tests/queries/0_stateless/05054_postgresql_protocol_message_size.sh @@ -0,0 +1,158 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: the PostgreSQL compatibility port is not enabled in fasttest. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Authentication messages are parsed before the client is authenticated, so a field of a message +# must not be allocated at the size the client declares for it, and parsing a message must not +# continue past the size the client declared for the message. + +USER_SCRAM="user_scram_${CLICKHOUSE_DATABASE}" +USER_PLAIN="user_plain_${CLICKHOUSE_DATABASE}" + +$CLICKHOUSE_CLIENT --query "CREATE USER ${USER_SCRAM} IDENTIFIED WITH scram_sha256_password BY 'x'" +$CLICKHOUSE_CLIENT --query "CREATE USER ${USER_PLAIN} IDENTIFIED WITH plaintext_password BY 'x'" + +CLICKHOUSE_PORT_POSTGRESQL="$CLICKHOUSE_PORT_POSTGRESQL" USER_SCRAM="$USER_SCRAM" USER_PLAIN="$USER_PLAIN" python3 - <<'PYTHON' +import os +import socket +import struct + +port = int(os.environ["CLICKHOUSE_PORT_POSTGRESQL"]) +user_scram = os.environ["USER_SCRAM"] +user_plain = os.environ["USER_PLAIN"] + + +def connect(): + sock = socket.create_connection(("127.0.0.1", port), timeout=30) + sock.settimeout(30) + return sock + + +def startup(sock, user): + payload = ("user\x00" + user + "\x00\x00").encode() + sock.sendall(struct.pack(">ii", 8 + len(payload), 196608) + payload) + + +def outcome(sock): + """Drain the reply and describe it: a rejected message gets an error response, an accepted one + that is never completed leaves the server waiting for the payload.""" + data = b"" + while True: + try: + chunk = sock.recv(4096) + except socket.timeout: + return "TIMEOUT" + except ConnectionResetError: + break + if not chunk: + break + data += chunk + if data[0:1] == b"E": + return "error response" + if not data: + return "NO REPLY" + return "UNEXPECTED REPLY " + repr(data[:64]) + + +def first_reply(sock): + """Describe the first reply message, for a case where the server rejects the message but keeps + the connection open, so waiting for it to close would only hit the timeout.""" + try: + data = sock.recv(4096) + except (socket.timeout, ConnectionResetError): + return "TIMEOUT" + if data[0:1] == b"E": + return "error response" + if not data: + return "NO REPLY" + return "UNEXPECTED REPLY " + repr(data[:64]) + + +# A SASL initial response that declares a two-gigabyte mechanism in a thirty-byte message. +sock = connect() +startup(sock, user_scram) +sock.recv(4096) +body = b"SCRAM-SHA-256\x00" + struct.pack(">i", 0x7FFFFFFF) +sock.sendall(b"p" + struct.pack(">i", 4 + len(body)) + body) +print("SASL mechanism longer than the message:", outcome(sock)) +sock.close() + +# A password message that declares a small size and then streams a password without a terminator. +sock = connect() +startup(sock, user_plain) +sock.recv(4096) +sock.sendall(b"p" + struct.pack(">i", 4 + 10)) +try: + sock.sendall(b"x" * 200000) +except (BrokenPipeError, ConnectionResetError, socket.timeout): + pass +print("password without a terminator:", outcome(sock)) +sock.close() + +# A well-formed exchange still works: authenticate and run a query. +def read_until_ready(sock): + data = b"" + while not data.endswith(b"Z\x00\x00\x00\x05I"): + chunk = sock.recv(4096) + if not chunk: + break + data += chunk + return data + + +sock = connect() +startup(sock, user_plain) +sock.recv(4096) +sock.sendall(b"p" + struct.pack(">i", 4 + 2) + b"x\x00") +read_until_ready(sock) +sock.sendall(b"Q" + struct.pack(">i", 4 + 9) + b"SELECT 1\x00") +data = read_until_ready(sock) +print("well-formed exchange:", "query result received" if b"C\x00\x00\x00\rSELECT 1\x00" in data else "UNEXPECTED REPLY " + repr(data[:96])) +sock.close() + +# A `Query` that terminates its string early and smuggles a whole second `Query` into the tail of the +# same declared frame. The declared length is a frame boundary, so the tail must be rejected instead +# of being read as the next message. +sock = connect() +startup(sock, user_plain) +sock.recv(4096) +sock.sendall(b"p" + struct.pack(">i", 4 + 2) + b"x\x00") +read_until_ready(sock) +body = b"SELECT 1\x00" + b"SELECT 2\x00" +sock.sendall(b"Q" + struct.pack(">i", 4 + len(body)) + body) +print("query with a smuggled tail:", first_reply(sock)) +sock.close() + +# A `Query` that declares more bytes than the client sends and then closes the write side of the +# connection. The declared length is a frame boundary in both directions, so the message must be +# rejected instead of being executed with the part of the payload that did arrive. +sock = connect() +startup(sock, user_plain) +sock.recv(4096) +sock.sendall(b"p" + struct.pack(">i", 4 + 2) + b"x\x00") +read_until_ready(sock) +body = b"SELECT 1\x00" +sock.sendall(b"Q" + struct.pack(">i", 4 + len(body) + 1000) + body) +sock.shutdown(socket.SHUT_WR) +print("query shorter than declared:", outcome(sock)) +sock.close() + +# An oversized `Sync`, whose parser reads nothing at all: its payload must not survive the message +# boundary and be reinterpreted as the next message. +sock = connect() +startup(sock, user_plain) +sock.recv(4096) +sock.sendall(b"p" + struct.pack(">i", 4 + 2) + b"x\x00") +read_until_ready(sock) +body = b"Q" + struct.pack(">i", 4 + 9) + b"SELECT 2\x00" +sock.sendall(b"S" + struct.pack(">i", 4 + len(body)) + body) +print("oversized sync:", outcome(sock)) +sock.close() +PYTHON + +$CLICKHOUSE_CLIENT --query "DROP USER ${USER_SCRAM}" +$CLICKHOUSE_CLIENT --query "DROP USER ${USER_PLAIN}" diff --git a/tests/queries/0_stateless/05055_json_flattened_native_truncated.reference b/tests/queries/0_stateless/05055_json_flattened_native_truncated.reference new file mode 100644 index 0000000000000..68961ec9d0d83 --- /dev/null +++ b/tests/queries/0_stateless/05055_json_flattened_native_truncated.reference @@ -0,0 +1,9 @@ +t_json_flattened: accepted truncated blocks 0 +0 +12 18 6 51 +t_json_array_flattened: accepted truncated blocks 0 +0 +12 12 2 +t_dynamic_flattened: accepted truncated blocks 0 +0 +12 6 27 str8 diff --git a/tests/queries/0_stateless/05055_json_flattened_native_truncated.sh b/tests/queries/0_stateless/05055_json_flattened_native_truncated.sh new file mode 100755 index 0000000000000..7577adec5191a --- /dev/null +++ b/tests/queries/0_stateless/05055_json_flattened_native_truncated.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Tags: no-old-analyzer +# no-old-analyzer: the queries building the payloads read JSON/Dynamic subcolumns, which it cannot resolve. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A flattened JSON/Dynamic Native block must be rejected when a stream ends before the declared row count. +# The data starts with rows that have no dynamic paths, so the Dynamic indexes stream starts with a run of +# NULL markers: truncated inside that run it demands no values at all and passes every per-type check. + +DATA_FILE=$CLICKHOUSE_TMP/flattened_native_$CLICKHOUSE_DATABASE.bin + +function check_truncated_blocks() +{ + local table=$1 + local query=$2 + + $CLICKHOUSE_CLIENT --output_format_native_use_flattened_dynamic_and_json_serialization=1 -q "$query FORMAT Native" > "$DATA_FILE" + + local size + size=$(stat -c%s "$DATA_FILE") + local accepted=0 + local n + # Sent over HTTP: a client process per truncation is what made this test time out under sanitizers. + for ((n = 1; n < size; n += 5)); do + if head -c "$n" "$DATA_FILE" \ + | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=INSERT+INTO+$table+FORMAT+Native" --data-binary @- \ + | grep -q "Exception" + then + continue + fi + accepted=$((accepted + 1)) + done + + echo "$table: accepted truncated blocks $accepted" + $CLICKHOUSE_CLIENT -q "SELECT count() FROM $table" + + $CLICKHOUSE_CLIENT -q "INSERT INTO $table FORMAT Native" < "$DATA_FILE" + rm "$DATA_FILE" +} + +$CLICKHOUSE_CLIENT -q " + CREATE TABLE t_json_flattened (json JSON(a UInt64, max_dynamic_paths=2)) ENGINE = Memory; + CREATE TABLE t_json_array_flattened (arr Array(JSON)) ENGINE = Memory; + CREATE TABLE t_dynamic_flattened (dyn Dynamic) ENGINE = Memory; +" + +check_truncated_blocks t_json_flattened " + SELECT (number < 6 ? '{\"a\":1}' : '{\"a\":2,\"b\":\"str' || toString(number) || '\",\"c\":' || toString(number) || '}')::JSON(a UInt64, max_dynamic_paths=2) AS json + FROM numbers(12)" +$CLICKHOUSE_CLIENT -q "SELECT count(), sum(json.a), countIf(json.b IS NULL), sum(json.c.:Int64) FROM t_json_flattened" + +check_truncated_blocks t_json_array_flattened " + SELECT arrayMap(x -> (number < 6 ? '{}' : '{\"a\":' || toString(x) || '}')::JSON, range(number % 3)) AS arr + FROM numbers(12)" +$CLICKHOUSE_CLIENT -q "SELECT count(), sum(length(arr)), sum(arraySum(arrayMap(x -> x.a::UInt64, arr))) FROM t_json_array_flattened" + +check_truncated_blocks t_dynamic_flattened " + SELECT json.a AS dyn + FROM (SELECT (number < 6 ? '{}' : (number % 2 ? '{\"a\":' || toString(number) || '}' : '{\"a\":\"str' || toString(number) || '\"}'))::JSON AS json FROM numbers(12))" +$CLICKHOUSE_CLIENT -q "SELECT count(), countIf(dyn IS NULL), sum(dyn.Int64), max(dyn.String) FROM t_dynamic_flattened" + +$CLICKHOUSE_CLIENT -q " + DROP TABLE t_json_flattened; + DROP TABLE t_json_array_flattened; + DROP TABLE t_dynamic_flattened; +" diff --git a/tests/queries/0_stateless/05055_json_native_flattened_paths_memory_tracking.reference b/tests/queries/0_stateless/05055_json_native_flattened_paths_memory_tracking.reference new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/queries/0_stateless/05055_json_native_flattened_paths_memory_tracking.sql b/tests/queries/0_stateless/05055_json_native_flattened_paths_memory_tracking.sql new file mode 100644 index 0000000000000..cb3fe2da954e8 --- /dev/null +++ b/tests/queries/0_stateless/05055_json_native_flattened_paths_memory_tracking.sql @@ -0,0 +1,4 @@ +-- The FLATTENED (version 3) path list is sized from an untrusted count in the Native prefix, so its +-- container must be memory-tracked. +-- Frame: 1 column, 1 row, name "j", type "JSON", 8-byte LE structure version, VarUInt path count 1000000. +SELECT * FROM format(Native, 'j JSON', unhex('0101016A044A534F4E0300000000000000C0843D')) SETTINGS max_memory_usage = 10000000; -- { serverError MEMORY_LIMIT_EXCEEDED } diff --git a/tests/queries/0_stateless/05055_native_protocol_collection_limits.python b/tests/queries/0_stateless/05055_native_protocol_collection_limits.python new file mode 100644 index 0000000000000..df47b8da1ff3b --- /dev/null +++ b/tests/queries/0_stateless/05055_native_protocol_collection_limits.python @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 + +# The server allocates from counts and sizes that a client puts on the wire, before the corresponding +# data arrives, so a collection read from the protocol has to be bounded by what the data can +# plausibly hold. In interserver mode some of these fields are read before the secret hash of the +# query is checked. + +import os +import socket +import uuid + +CLICKHOUSE_HOST = os.environ.get("CLICKHOUSE_HOST", "127.0.0.1") +CLICKHOUSE_PORT = int(os.environ.get("CLICKHOUSE_PORT_TCP", "900000")) +CLICKHOUSE_DATABASE = os.environ.get("CLICKHOUSE_DATABASE", "default") +CLIENT_NAME = "05055 native protocol" +CLIENT_REVISION = 54459 # the first revision with query parameters + + +def writeVarUInt(x, ba): + for _ in range(0, 9): + byte = x & 0x7F + if x > 0x7F: + byte |= 0x80 + + ba.append(byte) + + x >>= 7 + if x == 0: + return + + +def writeStringBinary(s, ba): + b = bytes(s, "utf-8") + writeVarUInt(len(b), ba) + ba.extend(b) + + +def readStrict(s, size=1): + res = bytearray() + while size: + cur = s.recv(size) + if not cur: + raise RuntimeError("Socket is closed") + size -= len(cur) + res.extend(cur) + + return res + + +def readUInt(s, size=1): + res = readStrict(s, size) + val = 0 + for i in range(len(res)): + val += res[i] << (i * 8) + return val + + +def readVarUInt(s): + x = 0 + for i in range(9): + byte = readStrict(s)[0] + x |= (byte & 0x7F) << (7 * i) + + if not byte & 0x80: + return x + + return x + + +def readStringBinary(s): + size = readVarUInt(s) + return readStrict(s, size).decode("utf-8", "replace") + + +def connect(): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(30) + s.connect((CLICKHOUSE_HOST, CLICKHOUSE_PORT)) + + ba = bytearray() + writeVarUInt(0, ba) # Hello + writeStringBinary(CLIENT_NAME, ba) + writeVarUInt(21, ba) + writeVarUInt(9, ba) + writeVarUInt(CLIENT_REVISION, ba) + writeStringBinary(CLICKHOUSE_DATABASE, ba) + writeStringBinary("default", ba) + writeStringBinary("", ba) + s.sendall(ba) + + assert readVarUInt(s) == 0 # Hello + readStringBinary(s) # server name + readVarUInt(s) # version major + readVarUInt(s) # version minor + readVarUInt(s) # revision + readStringBinary(s) # timezone + readStringBinary(s) # display name + readVarUInt(s) # version patch + + ba = bytearray() + writeStringBinary("", ba) # addendum: quota key + s.sendall(ba) + return s + + +def serializeClientInfo(ba, query_id, os_user_declared_size=None): + writeStringBinary("default", ba) # initial_user + writeStringBinary(query_id, ba) # initial_query_id + writeStringBinary("127.0.0.1:9000", ba) # initial_address + ba.extend([0] * 8) # initial_query_start_time_microseconds + ba.append(1) # TCP + if os_user_declared_size is None: + writeStringBinary("os_user", ba) # os_user + else: + writeVarUInt(os_user_declared_size, ba) # os_user, but the value never follows + return + writeStringBinary("client_hostname", ba) # client_hostname + writeStringBinary(CLIENT_NAME, ba) # client_name + writeVarUInt(21, ba) + writeVarUInt(9, ba) + writeVarUInt(CLIENT_REVISION, ba) + writeStringBinary("", ba) # quota_key + writeVarUInt(0, ba) # distributed_depth + writeVarUInt(1, ba) # client_version_patch + ba.append(0) # no telemetry + writeVarUInt(0, ba) # collaborate_with_initiator + writeVarUInt(0, ba) # count_participating_replicas + writeVarUInt(0, ba) # number_of_current_replica + + +def sendQuery( + s, + query, + num_settings=0, + os_user_declared_size=None, + setting_name_declared_size=None, + parameter_name_declared_size=None, +): + ba = bytearray() + query_id = uuid.uuid4().hex + writeVarUInt(1, ba) # Query + writeStringBinary(query_id, ba) + ba.append(1) # INITIAL_QUERY + serializeClientInfo(ba, query_id, os_user_declared_size) + if os_user_declared_size is not None: + s.sendall(ba) + return + + if setting_name_declared_size is not None: + writeVarUInt( + setting_name_declared_size, ba + ) # a setting name that never follows + s.sendall(ba) + return + + for _ in range(num_settings): + writeStringBinary("max_threads", ba) + writeVarUInt(0, ba) # flags + writeStringBinary("1", ba) + writeStringBinary("", ba) # end of settings + + writeStringBinary("", ba) # no interserver secret + writeVarUInt(2, ba) # stage: Complete + ba.append(0) # no compression + writeStringBinary(query, ba) + + if parameter_name_declared_size is not None: + writeVarUInt( + parameter_name_declared_size, ba + ) # a parameter name that never follows + s.sendall(ba) + return + + writeStringBinary("", ba) # end of query parameters + s.sendall(ba) + + +def sendTablesStatusRequest(s, num_tables, database_declared_size=None): + ba = bytearray() + writeVarUInt(5, ba) # TablesStatusRequest + writeVarUInt(num_tables, ba) + if database_declared_size is not None: + writeVarUInt( + database_declared_size, ba + ) # a database name whose value never follows + s.sendall(ba) + + +def sendEmptyBlock(s): + ba = bytearray() + writeVarUInt(2, ba) # Data + writeStringBinary("", ba) + writeVarUInt(1, ba) + ba.append(0) # is_overflows + writeVarUInt(2, ba) + writeVarUInt(0, ba) + ba.extend([0] * 4) # bucket_num + writeVarUInt(0, ba) # columns + writeVarUInt(0, ba) # rows + s.sendall(ba) + + +def readException(s): + readUInt(s, 4) # code + readStringBinary(s) # name + message = readStringBinary(s) + readStringBinary(s) # stack trace + readUInt(s) # has_nested + for known in ( + "Too many settings received", + "Too large string size", + "Too large collection size", + "Attempt to read after eof", + "Cannot read all data", + ): + if known in message: + return known + return "UNEXPECTED " + message[:120] + + +def outcome(s): + """The name of the exception the server answered with, or how the exchange ended.""" + try: + while True: + packet_type = readVarUInt(s) + if packet_type == 2: # Exception + return readException(s) + if packet_type == 5: # EndOfStream + return "end of stream" + if packet_type == 1: # Data + return "data" + if packet_type == 3: # Progress + continue + return "unexpected packet {}".format(packet_type) + except socket.timeout: + return "TIMEOUT" + except RuntimeError: + return "closed" + + +def rejected(s): + """A `TablesStatusRequest` is read before there is a query state to attach an exception to, so + the server may either answer with one or simply close the connection; either way it stops + reading instead of waiting for data that never comes.""" + result = outcome(s) + if result in ("closed", "Too large collection size", "Too large string size"): + return "rejected" + return result + + +def endOfStream(s): + """The client closed its side in the middle of a packet: the server may either answer with the + exception about the end of the stream or simply close the connection.""" + result = outcome(s) + if result in ("closed", "Attempt to read after eof", "Cannot read all data"): + return "end of stream" + return result + + +def main(): + # A settings sequence longer than any real query. + s = connect() + sendQuery(s, "SELECT 1", num_settings=65537) + print("too many settings:", outcome(s)) + s.close() + + # A `ClientInfo` string that declares a gigabyte and never sends it. It is read growing as the + # bytes arrive, so the server waits for them instead of allocating the declared size, and stops + # at the end of the stream. + s = connect() + sendQuery(s, "SELECT 1", os_user_declared_size=1 << 30) + s.shutdown(socket.SHUT_WR) + print("client info string of a gigabyte:", endOfStream(s)) + s.close() + + # A setting name that declares a gigabyte and never sends it: the count bound alone does not + # cover a single pair. + s = connect() + sendQuery(s, "SELECT 1", setting_name_declared_size=1 << 30) + print("setting name of a gigabyte:", outcome(s)) + s.close() + + # The same for a query parameter name. + s = connect() + sendQuery(s, "SELECT 1", parameter_name_declared_size=1 << 30) + print("query parameter name of a gigabyte:", outcome(s)) + s.close() + + # A `TablesStatusRequest` about more tables than a server can hold. + s = connect() + sendTablesStatusRequest(s, 1 << 30) + print("too many tables in a tables status request:", rejected(s)) + s.close() + + # A `TablesStatusRequest` whose database name declares a gigabyte and never sends it. + s = connect() + sendTablesStatusRequest(s, 1, database_declared_size=1 << 30) + print("table status identifier of a gigabyte:", rejected(s)) + s.close() + + # A well-formed query still works. + s = connect() + sendQuery(s, "SELECT 1", num_settings=1) + sendEmptyBlock(s) + print("well-formed query:", outcome(s)) + s.close() + + +if __name__ == "__main__": + main() diff --git a/tests/queries/0_stateless/05055_native_protocol_collection_limits.reference b/tests/queries/0_stateless/05055_native_protocol_collection_limits.reference new file mode 100644 index 0000000000000..4fc7c418dc5b5 --- /dev/null +++ b/tests/queries/0_stateless/05055_native_protocol_collection_limits.reference @@ -0,0 +1,7 @@ +too many settings: Too many settings received +client info string of a gigabyte: end of stream +setting name of a gigabyte: Too large string size +query parameter name of a gigabyte: Too large string size +too many tables in a tables status request: rejected +table status identifier of a gigabyte: rejected +well-formed query: data diff --git a/tests/queries/0_stateless/05055_native_protocol_collection_limits.sh b/tests/queries/0_stateless/05055_native_protocol_collection_limits.sh new file mode 100755 index 0000000000000..dc01e3f6e8b3d --- /dev/null +++ b/tests/queries/0_stateless/05055_native_protocol_collection_limits.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +# NOTE: this sh wrapper is required because of shell_config + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +python3 "$CUR_DIR"/05055_native_protocol_collection_limits.python diff --git a/tests/queries/0_stateless/05055_optimize_or_like_chain_end_anchor.reference b/tests/queries/0_stateless/05055_optimize_or_like_chain_end_anchor.reference new file mode 100644 index 0000000000000..4a548648e6bd3 --- /dev/null +++ b/tests/queries/0_stateless/05055_optimize_or_like_chain_end_anchor.reference @@ -0,0 +1,27 @@ +exact / rewrite off 1 +exact / rewrite on, hyperscan on 1 +exact / rewrite on, hyperscan off 1 +suffix / rewrite off 2 +suffix / rewrite on, hyperscan on 2 +suffix / rewrite on, hyperscan off 2 +underscore / rewrite off 2 +underscore / rewrite on, hyperscan on 2 +underscore / rewrite on, hyperscan off 2 +inner percent / rewrite off 2 +inner percent / rewrite on, hyperscan on 2 +inner percent / rewrite on, hyperscan off 2 +ilike / rewrite off 1 +ilike / rewrite on, hyperscan on 1 +ilike / rewrite on, hyperscan off 1 +empty pattern / rewrite off 1 +empty pattern / rewrite on, hyperscan on 1 +empty pattern / rewrite on, hyperscan off 1 +mixed group / rewrite off 1 +mixed group / rewrite on, hyperscan on 1 +mixed group / rewrite on, hyperscan off 1 +escaped dollar / rewrite off 2 +escaped dollar / rewrite on, hyperscan on 2 +escaped dollar / rewrite on, hyperscan off 2 +prefix / rewrite off 3 +prefix / rewrite on, hyperscan on 3 +prefix / rewrite on, hyperscan off 3 diff --git a/tests/queries/0_stateless/05055_optimize_or_like_chain_end_anchor.sql b/tests/queries/0_stateless/05055_optimize_or_like_chain_end_anchor.sql new file mode 100644 index 0000000000000..8bba65995e007 --- /dev/null +++ b/tests/queries/0_stateless/05055_optimize_or_like_chain_end_anchor.sql @@ -0,0 +1,58 @@ +-- `optimize_or_like_chain` rewrites an `OR` chain of `LIKE`/`ILIKE` into `multiMatchAny`, which runs +-- through Vectorscan, where `$` also matches before a final newline (PCRE); the original `like` uses +-- re2, where `$` is the absolute end. `likePatternToRegexp` anchors every pattern not ending in `%`, +-- so the rewrite used to widen such filters: `s LIKE 'tenant-a'` also returned `'tenant-a\n'`. +-- Results must not depend on `optimize_or_like_chain` or `allow_hyperscan`. + +DROP TABLE IF EXISTS t_or_like_end_anchor; +CREATE TABLE t_or_like_end_anchor (s String) ENGINE = Memory; + +-- The values ending in a newline are excluded by the patterns below, but accepted by a PCRE `$`. +INSERT INTO t_or_like_end_anchor VALUES ('tenant-a'), ('tenant-a\n'), ('tenant-a\n\n'), ('x-tenant-a'), ('x-tenant-a\n'), ('taxb'), ('ta\nb'), ('ta\nb\n'), (''), ('\n'), ('a$'), ('a$\n'); + +-- regexp `^tenant\-a$` +SELECT 'exact / rewrite off', count() FROM t_or_like_end_anchor WHERE s LIKE 'tenant-a' OR s LIKE 'never' SETTINGS optimize_or_like_chain = 0, enable_analyzer = 1; +SELECT 'exact / rewrite on, hyperscan on', count() FROM t_or_like_end_anchor WHERE s LIKE 'tenant-a' OR s LIKE 'never' SETTINGS optimize_or_like_chain = 1, allow_hyperscan = 1, enable_analyzer = 1; +SELECT 'exact / rewrite on, hyperscan off', count() FROM t_or_like_end_anchor WHERE s LIKE 'tenant-a' OR s LIKE 'never' SETTINGS optimize_or_like_chain = 1, allow_hyperscan = 0, enable_analyzer = 1; + +-- regexp `tenant\-a$` +SELECT 'suffix / rewrite off', count() FROM t_or_like_end_anchor WHERE s LIKE '%tenant-a' OR s LIKE '%never' SETTINGS optimize_or_like_chain = 0, enable_analyzer = 1; +SELECT 'suffix / rewrite on, hyperscan on', count() FROM t_or_like_end_anchor WHERE s LIKE '%tenant-a' OR s LIKE '%never' SETTINGS optimize_or_like_chain = 1, allow_hyperscan = 1, enable_analyzer = 1; +SELECT 'suffix / rewrite on, hyperscan off', count() FROM t_or_like_end_anchor WHERE s LIKE '%tenant-a' OR s LIKE '%never' SETTINGS optimize_or_like_chain = 1, allow_hyperscan = 0, enable_analyzer = 1; + +-- regexp `^ta.b$`. `.` matches a newline on both paths (`RE_DOT_NL`, `HS_FLAG_DOTALL`), only `$` differs. +SELECT 'underscore / rewrite off', count() FROM t_or_like_end_anchor WHERE s LIKE 'ta_b' OR s LIKE 'ne_er' SETTINGS optimize_or_like_chain = 0, enable_analyzer = 1; +SELECT 'underscore / rewrite on, hyperscan on', count() FROM t_or_like_end_anchor WHERE s LIKE 'ta_b' OR s LIKE 'ne_er' SETTINGS optimize_or_like_chain = 1, allow_hyperscan = 1, enable_analyzer = 1; +SELECT 'underscore / rewrite on, hyperscan off', count() FROM t_or_like_end_anchor WHERE s LIKE 'ta_b' OR s LIKE 'ne_er' SETTINGS optimize_or_like_chain = 1, allow_hyperscan = 0, enable_analyzer = 1; + +-- regexp `^ta.*b$` +SELECT 'inner percent / rewrite off', count() FROM t_or_like_end_anchor WHERE s LIKE 'ta%b' OR s LIKE 'ne%er' SETTINGS optimize_or_like_chain = 0, enable_analyzer = 1; +SELECT 'inner percent / rewrite on, hyperscan on', count() FROM t_or_like_end_anchor WHERE s LIKE 'ta%b' OR s LIKE 'ne%er' SETTINGS optimize_or_like_chain = 1, allow_hyperscan = 1, enable_analyzer = 1; +SELECT 'inner percent / rewrite on, hyperscan off', count() FROM t_or_like_end_anchor WHERE s LIKE 'ta%b' OR s LIKE 'ne%er' SETTINGS optimize_or_like_chain = 1, allow_hyperscan = 0, enable_analyzer = 1; + +-- regexp `(?i)^tenant\-a$` +SELECT 'ilike / rewrite off', count() FROM t_or_like_end_anchor WHERE s ILIKE 'TENANT-A' OR s ILIKE 'NEVER' SETTINGS optimize_or_like_chain = 0, enable_analyzer = 1; +SELECT 'ilike / rewrite on, hyperscan on', count() FROM t_or_like_end_anchor WHERE s ILIKE 'TENANT-A' OR s ILIKE 'NEVER' SETTINGS optimize_or_like_chain = 1, allow_hyperscan = 1, enable_analyzer = 1; +SELECT 'ilike / rewrite on, hyperscan off', count() FROM t_or_like_end_anchor WHERE s ILIKE 'TENANT-A' OR s ILIKE 'NEVER' SETTINGS optimize_or_like_chain = 1, allow_hyperscan = 0, enable_analyzer = 1; + +-- regexp `^$`, which a PCRE `$` lets match a lone newline +SELECT 'empty pattern / rewrite off', count() FROM t_or_like_end_anchor WHERE s LIKE '' OR s LIKE 'never' SETTINGS optimize_or_like_chain = 0, enable_analyzer = 1; +SELECT 'empty pattern / rewrite on, hyperscan on', count() FROM t_or_like_end_anchor WHERE s LIKE '' OR s LIKE 'never' SETTINGS optimize_or_like_chain = 1, allow_hyperscan = 1, enable_analyzer = 1; +SELECT 'empty pattern / rewrite on, hyperscan off', count() FROM t_or_like_end_anchor WHERE s LIKE '' OR s LIKE 'never' SETTINGS optimize_or_like_chain = 1, allow_hyperscan = 0, enable_analyzer = 1; + +-- An anchored branch stays as the original `LIKE` while the rest of the chain is still rewritten. +SELECT 'mixed group / rewrite off', count() FROM t_or_like_end_anchor WHERE s LIKE 'never%' OR s LIKE 'tenant-a' SETTINGS optimize_or_like_chain = 0, enable_analyzer = 1; +SELECT 'mixed group / rewrite on, hyperscan on', count() FROM t_or_like_end_anchor WHERE s LIKE 'never%' OR s LIKE 'tenant-a' SETTINGS optimize_or_like_chain = 1, allow_hyperscan = 1, enable_analyzer = 1; +SELECT 'mixed group / rewrite on, hyperscan off', count() FROM t_or_like_end_anchor WHERE s LIKE 'never%' OR s LIKE 'tenant-a' SETTINGS optimize_or_like_chain = 1, allow_hyperscan = 0, enable_analyzer = 1; + +-- A trailing `%` after a literal `$` emits no anchor (regexp `^a\$`): still eligible, still correct. +SELECT 'escaped dollar / rewrite off', count() FROM t_or_like_end_anchor WHERE s LIKE 'a$%' OR s LIKE 'b$%' SETTINGS optimize_or_like_chain = 0, enable_analyzer = 1; +SELECT 'escaped dollar / rewrite on, hyperscan on', count() FROM t_or_like_end_anchor WHERE s LIKE 'a$%' OR s LIKE 'b$%' SETTINGS optimize_or_like_chain = 1, allow_hyperscan = 1, enable_analyzer = 1; +SELECT 'escaped dollar / rewrite on, hyperscan off', count() FROM t_or_like_end_anchor WHERE s LIKE 'a$%' OR s LIKE 'b$%' SETTINGS optimize_or_like_chain = 1, allow_hyperscan = 0, enable_analyzer = 1; + +-- Prefix patterns emit no anchor (regexp `^tenant`) and keep being rewritten. +SELECT 'prefix / rewrite off', count() FROM t_or_like_end_anchor WHERE s LIKE 'tenant%' OR s LIKE 'never%' SETTINGS optimize_or_like_chain = 0, enable_analyzer = 1; +SELECT 'prefix / rewrite on, hyperscan on', count() FROM t_or_like_end_anchor WHERE s LIKE 'tenant%' OR s LIKE 'never%' SETTINGS optimize_or_like_chain = 1, allow_hyperscan = 1, enable_analyzer = 1; +SELECT 'prefix / rewrite on, hyperscan off', count() FROM t_or_like_end_anchor WHERE s LIKE 'tenant%' OR s LIKE 'never%' SETTINGS optimize_or_like_chain = 1, allow_hyperscan = 0, enable_analyzer = 1; + +DROP TABLE t_or_like_end_anchor; diff --git a/tests/queries/0_stateless/05055_sqlite_insert_identifier_escaping.reference b/tests/queries/0_stateless/05055_sqlite_insert_identifier_escaping.reference new file mode 100644 index 0000000000000..d50220788b0b6 --- /dev/null +++ b/tests/queries/0_stateless/05055_sqlite_insert_identifier_escaping.reference @@ -0,0 +1,20 @@ +--- A1 control: the boundary the injection bypasses is enforced when the path is named directly +PATH_ACCESS_DENIED +--- A2 injection through the remote table name +no such table +objects created in the anchor: none +canary read from outside user_files: none +objects written outside user_files: none +--- A3 injection through a column name +no column named +objects created in the anchor: none +canary read from outside user_files: none +objects written outside user_files: none +--- A4 control: a table and column name that legitimately contain a double quote round-trip +NO_ERROR +11 +--- A5 control: a table name that legitimately contains a backslash round-trips +NO_ERROR +22 +--- A6 a NUL in the remote table name fails loudly, it is never silently truncated +unrecognized token diff --git a/tests/queries/0_stateless/05055_sqlite_insert_identifier_escaping.sh b/tests/queries/0_stateless/05055_sqlite_insert_identifier_escaping.sh new file mode 100755 index 0000000000000..9ec9abb0b9bae --- /dev/null +++ b/tests/queries/0_stateless/05055_sqlite_insert_identifier_escaping.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: Fast tests don't build external libraries (SQLite) + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +ANCHOR="${USER_FILES_PATH}/05055_anchor_${CLICKHOUSE_DATABASE}.db" +# Outside the user_files confinement. +OUTSIDE="${CLICKHOUSE_TMP}/05055_outside_${CLICKHOUSE_DATABASE}.db" + +cleanup() +{ + ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS t_name_inj" + ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS t_col_inj" + ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS t_quote_id" + ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS t_backslash_id" + ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS t_nul_name" + rm -f "${ANCHOR}" "${OUTSIDE}" +} +trap cleanup EXIT +cleanup + +# Odd names: the old backslash escaping produced them, so the injected script's first statement succeeds and the rest runs. +sqlite3 "${ANCHOR}" 'CREATE TABLE "target\"(id INTEGER)' +sqlite3 "${ANCHOR}" 'CREATE TABLE col_anchor("c\" INTEGER)' +sqlite3 "${ANCHOR}" 'CREATE TABLE "ta""ble" ("c""1" INTEGER)' +sqlite3 "${ANCHOR}" 'CREATE TABLE "a\b" (x INTEGER)' +chmod ugo+rw "${ANCHOR}" + +# A database outside user_files, holding a canary the user must never be able to reach. +sqlite3 "${OUTSIDE}" "CREATE TABLE secrets(s TEXT); INSERT INTO secrets VALUES ('CANARY')" +chmod ugo+rw "${OUTSIDE}" + +# One stable token per failure, so the reference embeds no path and no server version. +classify() +{ + local out + out=$(${CLICKHOUSE_CLIENT} --query="$1" 2>&1 \ + | grep -oF -e 'no such table' -e 'no column named' -e 'unrecognized token' \ + -e 'syntax error' -e 'PATH_ACCESS_DENIED' \ + | sed -n 1p) + echo "${out:-NO_ERROR}" +} + +# Read out of band: the ClickHouse read path cannot address these names, so an in-band readback would be a false negative. +injected_state() +{ + sqlite3 "${ANCHOR}" "SELECT 'objects created in the anchor: ' || coalesce((SELECT group_concat(name) FROM (SELECT name FROM sqlite_master WHERE name LIKE '%\_marker' ESCAPE '\' ORDER BY name)), 'none')" + sqlite3 "${ANCHOR}" "SELECT 'canary read from outside user_files: ' || coalesce((SELECT group_concat(s) FROM stolen_marker), 'none')" 2>/dev/null \ + || echo 'canary read from outside user_files: none' + sqlite3 "${OUTSIDE}" "SELECT 'objects written outside user_files: ' || coalesce((SELECT group_concat(name) FROM (SELECT name FROM sqlite_master WHERE name LIKE '%\_marker' ESCAPE '\' ORDER BY name)), 'none')" +} + +echo '--- A1 control: the boundary the injection bypasses is enforced when the path is named directly' +classify "CREATE TABLE t_ctl (s String) ENGINE = SQLite('${OUTSIDE}', 'secrets')" + +echo '--- A2 injection through the remote table name' +${CLICKHOUSE_CLIENT} --query="CREATE TABLE t_name_inj (id UInt32) ENGINE = SQLite('${ANCHOR}', \$\$target\" (id) VALUES (999); ATTACH DATABASE '${OUTSIDE}' AS v; CREATE TABLE stolen_marker AS SELECT s FROM v.secrets; CREATE TABLE v.written_marker(z); DETACH v; --\$\$)" +classify "INSERT INTO t_name_inj VALUES (0)" +injected_state + +echo '--- A3 injection through a column name' +${CLICKHOUSE_CLIENT} --query="CREATE TABLE t_col_inj (\`c') VALUES (1); CREATE TABLE col_marker(z); --\` UInt32) ENGINE = SQLite('${ANCHOR}', 'col_anchor')" +classify "INSERT INTO t_col_inj VALUES (0)" +injected_state + +echo '--- A4 control: a table and column name that legitimately contain a double quote round-trip' +${CLICKHOUSE_CLIENT} --query="CREATE TABLE t_quote_id (\`c\"1\` UInt32) ENGINE = SQLite('${ANCHOR}', 'ta\"ble')" +classify "INSERT INTO t_quote_id VALUES (11)" +sqlite3 "${ANCHOR}" 'SELECT "c""1" FROM "ta""ble"' + +echo '--- A5 control: a table name that legitimately contains a backslash round-trips' +${CLICKHOUSE_CLIENT} --query="CREATE TABLE t_backslash_id (x UInt32) ENGINE = SQLite('${ANCHOR}', 'a\\\\b')" +classify "INSERT INTO t_backslash_id VALUES (22)" +sqlite3 "${ANCHOR}" 'SELECT x FROM "a\b"' + +echo '--- A6 a NUL in the remote table name fails loudly, it is never silently truncated' +${CLICKHOUSE_CLIENT} --query="CREATE TABLE t_nul_name (id UInt32) ENGINE = SQLite('${ANCHOR}', 'a\0b')" +classify "INSERT INTO t_nul_name VALUES (0)" diff --git a/tests/queries/0_stateless/05055_timeseries_deserialize_size.reference b/tests/queries/0_stateless/05055_timeseries_deserialize_size.reference new file mode 100644 index 0000000000000..fedb880a82cb0 --- /dev/null +++ b/tests/queries/0_stateless/05055_timeseries_deserialize_size.reference @@ -0,0 +1,3 @@ +[('1970-01-01 00:00:01.000',1.5),('1970-01-01 00:00:02.000',2.5)] +10000 +12 diff --git a/tests/queries/0_stateless/05055_timeseries_deserialize_size.sql b/tests/queries/0_stateless/05055_timeseries_deserialize_size.sql new file mode 100644 index 0000000000000..93cbd2b3327cf --- /dev/null +++ b/tests/queries/0_stateless/05055_timeseries_deserialize_size.sql @@ -0,0 +1,37 @@ +-- Element counts stored in the states of the time series aggregate functions come from untrusted data, +-- so deserialization must not allocate memory for the claimed number of elements. + +SET allow_experimental_time_series_aggregate_functions = 1; + +-- A valid state of `timeSeriesGroupArray`: format version, number of elements, the timestamps, the values. +SELECT finalizeAggregation(CAST(unhex('0100' || '0200000000000000' || 'E803000000000000' || 'D007000000000000' || '000000000000F83F' || '0000000000000440') AS AggregateFunction(timeSeriesGroupArray, DateTime64(3, 'UTC'), Float64))); + +-- The number of elements is so big that its size in bytes overflows. +SELECT finalizeAggregation(CAST(unhex('0100' || '0000000000000010') AS AggregateFunction(timeSeriesGroupArray, DateTime64(3, 'UTC'), Float64))); -- { serverError CANNOT_READ_ALL_DATA } +SELECT finalizeAggregation(CAST(unhex('0100' || 'FFFFFFFFFFFFFFFF') AS AggregateFunction(timeSeriesGroupArray, DateTime64(3, 'UTC'), Float64))); -- { serverError CANNOT_READ_ALL_DATA } + +-- A state with more elements than are reserved before reading, so the array grows while it is read. +SELECT length(finalizeAggregation(CAST(CAST(state AS String) AS AggregateFunction(timeSeriesGroupArray, DateTime64(3, 'UTC'), Float64)))) +FROM (SELECT timeSeriesGroupArrayState(toDateTime64(number, 3, 'UTC'), number::Float64) AS state FROM numbers(10000)); + +-- The functions that keep samples per bucket, such as `timeSeriesChangesToGrid`, store the number of samples too. +SELECT length(finalizeAggregation(CAST(state AS AggregateFunction(timeSeriesChangesToGrid(10, 120, 10, 70), UInt32, Float64)))) +FROM (SELECT CAST(timeSeriesChangesToGridState(10, 120, 10, 70)(toUInt32(55), toFloat64(1)) AS String) AS state); + +-- The same state with the number of samples of its only bucket (the 8 bytes after the format version, +-- the bucket count, the number of buckets and the bucket index) replaced with a huge value. +SELECT length(finalizeAggregation(CAST(substring(state, 1, 26) || unhex('FFFFFFFFFFFFFFFF') || substring(state, 35) AS AggregateFunction(timeSeriesChangesToGrid(10, 120, 10, 70), UInt32, Float64)))) +FROM (SELECT CAST(timeSeriesChangesToGridState(10, 120, 10, 70)(toUInt32(55), toFloat64(1)) AS String) AS state); -- { serverError CANNOT_READ_ALL_DATA } + +-- The other functions that keep samples per bucket deserialize them independently in this release. +SELECT length(finalizeAggregation(CAST(substring(state, 1, 26) || unhex('FFFFFFFFFFFFFFFF') || substring(state, 35) AS AggregateFunction(timeSeriesRateToGrid(10, 120, 10, 70), UInt32, Float64)))) +FROM (SELECT CAST(timeSeriesRateToGridState(10, 120, 10, 70)(toUInt32(55), toFloat64(1)) AS String) AS state); -- { serverError CANNOT_READ_ALL_DATA } +SELECT length(finalizeAggregation(CAST(substring(state, 1, 26) || unhex('FFFFFFFFFFFFFFFF') || substring(state, 35) AS AggregateFunction(timeSeriesDerivToGrid(10, 120, 10, 70), UInt32, Float64)))) +FROM (SELECT CAST(timeSeriesDerivToGridState(10, 120, 10, 70)(toUInt32(55), toFloat64(1)) AS String) AS state); -- { serverError CANNOT_READ_ALL_DATA } + +-- The number of buckets of the outer state is only checked against the bucket count derived from the function +-- parameters, and a huge grid makes that count enormous, so the buckets must not be reserved before they are +-- read. The memory limit keeps a state claiming all of them from silently allocating gigabytes. +SELECT length(finalizeAggregation(CAST(substring(state, 1, 10) || substring(state, 3, 8) AS AggregateFunction(timeSeriesChangesToGrid(0, 2147483648, 1, 1), UInt32, Float64)))) +FROM (SELECT CAST(timeSeriesChangesToGridState(0, 2147483648, 1, 1)(toUInt32(0), toFloat64(1)) AS String) AS state) +SETTINGS max_memory_usage = 1073741824; -- { serverError CANNOT_READ_ALL_DATA } diff --git a/tests/queries/0_stateless/05056_json_dynamic_native_malformed_structure.reference b/tests/queries/0_stateless/05056_json_dynamic_native_malformed_structure.reference new file mode 100644 index 0000000000000..2c297afeb42e3 --- /dev/null +++ b/tests/queries/0_stateless/05056_json_dynamic_native_malformed_structure.reference @@ -0,0 +1,12 @@ +object_v3: Version 4 of Object structure serialization is written only into MergeTree data parts and is not allowed in Native format. (INCORRECT_DATA) +dynamic_v3: Version 4 of Dynamic structure serialization is written only into MergeTree data parts and is not allowed in Native format. (INCORRECT_DATA) +dynamic_duplicate_types: Duplicate type Int64 in the list of types of a flattened Dynamic column. (INCORRECT_DATA) +dynamic_nothing_type: Type Nothing is not allowed in the list of types of a flattened Dynamic column. (INCORRECT_DATA) +object_duplicate_flattened_path: Duplicate path x in the list of paths of a flattened Object column. (INCORRECT_DATA) +object_duplicate_dynamic_path: Duplicate path in the list of dynamic paths of an Object column. (INCORRECT_DATA) +object_typed_path_collision_flattened: Path x of an Object column is stored as a dynamic path, but it is a typed path in the type of the column. (INCORRECT_DATA) +object_typed_path_collision: Path x of an Object column is stored as a dynamic path, but it is a typed path in the type of the column. (INCORRECT_DATA) +object_duplicate_path_after_realloc: Duplicate path p0 in the list of paths of a flattened Object column. (INCORRECT_DATA) +0 +0 +0 diff --git a/tests/queries/0_stateless/05056_json_dynamic_native_malformed_structure.sh b/tests/queries/0_stateless/05056_json_dynamic_native_malformed_structure.sh new file mode 100755 index 0000000000000..1078ca8f04e0e --- /dev/null +++ b/tests/queries/0_stateless/05056_json_dynamic_native_malformed_structure.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Malformed structure prefixes of JSON/Dynamic columns in a Native block. Each of them used to be +# accepted and then read out of bounds, or to escape as an internal error instead of a data error. + +DATA_DIR=$CLICKHOUSE_TMP/native_structure_$CLICKHOUSE_DATABASE +rm -rf "$DATA_DIR" +mkdir -p "$DATA_DIR" + +python3 - "$DATA_DIR" <<'EOF' +import struct +import sys + +data_dir = sys.argv[1] + +def s(b): + return bytes([len(b)]) + b + +def block(column_type, tail, rows=2): + """One Native block with a single column and a hand-built serialization prefix.""" + return bytes([1, rows]) + s(b'j') + s(column_type) + tail + +def varuint(n): + out = b'' + while True: + byte = n & 0x7f + n >>= 7 + out += bytes([byte | 0x80]) if n else bytes([byte]) + if not n: + return out + +def object_prefix(version, paths): + return struct.pack(' secret String : 0 + COLUMN Const(String) -> \'aes-128-ecb\'_String String : 1 + COLUMN Const(String) -> [HIDDEN id: 1]_String String : 2 + ALIAS secret :: 0 -> __table2.secret String : 3 + FUNCTION encrypt(\'aes-128-ecb\'_String :: 1, __table2.secret :: 3, [HIDDEN id: 1]_String :: 2) -> encrypt(\'aes-128-ecb\'_String, __table2.secret, [HIDDEN id: 1]_String) String : 0 + FUNCTION hex(encrypt(\'aes-128-ecb\'_String, __table2.secret, [HIDDEN id: 1]_String) :: 0) -> hex(encrypt(\'aes-128-ecb\'_String, __table2.secret, [HIDDEN id: 1]_String)) String : 2 + ALIAS hex(encrypt(\'aes-128-ecb\'_String, __table2.secret, [HIDDEN id: 1]_String)) :: 2 -> encrypted_secret String : 0 + ALIAS encrypted_secret :: 0 -> __table1.encrypted_secret String : 2 + ALIAS __table1.encrypted_secret :: 2 -> encrypted_secret String : 0 + FUNCTION materialize(encrypted_secret :: 0) -> materialize(encrypted_secret) String : 2 + ALIAS materialize(encrypted_secret) :: 2 -> encrypted_secret String : 0 + ALIAS encrypted_secret :: 0 -> __table1.encrypted_secret String : 2 + ALIAS __table1.encrypted_secret :: 2 -> encrypted_secret String : 0 +Positions: 0 + ReadFromMemoryStorage +Expression ((Project names + (Projection + (Change column names to column identifiers + (Convert VIEW subquery result to VIEW table structure + (Materialize constants after VIEW subquery + (Project names + (Projection + (Change column names to column identifiers + (Project names + (Projection + Change column names to column identifiers))))))))))) +Actions: INPUT : 0 -> secret String : 0 + COLUMN Const(String) -> \'aes-128-ecb\'_String String : 1 + COLUMN Const(String) -> [HIDDEN id: 1]_String String : 2 + ALIAS secret :: 0 -> __table2.secret String : 3 + FUNCTION encrypt(\'aes-128-ecb\'_String :: 1, __table2.secret :: 3, [HIDDEN id: 1]_String :: 2) -> encrypt(\'aes-128-ecb\'_String, __table2.secret, [HIDDEN id: 1]_String) String : 0 + FUNCTION hex(encrypt(\'aes-128-ecb\'_String, __table2.secret, [HIDDEN id: 1]_String) :: 0) -> hex(encrypt(\'aes-128-ecb\'_String, __table2.secret, [HIDDEN id: 1]_String)) String : 2 + ALIAS hex(encrypt(\'aes-128-ecb\'_String, __table2.secret, [HIDDEN id: 1]_String)) :: 2 -> encrypted_secret String : 0 + ALIAS encrypted_secret :: 0 -> __table1.encrypted_secret String : 2 + ALIAS __table1.encrypted_secret :: 2 -> encrypted_secret String : 0 + FUNCTION materialize(encrypted_secret :: 0) -> materialize(encrypted_secret) String : 2 + ALIAS materialize(encrypted_secret) :: 2 -> encrypted_secret String : 0 + ALIAS encrypted_secret :: 0 -> __table1.encrypted_secret String : 2 + ALIAS __table1.encrypted_secret :: 2 -> encrypted_secret String : 0 +Positions: 0 + ReadFromMemoryStorage +Expression ((Project names + (Projection + (Change column names to column identifiers + (Convert VIEW subquery result to VIEW table structure + (Materialize constants after VIEW subquery + (Project names + (Projection + (Change column names to column identifiers + (Project names + (Projection + Change column names to column identifiers))))))))))) +Actions: INPUT : 0 -> secret String : 0 + COLUMN Const(String) -> _CAST(\'7B3EFB528CEB67F47C02977B6E2FAABB\'_String, \'String\'_String) String : 1 + ALIAS secret :: 0 -> __table2.secret String : 2 + FUNCTION concat(__table2.secret :: 2, _CAST(\'7B3EFB528CEB67F47C02977B6E2FAABB\'_String, \'String\'_String) :: 1) -> concat(__table2.secret, _CAST(\'7B3EFB528CEB67F47C02977B6E2FAABB\'_String, \'String\'_String)) String : 0 + ALIAS concat(__table2.secret, _CAST(\'7B3EFB528CEB67F47C02977B6E2FAABB\'_String, \'String\'_String)) :: 0 -> tagged String : 1 + ALIAS tagged :: 1 -> __table1.tagged String : 0 + ALIAS __table1.tagged :: 0 -> tagged String : 1 + FUNCTION materialize(tagged :: 1) -> materialize(tagged) String : 0 + ALIAS materialize(tagged) :: 0 -> tagged String : 1 + ALIAS tagged :: 1 -> __table1.tagged String : 0 + ALIAS __table1.tagged :: 0 -> tagged String : 1 +Positions: 1 + ReadFromMemoryStorage +Expression ((Project names + (Projection + (Change column names to column identifiers + (Convert VIEW subquery result to VIEW table structure + (Materialize constants after VIEW subquery + (Project names + (Projection + (Change column names to column identifiers + (Project names + Projection)))))))))) +Actions: INPUT : 0 -> __table2.secret String : 0 + ALIAS __table2.secret :: 0 -> secret String : 1 + ALIAS secret :: 1 -> __table1.secret String : 0 + ALIAS __table1.secret :: 0 -> secret String : 1 + FUNCTION materialize(secret :: 1) -> materialize(secret) String : 0 + ALIAS materialize(secret) :: 0 -> secret String : 1 + ALIAS secret :: 1 -> __table1.secret String : 0 + ALIAS __table1.secret :: 0 -> secret String : 1 +Positions: 1 + Filter ((WHERE + Change column names to column identifiers)) + Filter column: notEquals(__table2.secret, _CAST(\'7B3EFB528CEB67F47C02977B6E2FAABB\'_String, \'String\'_String)) (removed) + Actions: INPUT : 0 -> secret String : 0 + COLUMN Const(String) -> _CAST(\'7B3EFB528CEB67F47C02977B6E2FAABB\'_String, \'String\'_String) String : 1 + ALIAS secret : 0 -> __table2.secret String : 2 + FUNCTION notEquals(secret :: 0, _CAST(\'7B3EFB528CEB67F47C02977B6E2FAABB\'_String, \'String\'_String) :: 1) -> notEquals(__table2.secret, _CAST(\'7B3EFB528CEB67F47C02977B6E2FAABB\'_String, \'String\'_String)) UInt8 : 3 + Positions: 3 2 + ReadFromMemoryStorage +[\n {\n "Plan": {\n "Node Type": "Expression",\n "Node Id": "Expression_27",\n "Description": "(Project names + (Projection + (Change column names to column identifiers + (Convert VIEW subquery result to VIEW table structure + (Materialize constants after VIEW subquery + (Project names + (Projection + (Change column names to column identifiers + (Project names + (Projection + Change column names to column identifiers))))))))))",\n "Expression": {\n "Inputs": [\n {\n "Name": "secret",\n "Type": "String"\n }\n ],\n "Actions": [\n {\n "Node Type": "INPUT",\n "Result Type": "String",\n "Result Name": "secret",\n "Arguments": [0],\n "Removed Arguments": [],\n "Result": 0\n },\n {\n "Node Type": "COLUMN",\n "Result Type": "String",\n "Result Name": "\'aes-128-ecb\'_String",\n "Column": "Const(String)",\n "Arguments": [],\n "Removed Arguments": [],\n "Result": 1\n },\n {\n "Node Type": "COLUMN",\n "Result Type": "String",\n "Result Name": "[HIDDEN id: 1]_String",\n "Column": "Const(String)",\n "Arguments": [],\n "Removed Arguments": [],\n "Result": 2\n },\n {\n "Node Type": "ALIAS",\n "Result Type": "String",\n "Result Name": "__table2.secret",\n "Arguments": [0],\n "Removed Arguments": [0],\n "Result": 3\n },\n {\n "Node Type": "FUNCTION",\n "Result Type": "String",\n "Result Name": "encrypt(\'aes-128-ecb\'_String, __table2.secret, [HIDDEN id: 1]_String)",\n "Function": "encrypt",\n "Compiled": false,\n "Arguments": [1, 3, 2],\n "Removed Arguments": [1, 3, 2],\n "Result": 0\n },\n {\n "Node Type": "FUNCTION",\n "Result Type": "String",\n "Result Name": "hex(encrypt(\'aes-128-ecb\'_String, __table2.secret, [HIDDEN id: 1]_String))",\n "Function": "hex",\n "Compiled": false,\n "Arguments": [0],\n "Removed Arguments": [0],\n "Result": 2\n },\n {\n "Node Type": "ALIAS",\n "Result Type": "String",\n "Result Name": "encrypted_secret",\n "Arguments": [2],\n "Removed Arguments": [2],\n "Result": 0\n },\n {\n "Node Type": "ALIAS",\n "Result Type": "String",\n "Result Name": "__table1.encrypted_secret",\n "Arguments": [0],\n "Removed Arguments": [0],\n "Result": 2\n },\n {\n "Node Type": "ALIAS",\n "Result Type": "String",\n "Result Name": "encrypted_secret",\n "Arguments": [2],\n "Removed Arguments": [2],\n "Result": 0\n },\n {\n "Node Type": "FUNCTION",\n "Result Type": "String",\n "Result Name": "materialize(encrypted_secret)",\n "Function": "materialize",\n "Compiled": false,\n "Arguments": [0],\n "Removed Arguments": [0],\n "Result": 2\n },\n {\n "Node Type": "ALIAS",\n "Result Type": "String",\n "Result Name": "encrypted_secret",\n "Arguments": [2],\n "Removed Arguments": [2],\n "Result": 0\n },\n {\n "Node Type": "ALIAS",\n "Result Type": "String",\n "Result Name": "__table1.encrypted_secret",\n "Arguments": [0],\n "Removed Arguments": [0],\n "Result": 2\n },\n {\n "Node Type": "ALIAS",\n "Result Type": "String",\n "Result Name": "encrypted_secret",\n "Arguments": [2],\n "Removed Arguments": [2],\n "Result": 0\n }\n ],\n "Outputs": [\n {\n "Name": "encrypted_secret",\n "Type": "String"\n }\n ],\n "Positions": [0]\n },\n "Plans": [\n {\n "Node Type": "ReadFromMemoryStorage",\n "Node Id": "ReadFromMemoryStorage_6"\n }\n ]\n }\n }\n] +Expression ((Project names + (Projection + (Change column names to column identifiers + (Convert VIEW subquery result to VIEW table structure + (Materialize constants after VIEW subquery + (Project names + (Projection + (Change column names to column identifiers + (Project names + (Projection + Change column names to column identifiers))))))))))) +Header: encrypted_secret String + ReadFromMemoryStorage + Header: secret String +QUERY id: 0 + PROJECTION COLUMNS + encrypted_secret String + PROJECTION + LIST id: 1, nodes: 1 + COLUMN id: 2, column_name: encrypted_secret, result_type: String, source_id: 3 + JOIN TREE + TABLE id: 3, alias: __table1, table_name: default.encrypted_view + SETTINGS enable_analyzer=1 format_display_secrets_in_show_and_select=1 +SELECT * +FROM default.encrypted_view +SETTINGS enable_analyzer = 1, format_display_secrets_in_show_and_select = 1 +(Expression) +ExpressionTransform +Header: secret String: secret String String(size = 0) + (Sorting) + MergeSortingTransform + Header: hex(encrypt(\'aes-128-ecb\'_String, __table2.secret, [HIDDEN id: 1]_String)) String: hex(encrypt(\'aes-128-ecb\'_String, __table2.secret, [HIDDEN id: 1]_String)) String String(size = 0) + __table2.secret String: __table2.secret String String(size = 0) + LimitsCheckingTransform + Header: hex(encrypt(\'aes-128-ecb\'_String, __table2.secret, [HIDDEN id: 1]_String)) String: hex(encrypt(\'aes-128-ecb\'_String, __table2.secret, [HIDDEN id: 1]_String)) String String(size = 0) + __table2.secret String: __table2.secret String String(size = 0) + PartialSortingTransform + Header: hex(encrypt(\'aes-128-ecb\'_String, __table2.secret, [HIDDEN id: 1]_String)) String: hex(encrypt(\'aes-128-ecb\'_String, __table2.secret, [HIDDEN id: 1]_String)) String String(size = 0) + __table2.secret String: __table2.secret String String(size = 0) + (Expression) + ExpressionTransform + Header: hex(encrypt(\'aes-128-ecb\'_String, __table2.secret, [HIDDEN id: 1]_String)) String: hex(encrypt(\'aes-128-ecb\'_String, __table2.secret, [HIDDEN id: 1]_String)) String String(size = 0) + __table2.secret String: __table2.secret String String(size = 0) + (ReadFromMemoryStorage) +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +-- the view itself stays usable for the restricted user +1 diff --git a/tests/queries/0_stateless/05057_explain_hide_secrets_without_privilege.sh b/tests/queries/0_stateless/05057_explain_hide_secrets_without_privilege.sh new file mode 100755 index 0000000000000..56ab7f9a4329d --- /dev/null +++ b/tests/queries/0_stateless/05057_explain_hide_secrets_without_privilege.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-replicated-database +# Tag no-fasttest: the encryption functions are not available in the fast test build +# Tag no-replicated-database: SQL SECURITY DEFINER views and users are set up per-test + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A user with only SELECT on a SQL SECURITY DEFINER view must not be able to read the +# encryption key of the view definition through any flavour of EXPLAIN, no matter which +# value of `format_display_secrets_in_show_and_select` it sets (the stateless test server +# does not enable the `display_secrets_in_show_and_select` server setting, so the full +# gate always hides secrets here). + +user="user_05055_${CLICKHOUSE_DATABASE}_$RANDOM" +db=${CLICKHOUSE_DATABASE} +key='Sixteen byte key' + +${CLICKHOUSE_CLIENT} --query "DROP VIEW IF EXISTS $db.encrypted_view;" +${CLICKHOUSE_CLIENT} --query "DROP VIEW IF EXISTS $db.sorted_view;" +${CLICKHOUSE_CLIENT} --query "DROP VIEW IF EXISTS $db.nested_const_view;" +${CLICKHOUSE_CLIENT} --query "DROP VIEW IF EXISTS $db.where_const_view;" +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS $db.private_plaintext;" + +${CLICKHOUSE_CLIENT} <&1 | grep -o 'ACCESS_DENIED' +} + + +# old analyzer never masks, so EXPLAIN of a secret-bearing plan is refused +old_settings="SETTINGS enable_analyzer = 0" +expect_refused "EXPLAIN actions = 1 SELECT * FROM $db.encrypted_view $old_settings" +expect_refused "EXPLAIN actions = 1 SELECT * FROM $db.nested_const_view $old_settings" +expect_refused "EXPLAIN actions = 1 SELECT * FROM $db.where_const_view $old_settings" +expect_refused "EXPLAIN header = 1 SELECT * FROM $db.encrypted_view $old_settings" +expect_refused "EXPLAIN PIPELINE header = 1 SELECT * FROM $db.sorted_view $old_settings" +expect_refused "EXPLAIN PIPELINE SELECT * FROM $db.sorted_view $old_settings" + +echo "-- the view itself stays usable for the restricted user" +${CLICKHOUSE_CLIENT} --user "$user" --query "SELECT count() FROM $db.encrypted_view" + +${CLICKHOUSE_CLIENT} --query "DROP USER $user" diff --git a/tests/queries/0_stateless/05057_json_part_truncated_shared_data.reference b/tests/queries/0_stateless/05057_json_part_truncated_shared_data.reference new file mode 100644 index 0000000000000..e885136cdbbfe --- /dev/null +++ b/tests/queries/0_stateless/05057_json_part_truncated_shared_data.reference @@ -0,0 +1,4 @@ +paths indexes stream is there +CANNOT_READ_ALL_DATA +structure stream is there +CANNOT_READ_ALL_DATA diff --git a/tests/queries/0_stateless/05057_json_part_truncated_shared_data.sh b/tests/queries/0_stateless/05057_json_part_truncated_shared_data.sh new file mode 100755 index 0000000000000..c76b06b59cc15 --- /dev/null +++ b/tests/queries/0_stateless/05057_json_part_truncated_shared_data.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# Tags: no-darwin +# no-darwin: the names of the streams of a part are hashed there, so the test cannot find the two it truncates. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The ADVANCED shared data serialization of a JSON column keeps the paths indexes and the granule +# structure in their own files, and how much to read from them comes from the sizes stream. A file that +# ends earlier used to be read silently, leaving the paths column shorter than the offsets say, and +# reading the column then went out of bounds. + +DATA_DIR=$CLICKHOUSE_TMP/part_shared_data_$CLICKHOUSE_DATABASE + +# ADVANCED is used for merged parts, so two inserts and a merge are needed to get it. The serialization +# versions are pinned because the test harness randomizes them. +function create_part() +{ + rm -rf "$DATA_DIR" + mkdir -p "$DATA_DIR" + PART=$($CLICKHOUSE_LOCAL --path "$DATA_DIR" -q " + CREATE TABLE t (json JSON(max_dynamic_paths=0)) ENGINE = MergeTree ORDER BY tuple() + SETTINGS min_bytes_for_wide_part = 0, object_serialization_version = 'v3', object_shared_data_serialization_version = 'advanced'; + INSERT INTO t SELECT ('{\"p' || toString(number % 8) || '\":\"' || repeat('Q', 40) || '\"}')::JSON(max_dynamic_paths=0) FROM numbers(200); + INSERT INTO t SELECT ('{\"z' || toString(number % 8) || '\":\"' || repeat('W', 40) || '\"}')::JSON(max_dynamic_paths=0) FROM numbers(200); + OPTIMIZE TABLE t FINAL; + SELECT trim(TRAILING '/' FROM path) FROM system.parts WHERE database = currentDatabase() AND table = 't' AND active + ") + + if [ -z "$PART" ] + then + echo "the table has no active part" + return + fi + + # The parts the merge consumed are still on disk and are read instead if the merged one is broken. + local part + for part in "$(dirname "$PART")"/all_*; do + [ "$part" = "$PART" ] || rm -rf "$part" + done + rm -f "$PART"/checksums.txt +} + +# The stream names are matched, not spelled out, so the test does not depend on the part layout. +function truncate_streams() +{ + local label=$1 + shift + + if [ -s "$1" ] + then + echo "$label is there" + truncate -s 0 "$@" + else + echo "$label is missing, the part contains:" + ls "$PART" 2>&1 | head -10 + fi +} + +function read_part() +{ + $CLICKHOUSE_LOCAL --path "$DATA_DIR" -q "SELECT count(), sum(length(toString(json))) FROM t" 2>&1 \ + | grep -m1 -oE "ATTEMPT_TO_READ_AFTER_EOF|CANNOT_READ_ALL_DATA" +} + +create_part +truncate_streams "paths indexes stream" "$PART"/*object_shared_data*paths_indexes.bin +read_part + +create_part +truncate_streams "structure stream" "$PART"/*object_shared_data*structure.bin +read_part + +rm -rf "$DATA_DIR" diff --git a/tests/queries/0_stateless/05059_json_native_invalid_shared_data_buckets.sql b/tests/queries/0_stateless/05059_json_native_invalid_shared_data_buckets.sql deleted file mode 100644 index 5617f5e2325ce..0000000000000 --- a/tests/queries/0_stateless/05059_json_native_invalid_shared_data_buckets.sql +++ /dev/null @@ -1,13 +0,0 @@ --- Crafted Native blocks whose V3 JSON structure prefix carries an invalid shared-data bucket count. --- The count sizes the per-bucket reader state directly, so it must be rejected before use. --- The structure is passed to `format` explicitly: otherwise schema inference wraps the error into --- CANNOT_EXTRACT_TABLE_STRUCTURE and hides the code the guard raises. - --- buckets = 0 -SELECT count() FROM format('Native', 'j JSON', unhex('0101016a044a534f4e0400000000000000000100')); -- { serverError INCORRECT_DATA } --- buckets = SIZE_MAX -SELECT count() FROM format('Native', 'j JSON', unhex('0101016a044a534f4e04000000000000000001ffffffffffffffffff01')); -- { serverError INCORRECT_DATA } --- buckets = 257, one past the maximum -SELECT count() FROM format('Native', 'j JSON', unhex('0101016a044a534f4e040000000000000000018102')); -- { serverError INCORRECT_DATA } --- buckets = 256, the maximum, passes the guard and fails only on the truncated payload -SELECT count() FROM format('Native', 'j JSON', unhex('0101016a044a534f4e040000000000000000018002')); -- { serverError CANNOT_READ_ALL_DATA, ATTEMPT_TO_READ_AFTER_EOF } diff --git a/tests/queries/0_stateless/05076_additional_table_filters_column_access.reference b/tests/queries/0_stateless/05076_additional_table_filters_column_access.reference new file mode 100644 index 0000000000000..9b0e775e7ff83 --- /dev/null +++ b/tests/queries/0_stateless/05076_additional_table_filters_column_access.reference @@ -0,0 +1,132 @@ +-- analyzer=1: 01 control +ACCESS_DENIED +-- analyzer=1: 02 filter over denied column +ACCESS_DENIED +-- analyzer=1: 03 filter over denied column, qualified key and expression +ACCESS_DENIED +-- analyzer=1: 04 filter over denied tuple subcolumn +ACCESS_DENIED +-- analyzer=1: 05 filter over alias of a denied column +ACCESS_DENIED +-- analyzer=1: 07 filter over denied column with PREWHERE +ACCESS_DENIED +-- analyzer=1: 08 filter over granted column +1 +-- analyzer=1: 09 constant filter +2 +-- analyzer=1: 10 invoker view, filter keyed by underlying table over denied column +ACCESS_DENIED +-- analyzer=1: 11 definer view, filter keyed by underlying table over denied column +2 +-- analyzer=1: 12 definer view, filter keyed by the view itself +1 +-- analyzer=1: 13 distributed table, filter keyed by itself over denied column +ACCESS_DENIED +-- analyzer=1: 14 distributed table, filter keyed by itself over granted column +1 +-- analyzer=1: 15 parallel replicas custom key (sampling) over denied column +ACCESS_DENIED +-- analyzer=1: 16 parallel replicas custom key (range) over denied column +ACCESS_DENIED +-- analyzer=1: 17 parallel replicas custom key (sampling) over granted column +1 +-- analyzer=1: 18 custom key shipped to the replicas, over denied column +ACCESS_DENIED +-- analyzer=1: 19 custom key shipped to the replicas, over granted column +2 +-- analyzer=1: 20 definer view, custom key over a view column shadowing a denied table column +2 +-- analyzer=1: 21 none view, custom key over a view column shadowing a denied table column +2 +-- analyzer=1: 22 definer view, custom key over a denied table column that the view does not expose +UNKNOWN_IDENTIFIER +-- analyzer=1: 23 definer view, custom key over a view column +1 +-- analyzer=1: 24 none view, filter keyed by underlying table over denied column +2 +-- analyzer=1: 25 definer view, custom key supplied by the invoker, activated by the definer's profile +2 +-- analyzer=1: 26 filter over granted column, qualified key and expression +1 +-- analyzer=1: 27 parallel replicas custom key (sampling) over granted column, qualified +1 +-- analyzer=0: 01 control +ACCESS_DENIED +-- analyzer=0: 02 filter over denied column +ACCESS_DENIED +-- analyzer=0: 03 filter over denied column, qualified key and expression +ACCESS_DENIED +-- analyzer=0: 04 filter over denied tuple subcolumn +ACCESS_DENIED +-- analyzer=0: 05 filter over alias of a denied column +ACCESS_DENIED +-- analyzer=0: 07 filter over denied column with PREWHERE +ACCESS_DENIED +-- analyzer=0: 08 filter over granted column +1 +-- analyzer=0: 09 constant filter +2 +-- analyzer=0: 10 invoker view, filter keyed by underlying table over denied column +ACCESS_DENIED +-- analyzer=0: 11 definer view, filter keyed by underlying table over denied column +2 +-- analyzer=0: 12 definer view, filter keyed by the view itself +1 +-- analyzer=0: 13 distributed table, filter keyed by itself over denied column +ACCESS_DENIED +-- analyzer=0: 14 distributed table, filter keyed by itself over granted column +1 +-- analyzer=0: 15 parallel replicas custom key (sampling) over denied column +ACCESS_DENIED +-- analyzer=0: 16 parallel replicas custom key (range) over denied column +ACCESS_DENIED +-- analyzer=0: 17 parallel replicas custom key (sampling) over granted column +1 +-- analyzer=0: 18 custom key shipped to the replicas, over denied column +ACCESS_DENIED +-- analyzer=0: 19 custom key shipped to the replicas, over granted column +2 +-- analyzer=0: 20 definer view, custom key over a view column shadowing a denied table column +2 +-- analyzer=0: 21 none view, custom key over a view column shadowing a denied table column +2 +-- analyzer=0: 22 definer view, custom key over a denied table column that the view does not expose +UNKNOWN_IDENTIFIER +-- analyzer=0: 23 definer view, custom key over a view column +1 +-- analyzer=0: 24 none view, filter keyed by underlying table over denied column +2 +-- analyzer=0: 25 definer view, custom key supplied by the invoker, activated by the definer's profile +2 +-- analyzer=0: 26 filter over granted column, qualified key and expression +1 +-- analyzer=0: 27 parallel replicas custom key (sampling) over granted column, qualified +1 +-- analyzer=1: 28 definer parameterized view, filter over denied view column +ACCESS_DENIED +-- analyzer=1: 29 definer parameterized view, qualified key, filter over denied view column +ACCESS_DENIED +-- analyzer=1: 30 definer parameterized view, filter over granted view column +1 +-- analyzer=1: 31 definer parameterized view, no filter +2 +-- analyzer=1: 37 definer view with SAMPLE, control +100 +-- analyzer=1: 38 definer view with SAMPLE, invoker supplies the replica slice +100 +-- analyzer=1: 39 SQL SECURITY NONE view with SAMPLE, invoker supplies the replica slice +100 +-- analyzer=1: 40 definer view with SAMPLE read through Distributed by three followers in sampling_key mode +100 +-- analyzer=1: 41 SQL SECURITY NONE view with SAMPLE read through Distributed by three followers in sampling_key mode +100 +-- analyzer=0: 37 definer view with SAMPLE, control +100 +-- analyzer=0: 38 definer view with SAMPLE, invoker supplies the replica slice +100 +-- analyzer=0: 39 SQL SECURITY NONE view with SAMPLE, invoker supplies the replica slice +100 +-- analyzer=0: 40 definer view with SAMPLE read through Distributed by three followers in sampling_key mode +100 +-- analyzer=0: 41 SQL SECURITY NONE view with SAMPLE read through Distributed by three followers in sampling_key mode +100 diff --git a/tests/queries/0_stateless/05076_additional_table_filters_column_access.sh b/tests/queries/0_stateless/05076_additional_table_filters_column_access.sh new file mode 100755 index 0000000000000..f579cd828c455 --- /dev/null +++ b/tests/queries/0_stateless/05076_additional_table_filters_column_access.sh @@ -0,0 +1,227 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# additional_table_filters must be gated by column-level SELECT access on the columns it references. + +DB=${CLICKHOUSE_DATABASE} +T='prod_customer_token_25_8_secret' + +u_low="u_low_${CLICKHOUSE_TEST_UNIQUE_NAME}" +u_view="u_view_${CLICKHOUSE_TEST_UNIQUE_NAME}" +u_def="u_def_${CLICKHOUSE_TEST_UNIQUE_NAME}" +p_def="p_def_${CLICKHOUSE_TEST_UNIQUE_NAME}" + +$CLICKHOUSE_CLIENT -n -q " +DROP USER IF EXISTS $u_low, $u_view, $u_def; +DROP SETTINGS PROFILE IF EXISTS $p_def; + +CREATE TABLE t (id UInt8, public_label String, secret_token String, tup Tuple(a String), alias_col String ALIAS secret_token) ENGINE = MergeTree ORDER BY id; +INSERT INTO t VALUES (1, 'customer_1', 'prod_customer_token_25_8_secret', ('ta')), (2, 'customer_2', 'other_customer_private_token', ('tb')); +CREATE TABLE d (id UInt8, public_label String, secret_token String) ENGINE = Distributed(test_shard_localhost, currentDatabase(), t); +CREATE TABLE d3 (id UInt8, public_label String, secret_token String) ENGINE = Distributed(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), t); +CREATE VIEW v_inv SQL SECURITY INVOKER AS SELECT id, public_label FROM t; +CREATE VIEW v_def SQL SECURITY DEFINER DEFINER = CURRENT_USER AS SELECT id, public_label FROM t; +-- The view column secret_token shadows the denied table column: a custom key over it resolves on the view, +-- and must not be re-evaluated over the table inside the DEFINER/NONE body. +CREATE VIEW v_def_shadow SQL SECURITY DEFINER DEFINER = CURRENT_USER AS SELECT id, public_label, '' AS secret_token FROM t; +CREATE VIEW v_none_shadow SQL SECURITY NONE AS SELECT id, public_label, '' AS secret_token FROM t; +-- The DEFINER body reads secret_token from t as the definer, so the view's own column-level grants are the only +-- gate for a filter keyed by the parameterized view. +CREATE VIEW pv_def SQL SECURITY DEFINER DEFINER = CURRENT_USER AS SELECT id, public_label, secret_token FROM t WHERE id >= {min_id:UInt8}; + +-- A sampled body: the definer picks the sample, and the invoker must not be able to narrow it down further. +CREATE TABLE ts (id UInt8) ENGINE = MergeTree ORDER BY id SAMPLE BY id; +INSERT INTO ts SELECT number FROM numbers(100); +CREATE VIEW v_def_sample SQL SECURITY DEFINER DEFINER = CURRENT_USER AS SELECT count() AS c FROM ts SAMPLE 1/2; +CREATE VIEW v_none_sample SQL SECURITY NONE AS SELECT count() AS c FROM ts SAMPLE 1/2; +CREATE TABLE d_def_sample (c UInt64) ENGINE = Distributed(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), v_def_sample); +CREATE TABLE d_none_sample (c UInt64) ENGINE = Distributed(test_cluster_one_shard_three_replicas_localhost, currentDatabase(), v_none_sample); + +CREATE USER $u_low IDENTIFIED WITH plaintext_password BY 'password'; +CREATE USER $u_view IDENTIFIED WITH plaintext_password BY 'password'; +-- The definer's profile activates custom-key parallel replicas, so the invoker only has to supply the key. +CREATE USER $u_def IDENTIFIED WITH plaintext_password BY 'password'; +CREATE SETTINGS PROFILE $p_def SETTINGS cluster_for_parallel_replicas = 'test_shard_localhost', max_parallel_replicas = 2, allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_mode = 'custom_key_sampling' TO $u_def; +GRANT SELECT ON $DB.t TO $u_def; +CREATE VIEW v_def_prof SQL SECURITY DEFINER DEFINER = $u_def AS SELECT id, public_label FROM t; + +GRANT SELECT(id, public_label) ON $DB.t TO $u_low; +GRANT SELECT(id, public_label) ON $DB.d TO $u_low; +GRANT SELECT(id, public_label) ON $DB.d3 TO $u_low; +GRANT SELECT ON $DB.v_inv TO $u_low; +GRANT SELECT ON $DB.v_def TO $u_view; +GRANT SELECT ON $DB.v_def_shadow TO $u_view; +GRANT SELECT ON $DB.v_none_shadow TO $u_view; +GRANT SELECT ON $DB.v_def_prof TO $u_view; +GRANT SELECT(id, public_label) ON $DB.pv_def TO $u_low; +GRANT SELECT ON $DB.v_def_sample TO $u_view; +GRANT SELECT ON $DB.v_none_sample TO $u_view; +GRANT SELECT ON $DB.d_def_sample TO $u_view; +GRANT SELECT ON $DB.d_none_sample TO $u_view; +" + +# Every case is a separate query, so run them over HTTP: a client process per query is too slow under sanitizers. +# On success print only the result; on failure print the error code name from the trailing (CODE_NAME) token. +function run() +{ + local user=$1 + local query=$2 + local out + out=$(${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&user=${user}&password=password" --data-binary "$query") + if [[ "$out" == *"DB::Exception"* ]]; then + grep -oE '\([A-Z_]+\)' <<< "$out" | tail -1 | tr -d '()' + else + echo "$out" + fi +} + +for analyzer in 1 0; do + +echo "-- analyzer=$analyzer: 01 control" +run "$u_low" "SELECT count() FROM t WHERE secret_token = '$T' SETTINGS enable_analyzer=$analyzer" + +echo "-- analyzer=$analyzer: 02 filter over denied column" +run "$u_low" "SELECT count() FROM t SETTINGS enable_analyzer=$analyzer, additional_table_filters = {'t': 'secret_token = ''$T'''}" + +echo "-- analyzer=$analyzer: 03 filter over denied column, qualified key and expression" +run "$u_low" "SELECT count() FROM t SETTINGS enable_analyzer=$analyzer, additional_table_filters = {'$DB.t': '$DB.t.secret_token = ''$T'''}" + +echo "-- analyzer=$analyzer: 04 filter over denied tuple subcolumn" +run "$u_low" "SELECT count() FROM t SETTINGS enable_analyzer=$analyzer, additional_table_filters = {'t': 'tup.a = ''ta'''}" + +echo "-- analyzer=$analyzer: 05 filter over alias of a denied column" +run "$u_low" "SELECT count() FROM t SETTINGS enable_analyzer=$analyzer, additional_table_filters = {'t': 'alias_col = ''$T'''}" + +echo "-- analyzer=$analyzer: 07 filter over denied column with PREWHERE" +run "$u_low" "SELECT count() FROM t PREWHERE id = 1 SETTINGS enable_analyzer=$analyzer, additional_table_filters = {'t': 'secret_token = ''$T'''}" + +echo "-- analyzer=$analyzer: 08 filter over granted column" +run "$u_low" "SELECT count() FROM t SETTINGS enable_analyzer=$analyzer, additional_table_filters = {'t': 'public_label = ''customer_1'''}" + +echo "-- analyzer=$analyzer: 09 constant filter" +run "$u_low" "SELECT count() FROM t SETTINGS enable_analyzer=$analyzer, additional_table_filters = {'t': '1'}" + +echo "-- analyzer=$analyzer: 10 invoker view, filter keyed by underlying table over denied column" +run "$u_low" "SELECT count() FROM v_inv SETTINGS enable_analyzer=$analyzer, additional_table_filters = {'$DB.t': 'secret_token = ''$T'''}" + +echo "-- analyzer=$analyzer: 11 definer view, filter keyed by underlying table over denied column" +run "$u_view" "SELECT count() FROM v_def SETTINGS enable_analyzer=$analyzer, additional_table_filters = {'$DB.t': 'secret_token = ''$T'''}" + +echo "-- analyzer=$analyzer: 12 definer view, filter keyed by the view itself" +run "$u_view" "SELECT count() FROM v_def SETTINGS enable_analyzer=$analyzer, additional_table_filters = {'$DB.v_def': 'id = 1'}" + +echo "-- analyzer=$analyzer: 13 distributed table, filter keyed by itself over denied column" +run "$u_low" "SELECT count() FROM d SETTINGS enable_analyzer=$analyzer, additional_table_filters = {'$DB.d': 'secret_token = ''$T'''}" + +echo "-- analyzer=$analyzer: 14 distributed table, filter keyed by itself over granted column" +run "$u_low" "SELECT count() FROM d SETTINGS enable_analyzer=$analyzer, additional_table_filters = {'$DB.d': 'public_label = ''customer_1'''}" + +echo "-- analyzer=$analyzer: 15 parallel replicas custom key (sampling) over denied column" +run "$u_low" "SELECT count() FROM t SETTINGS enable_analyzer=$analyzer, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_shard_localhost', max_parallel_replicas = 2, allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_mode = 'custom_key_sampling', parallel_replicas_count = 2, parallel_replica_offset = 1, parallel_replicas_custom_key = 'secret_token = ''$T'''" + +echo "-- analyzer=$analyzer: 16 parallel replicas custom key (range) over denied column" +run "$u_low" "SELECT count() FROM t SETTINGS enable_analyzer=$analyzer, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_shard_localhost', max_parallel_replicas = 2, allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_mode = 'custom_key_range', parallel_replicas_custom_key = 'cityHash64(secret_token)', parallel_replicas_custom_key_range_lower = 0, parallel_replicas_custom_key_range_upper = 1000, parallel_replicas_count = 2, parallel_replica_offset = 1" + +echo "-- analyzer=$analyzer: 17 parallel replicas custom key (sampling) over granted column" +run "$u_low" "SELECT count() FROM t SETTINGS enable_analyzer=$analyzer, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_shard_localhost', max_parallel_replicas = 2, allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_mode = 'custom_key_sampling', parallel_replicas_custom_key = 'id', parallel_replicas_count = 2, parallel_replica_offset = 1" + +# One shard, three replicas: the key is shipped to the replicas; sum() merges the per-replica partial counts. +echo "-- analyzer=$analyzer: 18 custom key shipped to the replicas, over denied column" +run "$u_low" "SELECT sum(c) FROM (SELECT count() AS c FROM d3) SETTINGS enable_analyzer=$analyzer, max_parallel_replicas = 3, allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_mode = 'custom_key_sampling', parallel_replicas_custom_key = 'secret_token = ''$T'''" + +echo "-- analyzer=$analyzer: 19 custom key shipped to the replicas, over granted column" +run "$u_low" "SELECT sum(c) FROM (SELECT count() AS c FROM d3) SETTINGS enable_analyzer=$analyzer, max_parallel_replicas = 3, allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_mode = 'custom_key_sampling', parallel_replicas_custom_key = 'id'" + +# The invoker's key is applied to the view's own columns, where it is 0 for every row, so replica 0 of 2 keeps +# both rows. The DEFINER/NONE body reads without the key: re-evaluated over t as the definer, the key +# would be 1 for the row with the real secret, that row would drop out, and the count would reveal it. +echo "-- analyzer=$analyzer: 20 definer view, custom key over a view column shadowing a denied table column" +run "$u_view" "SELECT count() FROM v_def_shadow SETTINGS enable_analyzer=$analyzer, cluster_for_parallel_replicas = 'test_shard_localhost', max_parallel_replicas = 2, allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_mode = 'custom_key_sampling', parallel_replicas_count = 2, parallel_replica_offset = 0, parallel_replicas_custom_key = 'secret_token = ''$T'''" + +echo "-- analyzer=$analyzer: 21 none view, custom key over a view column shadowing a denied table column" +run "$u_view" "SELECT count() FROM v_none_shadow SETTINGS enable_analyzer=$analyzer, cluster_for_parallel_replicas = 'test_shard_localhost', max_parallel_replicas = 2, allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_mode = 'custom_key_sampling', parallel_replicas_count = 2, parallel_replica_offset = 0, parallel_replicas_custom_key = 'secret_token = ''$T'''" + +echo "-- analyzer=$analyzer: 22 definer view, custom key over a denied table column that the view does not expose" +run "$u_view" "SELECT count() FROM v_def SETTINGS enable_analyzer=$analyzer, cluster_for_parallel_replicas = 'test_shard_localhost', max_parallel_replicas = 2, allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_mode = 'custom_key_sampling', parallel_replicas_count = 2, parallel_replica_offset = 1, parallel_replicas_custom_key = 'secret_token = ''$T'''" + +echo "-- analyzer=$analyzer: 23 definer view, custom key over a view column" +run "$u_view" "SELECT count() FROM v_def SETTINGS enable_analyzer=$analyzer, cluster_for_parallel_replicas = 'test_shard_localhost', max_parallel_replicas = 2, allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_mode = 'custom_key_sampling', parallel_replicas_count = 2, parallel_replica_offset = 1, parallel_replicas_custom_key = 'id'" + +echo "-- analyzer=$analyzer: 24 none view, filter keyed by underlying table over denied column" +run "$u_view" "SELECT count() FROM v_none_shadow SETTINGS enable_analyzer=$analyzer, additional_table_filters = {'$DB.t': 'secret_token = ''$T'''}" + +# The invoker's context does not activate custom-key parallel replicas, so the outer query never applies the key. +# The definer's profile activates them inside the body, where the invoker's key over the denied column would be +# evaluated as the definer: replica 1 of 2 would keep only the row with the real secret and the count would be 1. +echo "-- analyzer=$analyzer: 25 definer view, custom key supplied by the invoker, activated by the definer's profile" +run "$u_view" "SELECT count() FROM v_def_prof SETTINGS enable_analyzer=$analyzer, parallel_replicas_count = 2, parallel_replica_offset = 1, parallel_replicas_custom_key = 'secret_token = ''$T'''" + +# Table-qualified names are resolved against the table of the query, the same way as in a WHERE predicate. +echo "-- analyzer=$analyzer: 26 filter over granted column, qualified key and expression" +run "$u_low" "SELECT count() FROM t SETTINGS enable_analyzer=$analyzer, additional_table_filters = {'$DB.t': '$DB.t.public_label = ''customer_1'''}" + +echo "-- analyzer=$analyzer: 27 parallel replicas custom key (sampling) over granted column, qualified" +run "$u_low" "SELECT count() FROM t SETTINGS enable_analyzer=$analyzer, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_shard_localhost', max_parallel_replicas = 2, allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_mode = 'custom_key_sampling', parallel_replicas_custom_key = '$DB.t.id', parallel_replicas_count = 2, parallel_replica_offset = 1" + +done + +# The analyzer resolves a parameterized view as a table function node wrapping the view storage, and the filter +# access check has a dedicated branch for it. The legacy interpreter does not apply additional_table_filters +# keyed by a parameterized view at all, so these cases are analyzer-only. +echo "-- analyzer=1: 28 definer parameterized view, filter over denied view column" +run "$u_low" "SELECT count() FROM pv_def(min_id = 1) SETTINGS enable_analyzer=1, additional_table_filters = {'pv_def': 'secret_token = ''$T'''}" + +echo "-- analyzer=1: 29 definer parameterized view, qualified key, filter over denied view column" +run "$u_low" "SELECT count() FROM pv_def(min_id = 1) SETTINGS enable_analyzer=1, additional_table_filters = {'$DB.pv_def': 'secret_token = ''$T'''}" + +echo "-- analyzer=1: 30 definer parameterized view, filter over granted view column" +run "$u_low" "SELECT count() FROM pv_def(min_id = 1) SETTINGS enable_analyzer=1, additional_table_filters = {'pv_def': 'public_label = ''customer_1'''}" + +echo "-- analyzer=1: 31 definer parameterized view, no filter" +run "$u_low" "SELECT count() FROM pv_def(min_id = 1) SETTINGS enable_analyzer=1" + +# `parallel_replicas_count` and `parallel_replica_offset` are internal settings that only the initiator may set. +# `MergeTreeDataSelectExecutor` splits an active SAMPLE into `parallel_replicas_count` pieces and reads the piece +# numbered `parallel_replica_offset` even when parallel replicas are disabled, so an invoker-supplied pair would +# pick a slice of the definer's sample: the full sample is 100 rows, the second half of it is 36. +for analyzer in 1 0; do + +echo "-- analyzer=$analyzer: 37 definer view with SAMPLE, control" +run "$u_view" "SELECT c FROM v_def_sample SETTINGS enable_analyzer=$analyzer" + +echo "-- analyzer=$analyzer: 38 definer view with SAMPLE, invoker supplies the replica slice" +run "$u_view" "SELECT c FROM v_def_sample SETTINGS enable_analyzer=$analyzer, parallel_replicas_count = 2, parallel_replica_offset = 1" + +echo "-- analyzer=$analyzer: 39 SQL SECURITY NONE view with SAMPLE, invoker supplies the replica slice" +run "$u_view" "SELECT c FROM v_none_sample SETTINGS enable_analyzer=$analyzer, parallel_replicas_count = 2, parallel_replica_offset = 1" + +echo "-- analyzer=$analyzer: 40 definer view with SAMPLE read through Distributed by three followers in sampling_key mode" +run "$u_view" "SELECT sum(c) FROM d_def_sample SETTINGS enable_analyzer=$analyzer, max_parallel_replicas = 3, allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_mode = 'sampling_key', prefer_localhost_replica = 0" + +echo "-- analyzer=$analyzer: 41 SQL SECURITY NONE view with SAMPLE read through Distributed by three followers in sampling_key mode" +run "$u_view" "SELECT sum(c) FROM d_none_sample SETTINGS enable_analyzer=$analyzer, max_parallel_replicas = 3, allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_mode = 'sampling_key', prefer_localhost_replica = 0" + +done + +$CLICKHOUSE_CLIENT -n -q " +DROP TABLE IF EXISTS d_none_sample; +DROP TABLE IF EXISTS d_def_sample; +DROP VIEW IF EXISTS v_none_sample; +DROP VIEW IF EXISTS v_def_sample; +DROP TABLE IF EXISTS ts; +DROP VIEW IF EXISTS pv_def; +DROP VIEW IF EXISTS v_def_prof; +DROP VIEW IF EXISTS v_none_shadow; +DROP VIEW IF EXISTS v_def_shadow; +DROP VIEW IF EXISTS v_def; +DROP VIEW IF EXISTS v_inv; +DROP TABLE IF EXISTS d3; +DROP TABLE IF EXISTS d; +DROP TABLE IF EXISTS t; +DROP USER IF EXISTS $u_low, $u_view, $u_def; +DROP SETTINGS PROFILE IF EXISTS $p_def; +" + diff --git a/tests/queries/0_stateless/05076_truncate_database_on_cluster_access.reference b/tests/queries/0_stateless/05076_truncate_database_on_cluster_access.reference new file mode 100644 index 0000000000000..6a9664f10a342 --- /dev/null +++ b/tests/queries/0_stateless/05076_truncate_database_on_cluster_access.reference @@ -0,0 +1,11 @@ +-- TRUNCATE DATABASE ON CLUSTER, without DROP DATABASE +necessary to have the grant DROP DATABASE +1 +-- TRUNCATE ALL TABLES FROM ON CLUSTER, without DROP DATABASE +necessary to have the grant DROP DATABASE +3 +-- TRUNCATE DATABASE ON CLUSTER, with DROP DATABASE but readonly +Cannot execute query in readonly mode +1 +-- TRUNCATE DATABASE ON CLUSTER, with DROP DATABASE +0 diff --git a/tests/queries/0_stateless/05076_truncate_database_on_cluster_access.sh b/tests/queries/0_stateless/05076_truncate_database_on_cluster_access.sh new file mode 100755 index 0000000000000..ff8611be7442a --- /dev/null +++ b/tests/queries/0_stateless/05076_truncate_database_on_cluster_access.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# TRUNCATE of a database is authorized by `DROP DATABASE`, the same privilege the local spelling +# requires. The `ON CLUSTER` form is checked on the initiator before the task is queued, so a +# refused statement leaves the database untouched. + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +CLUSTER="test_shard_localhost" +# Users are server-global, so their names carry the test database to keep this test safe against a +# concurrent copy of itself. +DB="db_05076_${CLICKHOUSE_TEST_UNIQUE_NAME}" +READER="reader_05076_${CLICKHOUSE_TEST_UNIQUE_NAME}" +OWNER="owner_05076_${CLICKHOUSE_TEST_UNIQUE_NAME}" + +function cleanup() +{ + ${CLICKHOUSE_CLIENT} -mq " + DROP DATABASE IF EXISTS ${DB} SYNC; + DROP USER IF EXISTS ${READER}, ${OWNER}; + " +} +cleanup +trap cleanup EXIT + +${CLICKHOUSE_CLIENT} -mq " + CREATE USER ${READER} IDENTIFIED WITH no_password; + CREATE USER ${OWNER} IDENTIFIED WITH no_password; + GRANT CLUSTER ON *.* TO ${READER}, ${OWNER}; +" + +function fixture() +{ + ${CLICKHOUSE_CLIENT} -mq " + DROP DATABASE IF EXISTS ${DB} SYNC; + CREATE DATABASE ${DB}; + CREATE TABLE ${DB}.t (x UInt32) ENGINE = MergeTree ORDER BY x; + INSERT INTO ${DB}.t VALUES (1), (2), (3); + GRANT SELECT ON ${DB}.* TO ${READER}; + GRANT DROP DATABASE ON ${DB}.* TO ${OWNER}; + " +} + +# The message names the privilege that was required, so a check that asks for a different one is +# caught here and not only by the outcome. +echo "-- TRUNCATE DATABASE ON CLUSTER, without DROP DATABASE" +fixture +${CLICKHOUSE_CLIENT} --user "${READER}" --query "TRUNCATE DATABASE ${DB} ON CLUSTER ${CLUSTER}" 2>&1 | + grep -m1 -o -F "necessary to have the grant DROP DATABASE" +${CLICKHOUSE_CLIENT} --query "EXISTS TABLE ${DB}.t" + +# The second parse shape of the same statement: it names tables, so it empties them instead of +# dropping them. It reaches the same authorization branch. +echo "-- TRUNCATE ALL TABLES FROM ON CLUSTER, without DROP DATABASE" +fixture +${CLICKHOUSE_CLIENT} --user "${READER}" --query "TRUNCATE ALL TABLES FROM ${DB} ON CLUSTER ${CLUSTER}" 2>&1 | + grep -m1 -o -F "necessary to have the grant DROP DATABASE" +${CLICKHOUSE_CLIENT} --query "SELECT count() FROM ${DB}.t" + +# `readonly` is passed on the command line so that it is applied together with the settings the test +# harness already sends: constraints are checked against the value in force before the query, so +# both are accepted and the statement itself runs readonly. A user-level `readonly = 1` would reject +# those settings first, and this assertion would then hold without the statement ever being +# authorized. +echo "-- TRUNCATE DATABASE ON CLUSTER, with DROP DATABASE but readonly" +fixture +${CLICKHOUSE_CLIENT} --user "${OWNER}" --readonly 1 --query "TRUNCATE DATABASE ${DB} ON CLUSTER ${CLUSTER}" 2>&1 | + grep -m1 -o -F "Cannot execute query in readonly mode" +${CLICKHOUSE_CLIENT} --query "EXISTS TABLE ${DB}.t" + +# `DROP DATABASE` is enough on its own: a check that required anything besides it would refuse here. +echo "-- TRUNCATE DATABASE ON CLUSTER, with DROP DATABASE" +fixture +${CLICKHOUSE_CLIENT} --user "${OWNER}" --distributed_ddl_output_mode none --query "TRUNCATE DATABASE ${DB} ON CLUSTER ${CLUSTER}" +${CLICKHOUSE_CLIENT} --query "EXISTS TABLE ${DB}.t" diff --git a/tests/queries/0_stateless/05082_kill_and_system_on_cluster_access_types.reference b/tests/queries/0_stateless/05082_kill_and_system_on_cluster_access_types.reference new file mode 100644 index 0000000000000..e9efacb33c69d --- /dev/null +++ b/tests/queries/0_stateless/05082_kill_and_system_on_cluster_access_types.reference @@ -0,0 +1,12 @@ +KILL TRANSACTION -> KILL TRANSACTION ON *.* +KILL PART_MOVE_TO_SHARD -> SELECT ON system.part_moves_between_shards +SYSTEM STOP THREAD FUZZER -> SYSTEM THREAD FUZZER ON *.* +SYSTEM START THREAD FUZZER -> SYSTEM THREAD FUZZER ON *.* +SYSTEM RESET COVERAGE -> SYSTEM ON *.* +KILL TRANSACTION without system.transactions -> SELECT ON system.transactions +KILL PART_MOVE_TO_SHARD without move privileges -> ALTER MOVE PARTITION, MOVE PARTITION BETWEEN SHARDS ON *.* +no CLUSTER grant -> CLUSTER ON *.* +KILL TRANSACTION local -> allowed +KILL TRANSACTION on cluster -> allowed +KILL PART_MOVE_TO_SHARD local -> allowed +KILL PART_MOVE_TO_SHARD on cluster -> allowed diff --git a/tests/queries/0_stateless/05082_kill_and_system_on_cluster_access_types.sh b/tests/queries/0_stateless/05082_kill_and_system_on_cluster_access_types.sh new file mode 100755 index 0000000000000..9147c664dc395 --- /dev/null +++ b/tests/queries/0_stateless/05082_kill_and_system_on_cluster_access_types.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# Tags: no-encrypted-storage + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Holds CLUSTER and nothing else. +cluster_user="cluster_$CLICKHOUSE_TEST_UNIQUE_NAME" +# Holds every statement privilege but not CLUSTER. +no_cluster_user="no_cluster_$CLICKHOUSE_TEST_UNIQUE_NAME" +# Hold the complete privilege set of one statement each. +kill_txn_user="kill_txn_$CLICKHOUSE_TEST_UNIQUE_NAME" +move_user="move_$CLICKHOUSE_TEST_UNIQUE_NAME" +# Hold all but one privilege of one statement each. +partial_txn_user="partial_txn_$CLICKHOUSE_TEST_UNIQUE_NAME" +partial_move_user="partial_move_$CLICKHOUSE_TEST_UNIQUE_NAME" + +function cleanup() +{ + $CLICKHOUSE_CLIENT -mq " + DROP USER IF EXISTS $cluster_user; + DROP USER IF EXISTS $no_cluster_user; + DROP USER IF EXISTS $kill_txn_user; + DROP USER IF EXISTS $move_user; + DROP USER IF EXISTS $partial_txn_user; + DROP USER IF EXISTS $partial_move_user; + " +} +cleanup +trap cleanup EXIT + +# CLUSTER is a global privilege, so it cannot share a GRANT with a table-scoped one. +$CLICKHOUSE_CLIENT -mq " + CREATE USER $cluster_user, $no_cluster_user, $kill_txn_user, $move_user, + $partial_txn_user, $partial_move_user IDENTIFIED WITH no_password; + + GRANT CLUSTER ON *.* TO $cluster_user; + + GRANT KILL TRANSACTION, SYSTEM THREAD FUZZER, SYSTEM ON *.* TO $no_cluster_user; + + GRANT CLUSTER, KILL TRANSACTION ON *.* TO $kill_txn_user; + GRANT SELECT ON system.transactions TO $kill_txn_user; + + GRANT CLUSTER, ALTER MOVE PARTITION, MOVE PARTITION BETWEEN SHARDS ON *.* TO $move_user; + GRANT SELECT ON system.part_moves_between_shards TO $move_user; + + GRANT CLUSTER, KILL TRANSACTION ON *.* TO $partial_txn_user; + + GRANT CLUSTER ON *.* TO $partial_move_user; + GRANT SELECT ON system.part_moves_between_shards TO $partial_move_user; +" + +cluster="test_shard_localhost" +# Match no live transaction and no live part move, so every allowed arm below is a no-op. The +# transaction predicate deliberately does not compare `tid`: its tuple shape is not the same in +# every build, so a literal tuple fails type analysis before the access check this test is about. +txn_predicate="tid_hash = 0" +task_uuid="'00000000-0000-0000-0000-000000000000'" + +# Report the privilege the server asked for and let the reference hold the expected mapping. The +# privilege name is what discriminates: an ACCESS_DENIED-only assertion would also pass when the +# CLUSTER check, which runs before the required-access check, is what fired. +required_privilege() { + $CLICKHOUSE_CLIENT --distributed_ddl_output_mode none --user "$1" --query "$2" 2>&1 | + sed -n "/necessary to have the grant/{s/.*grant \(.*\)\. (ACCESS_DENIED).*/\1/p;q;}" +} + +# Each statement must name over ON CLUSTER the same privilege its local spelling names. +while IFS= read -r statement; do + echo "${statement%% ON CLUSTER*} -> $(required_privilege "$cluster_user" "$statement")" +done < $(required_privilege "$partial_txn_user" "KILL TRANSACTION ON CLUSTER $cluster WHERE $txn_predicate")" +echo "KILL PART_MOVE_TO_SHARD without move privileges -> $(required_privilege "$partial_move_user" "KILL PART_MOVE_TO_SHARD ON CLUSTER $cluster WHERE task_uuid = $task_uuid")" + +# In-range control: holding the statement privileges without CLUSTER is refused by the earlier check, +# so a mapping that refuses everything would not produce the five lines above. +echo "no CLUSTER grant -> $(required_privilege "$no_cluster_user" "KILL TRANSACTION ON CLUSTER $cluster WHERE $txn_predicate")" + +# The two statements whose privileges can be granted in full are allowed in both spellings, which +# proves the new elements are a gate rather than an unconditional refusal. The local half is asserted +# too: a user who is cluster-allowed while locally refused is the bypass this test exists to catch. +# The thread fuzzer and coverage statements get no allowed arm, because executing them would change +# server-global state that concurrent tests read and neither takes an argument that lets the host +# reject them harmlessly. +allowed() { + local out + out=$($CLICKHOUSE_CLIENT --distributed_ddl_output_mode none --user "$1" --query "$2" 2>&1) + if [ -z "$out" ]; then + echo "$3 -> allowed" + else + echo "$3 -> FAIL: $out" + fi +} + +allowed "$kill_txn_user" "KILL TRANSACTION WHERE $txn_predicate" "KILL TRANSACTION local" +allowed "$kill_txn_user" "KILL TRANSACTION ON CLUSTER $cluster WHERE $txn_predicate" "KILL TRANSACTION on cluster" +allowed "$move_user" "KILL PART_MOVE_TO_SHARD WHERE task_uuid = $task_uuid" "KILL PART_MOVE_TO_SHARD local" +allowed "$move_user" "KILL PART_MOVE_TO_SHARD ON CLUSTER $cluster WHERE task_uuid = $task_uuid" "KILL PART_MOVE_TO_SHARD on cluster" diff --git a/tests/queries/0_stateless/05083_nullable_key_exact_count.reference b/tests/queries/0_stateless/05083_nullable_key_exact_count.reference new file mode 100644 index 0000000000000..ca112be5f1f75 --- /dev/null +++ b/tests/queries/0_stateless/05083_nullable_key_exact_count.reference @@ -0,0 +1,22 @@ +the rows a negated equality really returns +s s +count of the same condition +1 +1 +and of a negated set +1 +1 +and of a negated pattern +1 +1 +a key of mostly NULLs +11 +11 +4 +12 +a nullable key without NULLs +15 +15 +the exact count of a non-nullable key +1 +16 diff --git a/tests/queries/0_stateless/05083_nullable_key_exact_count.sql b/tests/queries/0_stateless/05083_nullable_key_exact_count.sql new file mode 100644 index 0000000000000..45365a07c7917 --- /dev/null +++ b/tests/queries/0_stateless/05083_nullable_key_exact_count.sql @@ -0,0 +1,68 @@ +-- Tags: no-parallel-replicas, no-replicated-database +-- no-parallel-replicas: the EXPLAIN assertion below requires the exact-count projection to +-- short-circuit on the initiator, which is suppressed when reading from remote replicas. +-- no-replicated-database: EXPLAIN output differs for a replicated database. + +-- `SELECT count()` answered from the implicit exact-count projection reads a granule's key range +-- instead of its rows. A NULL key value is analysed as the `+inf` stand-in of the `NULLS LAST` order, +-- and a comparison is NULL - so `WHERE` rejects the row - for such a key, which the two-valued range +-- algebra cannot express: a granule of NULLs looked like a definite match of a negated comparison, so +-- `count()` counted rows that the same query without the projection, and `SELECT *`, do not return. + +SET optimize_use_projections = 1, optimize_use_implicit_projections = 1; + +DROP TABLE IF EXISTS t_nullable_key; +CREATE TABLE t_nullable_key (id Nullable(String), s Nullable(String)) ENGINE = MergeTree ORDER BY (id, s) +SETTINGS allow_nullable_key = 1, index_granularity = 1; +INSERT INTO t_nullable_key VALUES ('s', 's'), (NULL, 's1'), (NULL, NULL); + +SELECT 'the rows a negated equality really returns'; +SELECT id, s FROM t_nullable_key WHERE id != ''; + +SELECT 'count of the same condition'; +SELECT count() FROM t_nullable_key WHERE id != ''; +SELECT count() FROM t_nullable_key WHERE id != '' SETTINGS optimize_use_implicit_projections = 0; + +SELECT 'and of a negated set'; +SELECT count() FROM t_nullable_key WHERE id NOT IN (''); +SELECT count() FROM t_nullable_key WHERE id NOT IN ('') SETTINGS optimize_use_implicit_projections = 0; + +SELECT 'and of a negated pattern'; +SELECT count() FROM t_nullable_key WHERE id NOT LIKE 'z%'; +SELECT count() FROM t_nullable_key WHERE id NOT LIKE 'z%' SETTINGS optimize_use_implicit_projections = 0; + +-- More NULLs than values, so a count that trusts the key range is off by more than one row. +DROP TABLE IF EXISTS t_nullable_key_wide; +CREATE TABLE t_nullable_key_wide (id Nullable(UInt32)) ENGINE = MergeTree ORDER BY id +SETTINGS allow_nullable_key = 1, index_granularity = 2; +INSERT INTO t_nullable_key_wide SELECT if(number % 4 = 0, NULL, number) FROM numbers(16); + +SELECT 'a key of mostly NULLs'; +SELECT count() FROM t_nullable_key_wide WHERE id != 1; +SELECT count() FROM t_nullable_key_wide WHERE id != 1 SETTINGS optimize_use_implicit_projections = 0; +SELECT count() FROM t_nullable_key_wide WHERE id IS NULL; +SELECT count() FROM t_nullable_key_wide WHERE id IS NOT NULL; + +-- A key that holds no NULL, so no row is rejected by three-valued logic. +DROP TABLE IF EXISTS t_nullable_key_without_nulls; +CREATE TABLE t_nullable_key_without_nulls (id Nullable(UInt32)) ENGINE = MergeTree ORDER BY id +SETTINGS allow_nullable_key = 1, index_granularity = 2; +INSERT INTO t_nullable_key_without_nulls SELECT number FROM numbers(16); + +SELECT 'a nullable key without NULLs'; +SELECT count() FROM t_nullable_key_without_nulls WHERE id != 1; +SELECT count() FROM t_nullable_key_without_nulls WHERE id != 1 SETTINGS optimize_use_implicit_projections = 0; + +-- The exact-count projection is still used where NULL cannot be involved. +DROP TABLE IF EXISTS t_key; +CREATE TABLE t_key (id UInt32) ENGINE = MergeTree ORDER BY id; +INSERT INTO t_key SELECT number FROM numbers(16); + +SELECT 'the exact count of a non-nullable key'; +SELECT count() > 0 FROM (EXPLAIN SELECT count() FROM t_key WHERE id != 100) WHERE explain ILIKE '%_exact_count_projection%'; +SELECT count() FROM t_key WHERE id != 100; + +DROP TABLE t_key; +DROP TABLE t_nullable_key_without_nulls; +DROP TABLE t_nullable_key_wide; +DROP TABLE t_nullable_key; diff --git a/tests/queries/0_stateless/05136_database_filesystem_missing_path_on_load.reference b/tests/queries/0_stateless/05136_database_filesystem_missing_path_on_load.reference new file mode 100644 index 0000000000000..1273bb862222f --- /dev/null +++ b/tests/queries/0_stateless/05136_database_filesystem_missing_path_on_load.reference @@ -0,0 +1,11 @@ +--- first run: create the database over an existing directory --- +fsdb +--- second run: the directory is gone; the database still loads and serves no tables --- +fsdb +0 +--- a user replay of that same definition is rejected (1 = the expected BAD_ARGUMENTS) --- +1 +1 +--- the database is still reachable through a replay on start, and drops --- +fsdb +0 diff --git a/tests/queries/0_stateless/05136_database_filesystem_missing_path_on_load.sh b/tests/queries/0_stateless/05136_database_filesystem_missing_path_on_load.sh new file mode 100755 index 0000000000000..d1a0c7484eeeb --- /dev/null +++ b/tests/queries/0_stateless/05136_database_filesystem_missing_path_on_load.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# A `Filesystem` database is rebuilt from its stored `ATTACH DATABASE` statement on every start, and metadata +# loading stops at the first exception. A backing directory that has been removed in the meantime must +# therefore leave the database attached without tables, not make the server refuse to start. + +WORKING_FOLDER="${CLICKHOUSE_TMP}/05136_database_filesystem_missing_path_on_load" +rm -rf "${WORKING_FOLDER}" +mkdir -p "${WORKING_FOLDER}/files" + +echo "--- first run: create the database over an existing directory ---" +${CLICKHOUSE_LOCAL} --path "${WORKING_FOLDER}/store" \ + -q "CREATE DATABASE fsdb ENGINE = Filesystem('${WORKING_FOLDER}/files'); + SELECT name FROM system.databases WHERE name = 'fsdb'" + +rm -rf "${WORKING_FOLDER}/files" + +echo "--- second run: the directory is gone; the database still loads and serves no tables ---" +${CLICKHOUSE_LOCAL} --path "${WORKING_FOLDER}/store" \ + -q "SELECT name FROM system.databases WHERE name = 'fsdb'; + SELECT count() FROM system.tables WHERE database = 'fsdb'" + +# The definition a start has just replayed must still be refused to a user who replays it by hand, plainly or +# wrapped in a clause that runs its children as internal queries. Each rejection ends its invocation, so they +# run one per start, and the start after each one replays the definition and attaches the database again. +echo "--- a user replay of that same definition is rejected (1 = the expected BAD_ARGUMENTS) ---" +${CLICKHOUSE_LOCAL} --path "${WORKING_FOLDER}/store" \ + -q "DETACH DATABASE fsdb; + ATTACH DATABASE fsdb" 2>&1 | grep -c "Code: 36.*BAD_ARGUMENTS" +${CLICKHOUSE_LOCAL} --path "${WORKING_FOLDER}/store" \ + -q "DETACH DATABASE fsdb; + ATTACH DATABASE fsdb PARALLEL WITH DROP TABLE IF EXISTS no_such_table_05136" 2>&1 \ + | grep -c "Code: 36.*BAD_ARGUMENTS" + +echo "--- the database is still reachable through a replay on start, and drops ---" +${CLICKHOUSE_LOCAL} --path "${WORKING_FOLDER}/store" \ + -q "SELECT name FROM system.databases WHERE name = 'fsdb'; + DROP DATABASE fsdb; + SELECT count() FROM system.databases WHERE name = 'fsdb'" + +rm -rf "${WORKING_FOLDER}" diff --git a/tests/queries/0_stateless/05136_json_shared_data_bucketed_map_without_statistics.reference b/tests/queries/0_stateless/05136_json_shared_data_bucketed_map_without_statistics.reference new file mode 100644 index 0000000000000..23a4dd0460b91 --- /dev/null +++ b/tests/queries/0_stateless/05136_json_shared_data_bucketed_map_without_statistics.reference @@ -0,0 +1,2 @@ +Map(String, UInt64) {'k':1} Map(String, Tuple(a UInt64)) {'k':(2)} +1 2 diff --git a/tests/queries/0_stateless/05136_json_shared_data_bucketed_map_without_statistics.sql b/tests/queries/0_stateless/05136_json_shared_data_bucketed_map_without_statistics.sql new file mode 100644 index 0000000000000..b6da283b5a30f --- /dev/null +++ b/tests/queries/0_stateless/05136_json_shared_data_bucketed_map_without_statistics.sql @@ -0,0 +1,17 @@ +DROP TABLE IF EXISTS json_shared_bucketed_map; +CREATE TABLE json_shared_bucketed_map (j JSON(max_dynamic_paths = 0)) +ENGINE = MergeTree ORDER BY tuple() +SETTINGS min_bytes_for_wide_part = 0, + object_shared_data_serialization_version_for_zero_level_parts = 'advanced', + map_serialization_version_for_zero_level_parts = 'with_buckets', + propagate_types_serialization_versions_to_nested_types = 1; + +-- Preserve runtime `Map` types in shared data instead of parsing them as `JSON` objects. +INSERT INTO json_shared_bucketed_map +SELECT * FROM format(RowBinary, 'j JSON(max_dynamic_paths = 0)', concat( + unhex('02016d'), formatRowNoNewline('RowBinary', CAST(map('k', toUInt64(1)) AS Dynamic)), + unhex('0174'), formatRowNoNewline('RowBinary', CAST(map('k', CAST(tuple(toUInt64(2)) AS Tuple(a UInt64))) AS Dynamic)))) +SETTINGS input_format_binary_read_json_as_string = 0; +SELECT dynamicType(j.m), j.m, dynamicType(j.t), j.t FROM json_shared_bucketed_map; +SELECT j.m.:`Map(String, UInt64)`['k'], j.t.:`Map(String, Tuple(a UInt64))`['k'].a FROM json_shared_bucketed_map; +DROP TABLE json_shared_bucketed_map; diff --git a/tests/queries/0_stateless/05139_dwarf_format_header_types.reference b/tests/queries/0_stateless/05139_dwarf_format_header_types.reference new file mode 100644 index 0000000000000..3558851467561 --- /dev/null +++ b/tests/queries/0_stateless/05139_dwarf_format_header_types.reference @@ -0,0 +1,18 @@ +-- native types are returned as they are +LowCardinality(UInt64) 0 +LowCardinality(String) compile_unit +-- a different requested type is converted +UInt64 0 +String 0 +LowCardinality(String) 0 +UInt64 31 +String compile_unit +LowCardinality(String) 0 +Array(String) 10 +Array(Tuple(Int64, Int64)) 46 +-- inserting into a table whose column types differ from the format +1 1 +-- a type that cannot be converted is a normal error +TYPE_MISMATCH +-- an unknown column name is a normal error +THERE_IS_NO_COLUMN diff --git a/tests/queries/0_stateless/05139_dwarf_format_header_types.sh b/tests/queries/0_stateless/05139_dwarf_format_header_types.sh new file mode 100755 index 0000000000000..0570005632bc6 --- /dev/null +++ b/tests/queries/0_stateless/05139_dwarf_format_header_types.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-msan, no-ubsan + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +FILE="$CURDIR/data_dwarf/tiny_dwarf5" + +# The DWARF format has a fixed schema, but the caller may request other types for its columns. +# The produced columns must be converted to the requested types, otherwise every reader downstream +# interprets their memory as the wrong type. + +# INSERT ... FORMAT below makes the client read stdin, so it is closed explicitly. +$CLICKHOUSE_LOCAL -m --query " +select '-- native types are returned as they are'; +select toTypeName(unit_offset), unit_offset from file('$FILE', DWARF, 'offset UInt64, unit_offset LowCardinality(UInt64)') order by offset limit 1 settings allow_suspicious_low_cardinality_types = 1; +select toTypeName(tag), tag from file('$FILE', DWARF, 'offset UInt64, tag LowCardinality(String)') order by offset limit 1; + +select '-- a different requested type is converted'; +select toTypeName(unit_offset), unit_offset from file('$FILE', DWARF, 'offset UInt64, unit_offset UInt64') order by offset limit 1; +select toTypeName(unit_offset), unit_offset from file('$FILE', DWARF, 'offset UInt64, unit_offset String') order by offset limit 1; +select toTypeName(unit_offset), unit_offset from file('$FILE', DWARF, 'offset UInt64, unit_offset LowCardinality(String)') order by offset limit 1; +select toTypeName(size), size from file('$FILE', DWARF, 'offset UInt64, size UInt64') order by offset limit 1; +select toTypeName(tag), tag from file('$FILE', DWARF, 'offset UInt64, tag String') order by offset limit 1; +select toTypeName(name), empty(name) from file('$FILE', DWARF, 'offset UInt64, name LowCardinality(String)') order by offset limit 1; +select toTypeName(attr_str), length(attr_str) from file('$FILE', DWARF, 'offset UInt64, attr_str Array(String)') order by offset limit 1; +select toTypeName(ranges), length(ranges) from file('$FILE', DWARF, 'offset UInt64, ranges Array(Tuple(Int64, Int64))') order by offset limit 1; + +select '-- inserting into a table whose column types differ from the format'; +create table dwarf_dest (offset UInt64, unit_offset LowCardinality(String), attr_str Array(String)) engine = MergeTree order by offset; +insert into dwarf_dest from infile '$FILE' format DWARF; +select count() > 0, countDistinct(unit_offset) > 0 from dwarf_dest; +drop table dwarf_dest; +" < /dev/null + +echo "-- a type that cannot be converted is a normal error" +$CLICKHOUSE_LOCAL -q "select * from file('$FILE', DWARF, 'unit_offset Array(UUID)')" 2>&1 | grep -o -m1 "CANNOT_CONVERT_TYPE\|ILLEGAL_TYPE_OF_ARGUMENT\|TYPE_MISMATCH" + +echo "-- an unknown column name is a normal error" +$CLICKHOUSE_LOCAL -q "select * from file('$FILE', DWARF, 'bogus_column String')" 2>&1 | grep -o -m1 "THERE_IS_NO_COLUMN" diff --git a/tests/queries/0_stateless/05141_http_dictionary_hide_header_values.reference b/tests/queries/0_stateless/05141_http_dictionary_hide_header_values.reference new file mode 100644 index 0000000000000..42de9969ea9e5 --- /dev/null +++ b/tests/queries/0_stateless/05141_http_dictionary_hide_header_values.reference @@ -0,0 +1,9 @@ +CREATE DICTIONARY default.d_05141\n(\n `id` UInt64,\n `v` String\n)\nPRIMARY KEY id\nSOURCE(HTTP(URL \'http://localhost:11111/x.tsv\' FORMAT \'TabSeparated\' CREDENTIALS (USER \'user\' PASSWORD \'[HIDDEN]\') HEADERS (\'[HIDDEN]\')))\nLIFETIME(MIN 0 MAX 0)\nLAYOUT(FLAT()) +0 0 1 +d_05141_flat HEADERS \'[HIDDEN]\')) LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT()) +d_05141_foo HEADERS (\'[HIDDEN]\'))) LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT()) +d_05141_key HEADERS (\'[HIDDEN]\'))) LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT()) +d_05141_nested HEADERS (\'[HIDDEN]\'))) LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT()) +d_05141_nobr HEADERS (\'[HIDDEN]\'))) LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT()) +d_05141_typo HEADERS (\'[HIDDEN]\'))) LIFETIME(MIN 0 MAX 0) LAYOUT(FLAT()) +2 13 0 diff --git a/tests/queries/0_stateless/05141_http_dictionary_hide_header_values.sql b/tests/queries/0_stateless/05141_http_dictionary_hide_header_values.sql new file mode 100644 index 0000000000000..1731e3590ef08 --- /dev/null +++ b/tests/queries/0_stateless/05141_http_dictionary_hide_header_values.sql @@ -0,0 +1,65 @@ +-- The values of custom HTTP headers of an `HTTP` dictionary source often carry credentials, +-- so they must be hidden in `SHOW CREATE DICTIONARY`, `system.tables` and `system.query_log`, +-- the same way as the password. They are hidden as a whole, header names included. + +SET format_display_secrets_in_show_and_select = 0; + +DROP DICTIONARY IF EXISTS d_05141; +CREATE DICTIONARY d_05141 (id UInt64, v String) +PRIMARY KEY id +SOURCE(HTTP( + url 'http://localhost:11111/x.tsv' + format 'TabSeparated' + credentials(user 'user' password 'SEKRIT_PW') + headers( + header(name 'API-KEY' value 'SEKRIT_TOKEN_1') + header(name 'X-Other' value 'SEKRIT_TOKEN_2') + ) +)) +LIFETIME(0) LAYOUT(FLAT()); + +SHOW CREATE DICTIONARY d_05141; + +SELECT create_table_query LIKE concat('%', 'SEKRIT', '%'), create_table_query LIKE '%API-KEY%', create_table_query LIKE '%HEADERS (\'[HIDDEN]\')%' +FROM system.tables WHERE database = currentDatabase() AND name = 'd_05141'; + +DROP DICTIONARY d_05141; + +-- The query is logged before the dictionary source validates its structure, so malformed header +-- definitions must not leak either. +CREATE DICTIONARY d_05141_typo (id UInt64, v String) PRIMARY KEY id +SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers(header(name 'API-KEY' vaule 'SEKRIT_TYPO')))) +LIFETIME(0) LAYOUT(FLAT()); +CREATE DICTIONARY d_05141_key (id UInt64, v String) PRIMARY KEY id +SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers(header(secret 'SEKRIT_KEY')))) +LIFETIME(0) LAYOUT(FLAT()); +CREATE DICTIONARY d_05141_nested (id UInt64, v String) PRIMARY KEY id +SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers(header(name(foo 'SEKRIT_NESTED'))))) +LIFETIME(0) LAYOUT(FLAT()); +CREATE DICTIONARY d_05141_func (id UInt64, v String) PRIMARY KEY id +SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers(header(name concat('X-', 'SEKRIT_FUNC') value 'SEKRIT_FUNC_VALUE')))) +LIFETIME(0) LAYOUT(FLAT()); -- { serverError INCORRECT_DICTIONARY_DEFINITION } +CREATE DICTIONARY d_05141_nobr (id UInt64, v String) PRIMARY KEY id +SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers(header 'SEKRIT_NOBR'))) +LIFETIME(0) LAYOUT(FLAT()); +CREATE DICTIONARY d_05141_foo (id UInt64, v String) PRIMARY KEY id +SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers(foo 'SEKRIT_FOO'))) +LIFETIME(0) LAYOUT(FLAT()); +CREATE DICTIONARY d_05141_flat (id UInt64, v String) PRIMARY KEY id +SOURCE(HTTP(url 'http://localhost:11111/x.tsv' format 'TabSeparated' headers 'SEKRIT_HEADERS')) +LIFETIME(0) LAYOUT(FLAT()); + +SELECT name, extract(create_table_query, 'HEADERS.*$') +FROM system.tables WHERE database = currentDatabase() AND name LIKE 'd\\_05141\\_%' ORDER BY name; + +SYSTEM FLUSH LOGS query_log; +SELECT countIf(query LIKE '%d\\_05141 %'), countIf(query LIKE '%d\\_05141\\_%'), countIf(query LIKE concat('%', 'SEKRIT', '%')) +FROM system.query_log +WHERE current_database = currentDatabase() AND query_kind = 'Create' AND event_date >= yesterday(); + +DROP DICTIONARY d_05141_typo; +DROP DICTIONARY d_05141_key; +DROP DICTIONARY d_05141_nested; +DROP DICTIONARY d_05141_nobr; +DROP DICTIONARY d_05141_foo; +DROP DICTIONARY d_05141_flat; diff --git a/tests/queries/0_stateless/05148_database_backup_modify_comment.reference b/tests/queries/0_stateless/05148_database_backup_modify_comment.reference new file mode 100644 index 0000000000000..b3863579a9540 --- /dev/null +++ b/tests/queries/0_stateless/05148_database_backup_modify_comment.reference @@ -0,0 +1,4 @@ +10 +10 +a comment +refused diff --git a/tests/queries/0_stateless/05148_database_backup_modify_comment.sh b/tests/queries/0_stateless/05148_database_backup_modify_comment.sh new file mode 100755 index 0000000000000..4af56ac37b84b --- /dev/null +++ b/tests/queries/0_stateless/05148_database_backup_modify_comment.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Tags: no-encrypted-storage + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +BACKUP_DATABASE_NAME=${CLICKHOUSE_TEST_UNIQUE_NAME}_backup +ATTACHED_DATABASE_NAME=${CLICKHOUSE_TEST_UNIQUE_NAME}_attached + +$CLICKHOUSE_CLIENT -q """ +DROP DATABASE IF EXISTS $BACKUP_DATABASE_NAME; +CREATE DATABASE $BACKUP_DATABASE_NAME; + +CREATE TABLE $BACKUP_DATABASE_NAME.test_table (id UInt64) ENGINE = MergeTree ORDER BY id; +INSERT INTO $BACKUP_DATABASE_NAME.test_table SELECT number FROM numbers(10); + +BACKUP DATABASE $BACKUP_DATABASE_NAME TO Disk('backups', '$BACKUP_DATABASE_NAME') FORMAT Null; + +DROP DATABASE IF EXISTS $ATTACHED_DATABASE_NAME; +CREATE DATABASE $ATTACHED_DATABASE_NAME ENGINE = Backup('$BACKUP_DATABASE_NAME', Disk('backups', '$BACKUP_DATABASE_NAME')); + +SELECT count() FROM $ATTACHED_DATABASE_NAME.test_table; +""" + +# A comment change rewrites the metadata file of the database, so the locator has to survive the +# round trip: loading it back is what the server does at every start. +$CLICKHOUSE_CLIENT -q """ +ALTER DATABASE $ATTACHED_DATABASE_NAME MODIFY COMMENT 'a comment'; +DETACH DATABASE $ATTACHED_DATABASE_NAME; +ATTACH DATABASE $ATTACHED_DATABASE_NAME; + +SELECT count() FROM $ATTACHED_DATABASE_NAME.test_table; +SELECT comment FROM system.databases WHERE name = '$ATTACHED_DATABASE_NAME'; +""" + +# A locator serialized as a string literal is accepted only while loading metadata an older server +# rewrote: in a statement a user writes it must be the function it is, because that is the form the +# secret masker redacts. +$CLICKHOUSE_CLIENT -q """ +DROP DATABASE $ATTACHED_DATABASE_NAME; +CREATE DATABASE $ATTACHED_DATABASE_NAME ENGINE = Backup('$BACKUP_DATABASE_NAME', 'Disk(\\'backups\\', \\'$BACKUP_DATABASE_NAME\\')'); +""" 2>&1 | grep -q -F 'Expected function' && echo 'refused' + +$CLICKHOUSE_CLIENT -q """ +DROP DATABASE IF EXISTS $ATTACHED_DATABASE_NAME; +DROP DATABASE $BACKUP_DATABASE_NAME; +""" diff --git a/tests/queries/0_stateless/05153_database_filesystem_cache_read_on_file_grant.reference b/tests/queries/0_stateless/05153_database_filesystem_cache_read_on_file_grant.reference new file mode 100644 index 0000000000000..008a72adf0b06 --- /dev/null +++ b/tests/queries/0_stateless/05153_database_filesystem_cache_read_on_file_grant.reference @@ -0,0 +1,9 @@ +READ ON FILE +7 +READ ON FILE +READ ON FILE +1 +1 +7 +CREATE TEMPORARY TABLE +0 diff --git a/tests/queries/0_stateless/05153_database_filesystem_cache_read_on_file_grant.sh b/tests/queries/0_stateless/05153_database_filesystem_cache_read_on_file_grant.sh new file mode 100755 index 0000000000000..91219eebf0249 --- /dev/null +++ b/tests/queries/0_stateless/05153_database_filesystem_cache_read_on_file_grant.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash + +# A `Filesystem` database caches every table it resolves, keyed on the table name alone, and hands +# the cached storage back to every later caller. Resolving a table of such a database requires the +# read source grant on the cached path as well as on the uncached one. +# https://github.com/ClickHouse/ClickHouse/issues/118042 + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +user="user_05153_${CLICKHOUSE_DATABASE}_$RANDOM" +fs_db="db_05153_${CLICKHOUSE_DATABASE}_$RANDOM" +dir="${USER_FILES_PATH}/d_05153_${CLICKHOUSE_DATABASE}_$RANDOM" + +mkdir -p "$dir" +echo 7 > "$dir/warm.csv" +echo 7 > "$dir/cold.csv" + +# The source grant is the variable under test, so the table grant is deliberately broad and cannot +# confound the result. It also shows that it does not confer the source grant by itself. +${CLICKHOUSE_CLIENT} <&1 | grep -c "READ ON FILE") >= 1 )) && echo "READ ON FILE" || echo "UNEXPECTED"; +# ... a user holding the grant caches the storage for that name, ... +${CLICKHOUSE_CLIENT} --query "SELECT * FROM $fs_db.\`warm.csv\`"; +# ... and the cached name must be refused the same way. +(( $(${CLICKHOUSE_CLIENT} --user "$user" --query "SELECT * FROM $fs_db.\`warm.csv\`" 2>&1 | grep -c "READ ON FILE") >= 1 )) && echo "READ ON FILE" || echo "UNEXPECTED"; +# The grant is checked before the filesystem is observed, so a name that does not exist reports the +# access error instead of telling the user whether the file is there. +(( $(${CLICKHOUSE_CLIENT} --user "$user" --query "SELECT * FROM $fs_db.\`missing.csv\`" 2>&1 | grep -c "READ ON FILE") >= 1 )) && echo "READ ON FILE" || echo "UNEXPECTED"; +# `EXISTS TABLE` needs only `SHOW TABLES`, and it reads the same cache and probes the same directory, +# so without the grant it must answer alike for a cached name and for one that is not there at all. +${CLICKHOUSE_CLIENT} --user "$user" --query "EXISTS TABLE $fs_db.\`warm.csv\`"; +${CLICKHOUSE_CLIENT} --user "$user" --query "EXISTS TABLE $fs_db.\`missing.csv\`"; + +${CLICKHOUSE_CLIENT} --query "GRANT READ ON FILE TO $user"; + +# With the grant, the cached name is served. A name that was never resolved takes the other path, +# which still requires `CREATE TEMPORARY TABLE`: the contrast is what shows that the query above was +# answered from the cache and not by resolving the file again. +${CLICKHOUSE_CLIENT} --user "$user" --query "SELECT * FROM $fs_db.\`warm.csv\`"; +(( $(${CLICKHOUSE_CLIENT} --user "$user" --query "SELECT * FROM $fs_db.\`cold.csv\`" 2>&1 | grep -c "CREATE TEMPORARY TABLE") >= 1 )) && echo "CREATE TEMPORARY TABLE" || echo "UNEXPECTED"; +# The grant restores the honest answer, so the two lines above are the check and not a constant. +${CLICKHOUSE_CLIENT} --user "$user" --query "EXISTS TABLE $fs_db.\`missing.csv\`"; + +${CLICKHOUSE_CLIENT} </dev/null)" == "0" ]] +then + rm -rd "$dir" +fi diff --git a/tests/queries/0_stateless/05153_settings_masking_non_string_value.reference b/tests/queries/0_stateless/05153_settings_masking_non_string_value.reference new file mode 100644 index 0000000000000..1779b1e66e24b --- /dev/null +++ b/tests/queries/0_stateless/05153_settings_masking_non_string_value.reference @@ -0,0 +1,5 @@ +(UNKNOWN_SETTING) +(UNKNOWN_SETTING) +(UNKNOWN_SETTING) +in query: SET nats_password = '[HIDDEN]', nats_url = 4222 +1 1 diff --git a/tests/queries/0_stateless/05153_settings_masking_non_string_value.sh b/tests/queries/0_stateless/05153_settings_masking_non_string_value.sh new file mode 100755 index 0000000000000..f83d603c095f1 --- /dev/null +++ b/tests/queries/0_stateless/05153_settings_masking_non_string_value.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash + +# Masking a setting value must never throw. `nats_url`, `rabbitmq_address` and +# `after_processing_move_connection_string` hold a URL or a connection string whose credential is +# masked in place, and masking runs before the setting is validated, so the value can be of any +# type. Writing the port where the URL goes used to throw `BAD_GET` out of the masking, and the +# server then logged the statement text raw, with every other credential in it in cleartext. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +CANARY="c05153natspassword" + +# 1. A value of the wrong type reports the setting's own error, not one from the masking. +for setting in nats_url rabbitmq_address after_processing_move_connection_string; do + $CLICKHOUSE_CLIENT -q "SET $setting = 4222" 2>&1 | grep -oE '\(UNKNOWN_SETTING\)|\(BAD_GET\)' | head -1 +done + +# 2. The credential next to the wrongly typed setting is still masked in what the server logs. +# Only the server's own log line is checked: the `(query: ...)` line next to it is the client +# echoing back what it sent, which is the text the user typed and is never masked. +$CLICKHOUSE_CLIENT --send_logs_level=error -q "SET nats_password = '$CANARY', nats_url = 4222" 2>&1 | + grep -oE "in query: [^)]*" | head -1 + +# 3. And in `system.query_log`, for the `SETTINGS` clause of a `CREATE TABLE` as well. +QUERY_ID="05153_$CLICKHOUSE_DATABASE" +$CLICKHOUSE_CLIENT --query_id="$QUERY_ID" --log_queries=1 -q \ + "CREATE TABLE ${CLICKHOUSE_DATABASE}.events (a String) ENGINE = MergeTree ORDER BY a + SETTINGS nats_password = '$CANARY', nats_url = 4222" > /dev/null 2>&1 + +for _ in {1..60}; do + $CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS query_log" + LOGGED=$($CLICKHOUSE_CLIENT -q "SELECT + position(query, '$CANARY') = 0, + position(query, 'nats_password = \'[HIDDEN]\'') > 0 + FROM system.query_log + WHERE current_database = currentDatabase() AND query_id = '$QUERY_ID' AND type = 'ExceptionBeforeStart'") + [ -n "$LOGGED" ] && break + sleep 0.5 +done +echo "$LOGGED" diff --git a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_create_table_as.reference b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_create_table_as.reference new file mode 100644 index 0000000000000..573541ac9702d --- /dev/null +++ b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_create_table_as.reference @@ -0,0 +1 @@ +0 diff --git a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_create_table_as.sql b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_create_table_as.sql new file mode 100644 index 0000000000000..09357fa59d889 --- /dev/null +++ b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_create_table_as.sql @@ -0,0 +1,5 @@ +-- A persisted table over `mergeTreeTextIndex` would resolve the source table without the reader's grants. +CREATE TABLE tab (s String, INDEX idx_s s TYPE text(tokenizer = 'splitByNonAlpha')) ENGINE = MergeTree ORDER BY tuple(); +CREATE TABLE tab_index AS mergeTreeTextIndex(currentDatabase(), tab, idx_s); -- { serverError BAD_ARGUMENTS } +CREATE TABLE tab_index (part_name String, token String) AS mergeTreeTextIndex(currentDatabase(), tab, idx_s); -- { serverError BAD_ARGUMENTS } +SELECT count() FROM system.tables WHERE database = currentDatabase() AND name = 'tab_index'; diff --git a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_interserver.reference b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_interserver.reference new file mode 100644 index 0000000000000..8734b92528426 --- /dev/null +++ b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_interserver.reference @@ -0,0 +1,2 @@ +ACCESS_DENIED +['hidden','token'] diff --git a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_interserver.sh b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_interserver.sh new file mode 100755 index 0000000000000..e6052c5ed95c7 --- /dev/null +++ b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_interserver.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# The interserver secret needs the SSL library, which the fast test build does not have. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Through an interserver connection the shard authenticates the initiating user, so the checks of +# `mergeTreeTextIndex` apply to that user instead of refusing the query as over an ordinary connection. + +user_name="${CLICKHOUSE_DATABASE}_user_05153" + +$CLICKHOUSE_CLIENT -q " +DROP USER IF EXISTS $user_name; + +CREATE TABLE tab (s String, INDEX idx_s s TYPE text(tokenizer = 'splitByNonAlpha')) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO tab VALUES ('hidden token'); + +CREATE USER $user_name IDENTIFIED WITH plaintext_password BY 'password'; +GRANT REMOTE ON *.* TO $user_name; +GRANT CREATE TEMPORARY TABLE ON *.* TO $user_name; +" + +function run_as_user() +{ + local output + if output=$($CLICKHOUSE_CLIENT --user "$user_name" --password "password" -q "$1" 2>&1); then + echo "$output" + else + echo "$output" | grep -oE '\([A-Z_]+\)' | tail -1 | tr -d '()' + fi +} + +query="SELECT arraySort(groupUniqArray(token)) FROM cluster('test_cluster_interserver_secret', mergeTreeTextIndex('$CLICKHOUSE_DATABASE', 'tab', 'idx_s')) SETTINGS prefer_localhost_replica = 0" + +run_as_user "$query" + +$CLICKHOUSE_CLIENT -q "GRANT SELECT ON $CLICKHOUSE_DATABASE.tab TO $user_name" + +run_as_user "$query" + +$CLICKHOUSE_CLIENT -q " +DROP TABLE tab; +DROP USER $user_name; +" diff --git a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_metadata_access.reference b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_metadata_access.reference new file mode 100644 index 0000000000000..361650ff1a48e --- /dev/null +++ b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_metadata_access.reference @@ -0,0 +1,10 @@ +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +BAD_ARGUMENTS +BAD_ARGUMENTS +NOT_IMPLEMENTED diff --git a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_metadata_access.sh b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_metadata_access.sh new file mode 100755 index 0000000000000..f49d9906b7bad --- /dev/null +++ b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_metadata_access.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Without any grant on the table, `mergeTreeTextIndex` must not reveal whether the table +# is a MergeTree table or which indexes it has: every error must be ACCESS_DENIED. + +user_name="${CLICKHOUSE_DATABASE}_user_05153" + +$CLICKHOUSE_CLIENT -q " +DROP USER IF EXISTS $user_name; + +CREATE TABLE tab +( + a String, + b String, + INDEX idx_text a TYPE text(tokenizer = 'splitByNonAlpha'), + INDEX idx_set b TYPE set(0) +) +ENGINE = MergeTree +ORDER BY tuple(); + +CREATE TABLE tab_memory (a String) ENGINE = Memory; + +INSERT INTO tab VALUES ('hello', 'world'); + +CREATE USER $user_name IDENTIFIED WITH plaintext_password BY 'password'; +" + +function run_as_user() +{ + local output + if output=$($CLICKHOUSE_CLIENT --user "$user_name" --password "password" -q "$1" 2>&1); then + echo "OK" + else + echo "$output" | grep -oE '\([A-Z_]+\)' | tail -1 | tr -d '()' + fi +} + +run_as_user "SELECT * FROM mergeTreeTextIndex(currentDatabase(), tab, idx_text)" +run_as_user "SELECT * FROM mergeTreeTextIndex(currentDatabase(), tab, idx_set)" +run_as_user "SELECT * FROM mergeTreeTextIndex(currentDatabase(), tab, idx_missing)" +run_as_user "SELECT * FROM mergeTreeTextIndex(currentDatabase(), tab_memory, idx_text)" +run_as_user "SELECT * FROM mergeTreeTextIndex(currentDatabase(), tab_missing, idx_text)" +run_as_user "INSERT INTO FUNCTION mergeTreeTextIndex(currentDatabase(), tab, idx_text) SELECT 'p', 'x', 'raw', 1, 1, 0, 0, 0" + +# A grant on any column implies SHOW TABLES, so index metadata becomes visible, +# while reading the index still requires SELECT on the indexed column. +$CLICKHOUSE_CLIENT -q "GRANT SELECT(b) ON $CLICKHOUSE_DATABASE.tab TO $user_name" + +run_as_user "SELECT * FROM mergeTreeTextIndex(currentDatabase(), tab, idx_text)" +run_as_user "SELECT * FROM mergeTreeTextIndex(currentDatabase(), tab, idx_set)" +run_as_user "SELECT * FROM mergeTreeTextIndex(currentDatabase(), tab, idx_missing)" + +# The index is read-only: the access check runs before the write is rejected. +$CLICKHOUSE_CLIENT -q "INSERT INTO FUNCTION mergeTreeTextIndex(currentDatabase(), tab, idx_text) SELECT 'p', 'x', 'raw', 1, 1, 0, 0, 0" 2>&1 | grep -oE '\([A-Z_]+\)' | tail -1 | tr -d '()' + +$CLICKHOUSE_CLIENT -q " +DROP TABLE tab; +DROP TABLE tab_memory; +DROP USER $user_name; +" diff --git a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_remote.reference b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_remote.reference new file mode 100644 index 0000000000000..b36ea9eb0f17d --- /dev/null +++ b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_remote.reference @@ -0,0 +1,9 @@ +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +2 +ACCESS_DENIED +2 diff --git a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_remote.sh b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_remote.sh new file mode 100755 index 0000000000000..5f949a6dc4304 --- /dev/null +++ b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_remote.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `remote` over a local shard validates the access of the current user while inferring the structure of a nested +# table function, and may then route the query over loopback with other credentials. `mergeTreeTextIndex` has a +# static structure, so it has to check the access of the user itself there. + +user_name="${CLICKHOUSE_DATABASE}_user_05153" + +$CLICKHOUSE_CLIENT -q " +DROP USER IF EXISTS $user_name; + +CREATE TABLE tab (s String, INDEX idx_s s TYPE text(tokenizer = 'splitByNonAlpha')) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO tab VALUES ('hidden token'); + +CREATE USER $user_name IDENTIFIED WITH plaintext_password BY 'password'; +GRANT REMOTE ON *.* TO $user_name; +GRANT CREATE TEMPORARY TABLE ON *.* TO $user_name; +" + +function run_as_user() +{ + local output + if output=$($CLICKHOUSE_CLIENT --user "$user_name" --password "password" -q "$1" 2>&1); then + echo "$output" + else + echo "$output" | grep -oE '\([A-Z_]+\)' | tail -1 | tr -d '()' + fi +} + +index="mergeTreeTextIndex('$CLICKHOUSE_DATABASE', 'tab', 'idx_s')" +query="SELECT count() FROM remote('127.0.0.1:$CLICKHOUSE_PORT_TCP', $index)" +same_user_query="SELECT count() FROM remote('127.0.0.1:$CLICKHOUSE_PORT_TCP', $index, '$user_name', 'password')" + +function run_remote_as_user() +{ + for localhost_replica in 0 1; do + run_as_user "$query SETTINGS prefer_localhost_replica = $localhost_replica" + done + for localhost_replica in 0 1; do + run_as_user "$same_user_query SETTINGS prefer_localhost_replica = $localhost_replica" + done +} + +run_as_user "DESCRIBE TABLE $index" +run_remote_as_user + +$CLICKHOUSE_CLIENT -q "GRANT SELECT ON $CLICKHOUSE_DATABASE.tab TO $user_name" + +# Over an ordinary connection the shard runs the query as the user of the connection, which the function refuses; +# with the local shortcut it runs as the user itself. +run_remote_as_user + +$CLICKHOUSE_CLIENT -q " +DROP TABLE tab; +DROP USER $user_name; +" diff --git a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_row_policy_any_column.reference b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_row_policy_any_column.reference new file mode 100644 index 0000000000000..2efecfaa49217 --- /dev/null +++ b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_row_policy_any_column.reference @@ -0,0 +1,4 @@ +['visible apple phrase'] +0 +ACCESS_DENIED +['apple','hidden','phrase','visible','zebra'] diff --git a/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_row_policy_any_column.sh b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_row_policy_any_column.sh new file mode 100755 index 0000000000000..d5a3aa5698697 --- /dev/null +++ b/tests/queries/0_stateless/05153_text_index_mergeTreeTextIndex_row_policy_any_column.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A row policy on a column that is not indexed must still deny reading the text index: +# the dictionary contains the tokens of the rows the policy hides. + +user_name="${CLICKHOUSE_DATABASE}_user_05153" + +$CLICKHOUSE_CLIENT -q " +DROP USER IF EXISTS $user_name; + +CREATE TABLE tab +( + tenant_id String, + doc_id UInt64, + secret_text String, + INDEX idx_secret secret_text TYPE text(tokenizer = 'splitByNonAlpha') GRANULARITY 1 +) +ENGINE = MergeTree +ORDER BY doc_id +SETTINGS index_granularity = 1; + +INSERT INTO tab VALUES ('tenant_a', 1, 'visible apple phrase'), ('tenant_b', 2, 'hidden zebra phrase'); + +CREATE USER $user_name IDENTIFIED WITH plaintext_password BY 'password'; +GRANT SELECT ON $CLICKHOUSE_DATABASE.tab TO $user_name; +CREATE ROW POLICY p_05153 ON $CLICKHOUSE_DATABASE.tab FOR SELECT USING tenant_id = 'tenant_a' TO $user_name; +" + +function run_as_user() +{ + local output + if output=$($CLICKHOUSE_CLIENT --user "$user_name" --password "password" -q "$1" 2>&1); then + echo "$output" + elif echo "$output" | grep -q "ACCESS_DENIED"; then + echo "ACCESS_DENIED" + else + echo "$output" + fi +} + +run_as_user "SELECT groupArray(secret_text) FROM tab" +run_as_user "SELECT count() FROM tab WHERE hasToken(secret_text, 'zebra')" +run_as_user "SELECT arraySort(groupArray(token)) FROM mergeTreeTextIndex(currentDatabase(), tab, idx_secret)" + +$CLICKHOUSE_CLIENT -q "DROP ROW POLICY p_05153 ON $CLICKHOUSE_DATABASE.tab" + +run_as_user "SELECT arraySort(groupArray(token)) FROM mergeTreeTextIndex(currentDatabase(), tab, idx_secret)" + +$CLICKHOUSE_CLIENT -q " +DROP TABLE tab; +DROP USER $user_name; +" diff --git a/tests/queries/0_stateless/05153_variant_truncated_compact_discriminators.reference b/tests/queries/0_stateless/05153_variant_truncated_compact_discriminators.reference new file mode 100644 index 0000000000000..91425ed846bbb --- /dev/null +++ b/tests/queries/0_stateless/05153_variant_truncated_compact_discriminators.reference @@ -0,0 +1,30 @@ +42 +\N +\N +\N +42 +\N +0 +\N +2 +\N +4 +\N +6 +\N +0 +\N +2 +\N +4 +\N +6 +\N +0 +1 +0 +1 +0 +1 +0 +1 diff --git a/tests/queries/0_stateless/05153_variant_truncated_compact_discriminators.sql b/tests/queries/0_stateless/05153_variant_truncated_compact_discriminators.sql new file mode 100644 index 0000000000000..a42982dfe5797 --- /dev/null +++ b/tests/queries/0_stateless/05153_variant_truncated_compact_discriminators.sql @@ -0,0 +1,18 @@ +-- Truncated `COMPACT` `Variant` discriminator streams must be rejected, not read past the buffer. +SELECT * FROM format(Native, 'v Variant(UInt8)', concat('\x01\x64\x01v\x0eVariant(UInt8)', unhex('01000000000000006400'))); -- { serverError CANNOT_READ_ALL_DATA } +SELECT * FROM format(Native, 'v Variant(UInt8)', concat('\x01\x02\x01v\x0eVariant(UInt8)', unhex('01000000000000000200ff'))); -- { serverError CANNOT_READ_ALL_DATA } +SELECT * FROM format(Native, 'v Variant(UInt8)', concat('\x01\x6c\x01v\x0eVariant(UInt8)', unhex('01000000000000000801ff6400ff'))); -- { serverError CANNOT_READ_ALL_DATA } + +-- Complete streams still decode correctly. +SELECT * FROM format(Native, 'v Variant(UInt8)', concat('\x01\x02\x01v\x0eVariant(UInt8)', unhex('0100000000000000020000ff2a'))); +SELECT * FROM format(Native, 'v Variant(UInt8)', concat('\x01\x04\x01v\x0eVariant(UInt8)', unhex('01000000000000000201ff020000ff2a'))); + +CREATE TABLE variant_compact_granules (id UInt8, v Variant(UInt8, String)) +ENGINE = MergeTree ORDER BY id +SETTINGS use_compact_variant_discriminators_serialization = 1, index_granularity = 8, + min_rows_for_wide_part = 0, min_bytes_for_wide_part = 0; +INSERT INTO variant_compact_granules SELECT number, if(number % 2 = 0, number::UInt8, NULL) FROM numbers(8); +SELECT v FROM variant_compact_granules ORDER BY id SETTINGS max_block_size = 1, max_threads = 1; +SELECT v.UInt8 FROM variant_compact_granules ORDER BY id SETTINGS max_block_size = 1, max_threads = 1; +SELECT v.UInt8.null FROM variant_compact_granules ORDER BY id SETTINGS max_block_size = 1, max_threads = 1; +DROP TABLE variant_compact_granules; diff --git a/tests/queries/0_stateless/05154_json_shared_data_legacy_bucketed_map.reference b/tests/queries/0_stateless/05154_json_shared_data_legacy_bucketed_map.reference new file mode 100644 index 0000000000000..c0412f2f07f69 --- /dev/null +++ b/tests/queries/0_stateless/05154_json_shared_data_legacy_bucketed_map.reference @@ -0,0 +1,9 @@ +legacy {'k':1} {'k':(2)} [{'k':3},{'k':4}] {"m":{"k":5}} +types Map(String, UInt64) Map(String, Tuple(a UInt64)) Array(Map(String, UInt64)) JSON(m Map(String, UInt64)) +subcolumns 1 2 +rewritten {'k':1} {'k':(2)} [{'k':3},{'k':4}] {"m":{"k":5}} +rewritten {'k':1} {'k':(2)} [{'k':3},{'k':4}] {"m":{"k":5}} +rewritten_types Map(String, UInt64) Map(String, Tuple(a UInt64)) Array(Map(String, UInt64)) JSON(m Map(String, UInt64)) +rewritten_types Map(String, UInt64) Map(String, Tuple(a UInt64)) Array(Map(String, UInt64)) JSON(m Map(String, UInt64)) +rewritten_subcolumns 1 2 +rewritten_subcolumns 1 2 diff --git a/tests/queries/0_stateless/05154_json_shared_data_legacy_bucketed_map.sh b/tests/queries/0_stateless/05154_json_shared_data_legacy_bucketed_map.sh new file mode 100755 index 0000000000000..250255f61c6c9 --- /dev/null +++ b/tests/queries/0_stateless/05154_json_shared_data_legacy_bucketed_map.sh @@ -0,0 +1,84 @@ +#!/usr/bin/env bash + +set -euo pipefail + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +workdir=$(mktemp -d "${CLICKHOUSE_TMP}/legacy_bucketed_map.XXXXXX") +trap 'rm -rf "$workdir"' EXIT + +part_dir=$($CLICKHOUSE_LOCAL --path "$workdir" --multiquery --query " +CREATE DATABASE legacy ENGINE = Atomic; +CREATE TABLE legacy.maps (j JSON(max_dynamic_paths = 0)) ENGINE = MergeTree ORDER BY tuple() +SETTINGS min_bytes_for_wide_part = 0, object_shared_data_buckets_for_wide_part = 1, + object_shared_data_serialization_version = 'advanced', object_shared_data_serialization_version_for_zero_level_parts = 'advanced', + map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets', + propagate_types_serialization_versions_to_nested_types = 1; +SELECT concat(data_paths[1], 'detached') FROM system.tables WHERE database = 'legacy' AND name = 'maps'; +") + +# A real part written by `91ffe225686`, before the `Map` statistics flag fix. +# Its `JSON` paths contain a scalar-valued map, a tuple-valued map, an array of maps, +# and nested `JSON` with a typed map. The original checksums are preserved. +base64 -d <<'PART' | tar -xz -C "$part_dir" +H4sIAAAAAAACA+2dC1xTZR/H/88uCAPGuKggyBnjfnGcjbHVwAuQiKAvKOYFL/M4DjjG2NzFFF8TZ1YWKb4mFZWa19Twfs8LlWaW +mn3MlKQsw8osyxKxlHyfcREbL5Kk6/PG+fP5c7bnPDuDnfN9fv/zPP/nGVVYqJLgHzJWPZlWa00WnUlsnmaG+2kkNrlM1rjFZr+V +SiTxLY+byiVS/ANCEhxgFpOZMuK3h65pt8+6ME9v1FFm4VTaaNLoi5RCGa9GGlHzTsw3PK2HRvpm6VpeWRAHINEJIGJ7bMPERK5F +CLKHe7qo9ZYis+2iQYWvPLrvYsWif6/NHjH6qg/3FhAFYv2kAlptVpkmU0Y6V5VLmSkxKZ6kKdqM6q/nT/vyswn1J9YcSytccXQo +qmYVlK6Nrtnks/jy+uf2Hxq4MDlIrTNqpZKFCeveebVhe2JS9NaNGaPy9IhcYnqqoUqjOtl74OHqd4feXBFmoMyTTSodZdSabEeP +5Y8a+fCVt3Oszz1Xs9DtVGUKGrDph4u3prI2JGbIwumpaet1EbH7L9WlFtef4wVL5jWUplecR2Q4O+XXiidQ6pYX+nHUJxPGy0yW +SSazkaZ0+Ni0meoPu9/6rvTY5+8EdU8jx1jmRKMStE5mrdW+lH/inNPHwaeq30by+dcuxyU2nC8r2MT6IfOx1z9B5Gfe39euNrjt +PNTfOVlYNWhdCD6mRW22GGnxLv2xwWs+OxW8e/uR+m3y0hikfHKxC582+FVQO5c3HK6dfDxs3LqjledWpRSPSTdHxZhX7EbkmuqD +xMhvT7zoOq4+su+aeetD69lffJm+tWfygB9Ljg96a8uyq0+jw/zLyHWPmjv24KrxIp/gyafdw6VB/cOXjebnJWzPKrpILeKOQuTg +fUPklQt+HZRbYTj1oSafE70P7SutOzL0lbjggzOnhJZRMfNRlZPp4rZr+r2/sG5d20bNGu5l6CN96lzx3FTltZGhR7+oEzUs6o/I +0d5HNvxkGLUy3fW9L1eGv3QjWq03TNcU5dLT6MDdt0JHu/SbM2H4FHddlalShTi1vsTwOYMGX3z/2WLvHFVlTOz7F3aeRiO2na6Y +5bv924ANpYhc+tGwZd25I/fWLqwRu1xY9l6wSVMsWh22hS94yn/t4z3XnzlFTV2CnB+pk29OfDOv/KvYHhz2hrAjoT8W0nM5Jfuz +066oImmoy0bknJXcA+IXT6Dw6yn9vjq57EbIVKrQkjvj95k3V30SMS6px2+e/kPY/VBO+tmNF57amPr4LzsneM0LoN8JW7t1EBF7 +xJrWMOXlgzt68MsRmX3deu2DC5l9e6uEmYqjFR498LVGuqQcTRqx+jU9v+bH1Kpbe/6D/CYK8tMjjYIVV13o4Qecglf4Sqd5sA6s +OfjuFf+80yvm9Yh0QeRzr08K+eCL1B17Jpiq4ycVcr1MtFFDFWqKKTPmT1xg0heVIn7Np0OvVl3iZb+gnpWdPPN98MTNlVljMmvU +JrGBUmvp3JnCa+WXE2azo+ZXBhz8RfbYSSCFZSKXg5+wdwtOqKoozazjvDMN3d1HH4ezgzyDz/7+hK9PtwHXdG877dSjEJczEb9Z +vy8vzXvi93OvocWuN+ddmf4E6r3Uj31MMbH3pT5k2sbZXFXkk4fcNxWnb80/vdOTzXd/5v2Zhn9HH3h+9kR3cenKXl9bXCaNfkG8 +txsw9v9mVKv+6wstuqL7r/4d6j8pIePs9F8iJxWM/jtE/5vOehv1l/AkwuZ9St7EgonC9OzMf0XoqGmq3OlFlE6jVjUqbl8yksdA +9E/iX9Ua8tyvpqAj/uXxpB3/MlIuZ/h3JP+tZ72dNkBC3lkHNxfNu4S4cVDyXFoD/ZbQ9s6yPwT/HdewbdrdeUek3/5b3P5L/0SV +Do7V9H52NwJ3+Qttca8Yx6q06W4Vmo7aFB/ftaItVsU1HMF/823cfX+PjvjHEYC9/kvjZQz/jjAJI4GM/jfyn0vnUZZCs0qt1xmM +tMkmAfhxLq3+y21Ch/xLpHb8x8fFkQz/jrCUzEcGpkQMyZFFMix0bf7vEovZelQfIP9Sub3+x0viGf4dYnU//7rwg17Jv+V88/YG +Z/7uN6yv4cIxCEDEAQSTUGB4dw5ygxAAFxiAELAE4OwNiA98AZBaLbsnyPALsoHjDATKAQ6XBiS3lWk9gfsw7CChxFmIkA4fZgpw +AiEcWCnQzRUgA4bhF2q53cDpIcCffi0XurHAQwDunAKYhcK741dRBGJxttmOXaZlMaz+ffw3DsM8OP1vy79MwsT/DrEDLxd3E+5a +Mfdqgzr55bOGp8uCbCcMmsb4hORM/ABwQzCGDbIElBvDjmGQ6Vr82w2sPhD+4xV2/Mvj45n+P4cY73vOl19r3IddMC/32FNzY5WV +wIUDsBOYeo7/WF/wf14AJQGIQYXhv5NhwL3zryCljP47xIZ51c9S7npxcPaOmxR33YWsMuIP+q9rfIJwi8AewQJXphXoovz/j+GP +Px8NdMi/wo5/qUSmiGP4d4Sdqv6uUP3KvKHjL8/3/Djb72NrLC4sadZ/n6gkFAm9NnmAEvqyPME/i4SoXk5CKFnG3I53ef7/dDRw +z/ovlSjimPwfh1gw/WrKXD79rXGO7nPjykR2O/pfwuh/F+W/NVW504MAHfMvt7//jyOZ/H+H2OTPzeneIqeowOvl9XUlZ16wBuJC +JfYDiIMopEN6ZMb898xiUOnq/Hd6EKBT/X/M+J9DbMc2S0jK6vMj4XzGkszKNcnt6L+S0f+uyn9rOnBnA4CO+FfI2/Avk0kZ/h1h +xkM3i1csP5Q/XPkWh+hZ0d/6DWZ8UDeA3wa6eolbcv1vi0B38VTKqKGKzKpcjUltVBmMdJ5mWoQ4yWikpkcMpQwR2Wajpig/Rvjo +4CKzXBYZKZ5kUWtpsy3XNU/P7w9E0EPgorflx5LBInDxJEMgVhIZArwpodKBWnq6KRqIkCjwJMJCwf1naEp/9RZnN16UIwUITYRe +fB8geEmoR2yQOAGBhxK5EAESYLE8USwp6g1sZx4iegcDjyACwcnjJvQguuNfS/uImyYxCNkInoxsvtxns4hoMa6qk+GjhCWAG7sf +ImLCwd05DRHhfcCHiIwAAWscIpKjgMfxQGgGa24LHEvY7AVsQp4CBKQD0VcBxId39Jcn9gNi6W1+6kDE7QusEiAysLP2g9OTkNV4 +N81DHgsgiCPiLsD32Xn481QugN5JtuzfgRwPd0SgBGAdgwXNycIJj4OINZvlUc1R+kWlc2aNsBgK6QgqwMmLCFWCK7uSTUSHgZvz +G2wiLAa8bf+LBysSZeKPl+qDXLMaT1+n+O9sANDx+L99/p9crmD6/x1iVoP/Vs/NG1W/V2uTZebNu8pEf9D/uQAP2fQftwhsARwu +21zKZTSzi+p/p5MAOsz/U9jP/1fI45j7f4fY6C+una5LLR9SFRqgXPW9ppu1EhcOYLeM/7v7gn93AfgHYg/CHon9YeyDsWcIgBgr +gAD/fBL89bhgGvYZ2OdgfxZ7Ka7wvK3COlxhCy7Yh/1d7Eewf4S9Gvun2M/jirUCGBLlgyggAgSQ6B+TBf4yvKsv9gHY07BnYR8m +gMAxAmC6I/42/u89Crhn/ZfiewKGf4fYpde/nm/ps8u4ZvFiQn7959p29B+3CIz+d0H+285VfAD6T8bZ9/8rFBKm/88h9tKM2Rmv +8/38d32ay/Ne9F2dlY8LObYYABCLzfDB8G/P/wPQ/zbj/6REzui/QyxqWEjJI9ffvGB6duYJ3ZGk3Hb6/zlM/3/X5b9xMYO/MAew +Q/5lbfr/pCTT/+cQ26eY8uxDN5555b3JWz/wTq32s3rhQmfsJRwGDob/O/jv/BzAToz/y+RM/O8Q+zAzZc1UN/HhUeQU+TwqJrYd +/Xdm9L/r8t80Bvvg8v8krftu5//FM/G/Qwyd+sFwK+BmTv76uOQfx1zyto7AhTnYt/jbpv6zENKyMf/JeMvhwSLg29YDQFrkAVX+ +d8ztj4BsHS7mBoDBq2nqfiyUMHME/lH8OzL/Ty5l8n8cYnuPT7x16mrGo5+9B6J0oe/GdvQ/h9H/rsP/X8/4v0f+5fbrf8YpSCb+ +d4iVHH80nDtjufbkL2NufHd5A88qxIV+2INt6//8hHAAwEF6LPIImbGqMw1AV+L/ry37cw/82/f/yZj1vxxkVpEwo/zGlZyf3Hzf +9Y38obzt+P8AAFfcFPgBJ4sFWw6mMMz8M/lvmderal7/+f4tBXzv6//GyZj7f8cYySDA8N/If9uvn7lv11hH+b8S+/6/OBwBMPw7 +wmaImlf4FynHjo8RGYx6A5VPmWmVebqBNqn+cFG0KINJZdarimiTmc5tqiZS4pCRjhHd7TUi5QyRjjKIlJIYkalxjhB+ODNG1Lzf +9oSh8W/l3/6LpcBR/Eva5P9LFcz8PwdZyx29V+v5VxWIbU9MLWt9Dmve+rEeL6nuNSZogrKwr0Xv52NNwoUDsSdzQIAP9LObX7vf +EoSCIBg8nYEjgDIms4Axxhhj7O+3/wKOKbQwAHgAAA== +PART + +$CLICKHOUSE_LOCAL --path "$workdir" --multiquery <<'SQL' +ALTER TABLE legacy.maps ATTACH PART 'all_1_1_0'; +SELECT 'legacy', j.m, j.t, j.a, j.o FROM legacy.maps; +SELECT 'types', dynamicType(j.m), dynamicType(j.t), dynamicType(j.a), dynamicType(j.o) FROM legacy.maps; +SELECT 'subcolumns', j.m.:`Map(String, UInt64)`['k'], j.t.:`Map(String, Tuple(a UInt64))`['k'].a FROM legacy.maps; +INSERT INTO legacy.maps SELECT j FROM legacy.maps; +OPTIMIZE TABLE legacy.maps FINAL; +SELECT 'rewritten', j.m, j.t, j.a, j.o FROM legacy.maps; +SELECT 'rewritten_types', dynamicType(j.m), dynamicType(j.t), dynamicType(j.a), dynamicType(j.o) FROM legacy.maps; +SELECT 'rewritten_subcolumns', j.m.:`Map(String, UInt64)`['k'], j.t.:`Map(String, Tuple(a UInt64))`['k'].a FROM legacy.maps; +SQL diff --git a/tests/queries/0_stateless/05175_trim_empty_chars_is_a_function_call.reference b/tests/queries/0_stateless/05175_trim_empty_chars_is_a_function_call.reference new file mode 100644 index 0000000000000..8ea3320035082 --- /dev/null +++ b/tests/queries/0_stateless/05175_trim_empty_chars_is_a_function_call.reference @@ -0,0 +1,68 @@ +-- { echo } +-- trim(BOTH|LEADING|TRAILING '' FROM x) parses to a function call, like every other trim spelling. +SET allow_statistics = 1; +SELECT count() FROM (EXPLAIN AST SELECT trim(BOTH '' FROM 'x')) WHERE explain ILIKE '%Function trimBoth%'; +1 +SELECT count() FROM (EXPLAIN AST SELECT trim(LEADING '' FROM 'x')) WHERE explain ILIKE '%Function trimLeft%'; +1 +SELECT count() FROM (EXPLAIN AST SELECT trim(TRAILING '' FROM 'x')) WHERE explain ILIKE '%Function trimRight%'; +1 +SELECT formatQuery($$SELECT trim(BOTH '' FROM 'x')$$); +SELECT trimBoth(\'x\', \'\') +SELECT formatQuery(formatQuery($$SELECT trim(BOTH '' FROM 'x')$$)) = formatQuery($$SELECT trim(BOTH '' FROM 'x')$$); +1 +-- Slots that require a function report their own error instead of crashing. +CREATE TABLE t_stat (c Int64 STATISTICS(trim(BOTH '' FROM 'x'))) ENGINE = MergeTree ORDER BY c; -- { serverError INCORRECT_QUERY } +CREATE TABLE t_codec (c Int64 CODEC(trim(BOTH '' FROM 'x'))) ENGINE = MergeTree ORDER BY c; -- { serverError UNKNOWN_CODEC } +CREATE TABLE t_index (c Int64, INDEX i c TYPE trim(BOTH '' FROM 'x')) ENGINE = MergeTree ORDER BY c; -- { serverError INCORRECT_QUERY } +CREATE TABLE t_auto (c Int64) ENGINE = MergeTree ORDER BY c SETTINGS auto_statistics_types = 'trim(BOTH '''' FROM ''x'')'; -- { serverError INCORRECT_QUERY } +SELECT * FROM trim(BOTH '' FROM 'x'); -- { serverError UNKNOWN_FUNCTION } +CREATE TABLE t_alter (c Int64) ENGINE = MergeTree ORDER BY c; +ALTER TABLE t_alter ADD STATISTICS c TYPE trim(BOTH '' FROM 'x'); -- { serverError INCORRECT_QUERY } +DROP TABLE t_alter; +-- The operand reaches the function's argument checks. +SELECT trim(BOTH '' FROM 123); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT toTypeName(trim(BOTH '' FROM CAST('x' AS FixedString(5)))); +String +SELECT toTypeName(trim(BOTH '' FROM toNullable('x'))), toTypeName(trim(BOTH '' FROM toLowCardinality('x'))), toTypeName(trim(BOTH '' FROM CAST('x' AS LowCardinality(Nullable(String))))); +Nullable(String) LowCardinality(String) LowCardinality(Nullable(String)) +-- String results are unchanged, for every keyword and every spelling. +SELECT trim(BOTH '' FROM ' x '), trim(LEADING '' FROM ' x '), trim(TRAILING '' FROM ' x '); + x x x +SELECT trim(BOTH 'a' FROM 'aXa'), trim(LEADING 'a' FROM 'aXa'), trim(TRAILING 'a' FROM 'aXa'); +X Xa aX +SELECT trim(' x '), ltrim(' x '), rtrim(' x '), trimBoth(' x ', ''), trimLeft(' x ', ''), trimRight(' x ', ''); +x x x x x x +SELECT trim(BOTH '' FROM materialize(' x ')), trim(BOTH '' FROM toNullable(' x ')), trim(BOTH '' FROM toLowCardinality(' x ')); + x x x +SELECT trim(BOTH '' FROM ''), length(trim(BOTH '' FROM ' ')), trim(BOTH '' FROM NULL); + 2 \N +SELECT count() FROM numbers(1000) WHERE trim(BOTH '' FROM toString(number)) != toString(number); +0 +-- The cases of #67792 and #69922, which is why an empty trim character is accepted at all. +SELECT trim(LEADING '' FROM 'foo'), trim(TRAILING '' FROM 'foo'), trim(BOTH '' FROM 'foo'); +foo foo foo +SELECT trim(LEADING concat('') FROM 'foo'), trim(BOTH concat('') FROM ' foo '), trimLeft('foo', concat('')); +foo foo foo +-- ParserStringLiteral spells an empty trim character five ways: quoted, hex, binary, heredoc and fancy quotes. +SELECT formatQuery($$SELECT trim(BOTH x'' FROM 'x')$$), formatQuery($$SELECT trim(LEADING b'' FROM 'x')$$), formatQuery($$SELECT trim(TRAILING ‘’ FROM 'x')$$), formatQuery($q$SELECT trim(BOTH $$$$ FROM 'x')$q$); +SELECT trimBoth(\'x\', \'\') SELECT trimLeft(\'x\', \'\') SELECT trimRight(\'x\', \'\') SELECT trimBoth(\'x\', \'\') +CREATE TABLE t_stat_hex (c Int64 STATISTICS(trim(BOTH x'' FROM 'x'))) ENGINE = MergeTree ORDER BY c; -- { serverError INCORRECT_QUERY } +SELECT trim(BOTH x'' FROM 123); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT toTypeName(trim(BOTH b'' FROM CAST('ä' AS FixedString(4)))), hex(trim(BOTH b'' FROM CAST('ä' AS FixedString(4)))); +String C3A40000 +-- Unicode and non-UTF-8 bytes reach the function untouched, and a non-empty character was never folded. +SELECT trim(BOTH x'' FROM 'äöü'), trim(BOTH ‘ä’ FROM 'äxä'), hex(trim(BOTH x'' FROM x'FF41FF')), hex(trim(BOTH x'FF' FROM x'FF41FF')), formatQuery($$SELECT trim(BOTH x'20' FROM ' x ')$$); +äöü x FF41FF 41 SELECT trimBoth(\' x \', \' \') +-- Slots that accept a function value keep it: the value is evaluated to the same constant. +CREATE DICTIONARY d (k UInt64, v String) PRIMARY KEY k SOURCE(CLICKHOUSE(HOST trim(BOTH '' FROM 'localhost') PORT 9000 TABLE 'x' DB 'y')) LAYOUT(FLAT()) LIFETIME(0); +SELECT position(create_table_query, 'SOURCE(CLICKHOUSE(HOST \'localhost\' PORT 9000 TABLE \'x\' DB \'y\'))') > 0 FROM system.tables WHERE database = currentDatabase() AND name = 'd'; +1 +DROP DICTIONARY d; +-- Ordinary spellings of the same slots still parse. +CREATE TABLE ok (c Int64 CODEC(ZSTD(3)) STATISTICS(tdigest), d Int64 CODEC(NONE) STATISTICS(uniq, minmax), INDEX i c TYPE minmax) ENGINE = MergeTree ORDER BY c SETTINGS auto_statistics_types = 'tdigest, uniq'; +SELECT position(create_table_query, '`c` Int64 CODEC(ZSTD(3)) STATISTICS(tdigest)') > 0, position(create_table_query, '`d` Int64 CODEC(NONE) STATISTICS(uniq, minmax)') > 0, position(create_table_query, 'INDEX i c TYPE minmax') > 0, position(create_table_query, 'auto_statistics_types = \'tdigest, uniq\'') > 0 FROM system.tables WHERE database = currentDatabase() AND name = 'ok'; +1 1 1 1 +DROP TABLE ok; +SELECT 1 SETTINGS max_threads = 4; +1 diff --git a/tests/queries/0_stateless/05175_trim_empty_chars_is_a_function_call.sql b/tests/queries/0_stateless/05175_trim_empty_chars_is_a_function_call.sql new file mode 100644 index 0000000000000..731d3f33a490f --- /dev/null +++ b/tests/queries/0_stateless/05175_trim_empty_chars_is_a_function_call.sql @@ -0,0 +1,49 @@ +-- { echo } +-- trim(BOTH|LEADING|TRAILING '' FROM x) parses to a function call, like every other trim spelling. +SET allow_statistics = 1; +SELECT count() FROM (EXPLAIN AST SELECT trim(BOTH '' FROM 'x')) WHERE explain ILIKE '%Function trimBoth%'; +SELECT count() FROM (EXPLAIN AST SELECT trim(LEADING '' FROM 'x')) WHERE explain ILIKE '%Function trimLeft%'; +SELECT count() FROM (EXPLAIN AST SELECT trim(TRAILING '' FROM 'x')) WHERE explain ILIKE '%Function trimRight%'; +SELECT formatQuery($$SELECT trim(BOTH '' FROM 'x')$$); +SELECT formatQuery(formatQuery($$SELECT trim(BOTH '' FROM 'x')$$)) = formatQuery($$SELECT trim(BOTH '' FROM 'x')$$); +-- Slots that require a function report their own error instead of crashing. +CREATE TABLE t_stat (c Int64 STATISTICS(trim(BOTH '' FROM 'x'))) ENGINE = MergeTree ORDER BY c; -- { serverError INCORRECT_QUERY } +CREATE TABLE t_codec (c Int64 CODEC(trim(BOTH '' FROM 'x'))) ENGINE = MergeTree ORDER BY c; -- { serverError UNKNOWN_CODEC } +CREATE TABLE t_index (c Int64, INDEX i c TYPE trim(BOTH '' FROM 'x')) ENGINE = MergeTree ORDER BY c; -- { serverError INCORRECT_QUERY } +CREATE TABLE t_auto (c Int64) ENGINE = MergeTree ORDER BY c SETTINGS auto_statistics_types = 'trim(BOTH '''' FROM ''x'')'; -- { serverError INCORRECT_QUERY } +SELECT * FROM trim(BOTH '' FROM 'x'); -- { serverError UNKNOWN_FUNCTION } +SELECT 1 SETTINGS max_threads = trim(BOTH '' FROM 'x'); -- { clientError SYNTAX_ERROR } +CREATE NAMED COLLECTION nc_05175 AS k = trim(BOTH '' FROM 'x'); -- { clientError SYNTAX_ERROR } +CREATE TABLE t_alter (c Int64) ENGINE = MergeTree ORDER BY c; +ALTER TABLE t_alter ADD STATISTICS c TYPE trim(BOTH '' FROM 'x'); -- { serverError INCORRECT_QUERY } +DROP TABLE t_alter; +-- The operand reaches the function's argument checks. +SELECT trim(BOTH '' FROM 123); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT toTypeName(trim(BOTH '' FROM CAST('x' AS FixedString(5)))); +SELECT toTypeName(trim(BOTH '' FROM toNullable('x'))), toTypeName(trim(BOTH '' FROM toLowCardinality('x'))), toTypeName(trim(BOTH '' FROM CAST('x' AS LowCardinality(Nullable(String))))); +-- String results are unchanged, for every keyword and every spelling. +SELECT trim(BOTH '' FROM ' x '), trim(LEADING '' FROM ' x '), trim(TRAILING '' FROM ' x '); +SELECT trim(BOTH 'a' FROM 'aXa'), trim(LEADING 'a' FROM 'aXa'), trim(TRAILING 'a' FROM 'aXa'); +SELECT trim(' x '), ltrim(' x '), rtrim(' x '), trimBoth(' x ', ''), trimLeft(' x ', ''), trimRight(' x ', ''); +SELECT trim(BOTH '' FROM materialize(' x ')), trim(BOTH '' FROM toNullable(' x ')), trim(BOTH '' FROM toLowCardinality(' x ')); +SELECT trim(BOTH '' FROM ''), length(trim(BOTH '' FROM ' ')), trim(BOTH '' FROM NULL); +SELECT count() FROM numbers(1000) WHERE trim(BOTH '' FROM toString(number)) != toString(number); +-- The cases of #67792 and #69922, which is why an empty trim character is accepted at all. +SELECT trim(LEADING '' FROM 'foo'), trim(TRAILING '' FROM 'foo'), trim(BOTH '' FROM 'foo'); +SELECT trim(LEADING concat('') FROM 'foo'), trim(BOTH concat('') FROM ' foo '), trimLeft('foo', concat('')); +-- ParserStringLiteral spells an empty trim character five ways: quoted, hex, binary, heredoc and fancy quotes. +SELECT formatQuery($$SELECT trim(BOTH x'' FROM 'x')$$), formatQuery($$SELECT trim(LEADING b'' FROM 'x')$$), formatQuery($$SELECT trim(TRAILING ‘’ FROM 'x')$$), formatQuery($q$SELECT trim(BOTH $$$$ FROM 'x')$q$); +CREATE TABLE t_stat_hex (c Int64 STATISTICS(trim(BOTH x'' FROM 'x'))) ENGINE = MergeTree ORDER BY c; -- { serverError INCORRECT_QUERY } +SELECT trim(BOTH x'' FROM 123); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } +SELECT toTypeName(trim(BOTH b'' FROM CAST('ä' AS FixedString(4)))), hex(trim(BOTH b'' FROM CAST('ä' AS FixedString(4)))); +-- Unicode and non-UTF-8 bytes reach the function untouched, and a non-empty character was never folded. +SELECT trim(BOTH x'' FROM 'äöü'), trim(BOTH ‘ä’ FROM 'äxä'), hex(trim(BOTH x'' FROM x'FF41FF')), hex(trim(BOTH x'FF' FROM x'FF41FF')), formatQuery($$SELECT trim(BOTH x'20' FROM ' x ')$$); +-- Slots that accept a function value keep it: the value is evaluated to the same constant. +CREATE DICTIONARY d (k UInt64, v String) PRIMARY KEY k SOURCE(CLICKHOUSE(HOST trim(BOTH '' FROM 'localhost') PORT 9000 TABLE 'x' DB 'y')) LAYOUT(FLAT()) LIFETIME(0); +SELECT position(create_table_query, 'SOURCE(CLICKHOUSE(HOST \'localhost\' PORT 9000 TABLE \'x\' DB \'y\'))') > 0 FROM system.tables WHERE database = currentDatabase() AND name = 'd'; +DROP DICTIONARY d; +-- Ordinary spellings of the same slots still parse. +CREATE TABLE ok (c Int64 CODEC(ZSTD(3)) STATISTICS(tdigest), d Int64 CODEC(NONE) STATISTICS(uniq, minmax), INDEX i c TYPE minmax) ENGINE = MergeTree ORDER BY c SETTINGS auto_statistics_types = 'tdigest, uniq'; +SELECT position(create_table_query, '`c` Int64 CODEC(ZSTD(3)) STATISTICS(tdigest)') > 0, position(create_table_query, '`d` Int64 CODEC(NONE) STATISTICS(uniq, minmax)') > 0, position(create_table_query, 'INDEX i c TYPE minmax') > 0, position(create_table_query, 'auto_statistics_types = \'tdigest, uniq\'') > 0 FROM system.tables WHERE database = currentDatabase() AND name = 'ok'; +DROP TABLE ok; +SELECT 1 SETTINGS max_threads = 4; diff --git a/tests/queries/0_stateless/05182_async_insert_log_masks_secrets.reference b/tests/queries/0_stateless/05182_async_insert_log_masks_secrets.reference new file mode 100644 index 0000000000000..f59cb48c5f505 --- /dev/null +++ b/tests/queries/0_stateless/05182_async_insert_log_masks_secrets.reference @@ -0,0 +1 @@ +1 0 diff --git a/tests/queries/0_stateless/05182_async_insert_log_masks_secrets.sh b/tests/queries/0_stateless/05182_async_insert_log_masks_secrets.sh new file mode 100755 index 0000000000000..2a4886dd37126 --- /dev/null +++ b/tests/queries/0_stateless/05182_async_insert_log_masks_secrets.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The insert goes over HTTP, so the `VALUES` data is parsed on the server (data kind `Parsed`). +# Port 1 refuses the connection at once, so the flush fails without any DNS lookup. The log element +# is written while parsing, so the failed flush does not hide a leak. +# The column name carries the database name: masking never touches it, so it identifies these rows. + +url="http://etl_user:SUPER_SECRET_PASSWORD@127.0.0.1:1/ingest" + +${CLICKHOUSE_CURL} -sS -X POST --data-binary \ + "INSERT INTO FUNCTION url('${url}', 'CSV', '${CLICKHOUSE_DATABASE} String') VALUES ('row1')" \ + "${CLICKHOUSE_URL}&async_insert=1&wait_for_async_insert=1&wait_for_async_insert_timeout=30" > /dev/null 2>&1 + +# The log element is written after the HTTP response is sent, so wait for it. +for _ in $(seq 1 60); do + ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS asynchronous_insert_log" + + count=$(${CLICKHOUSE_CLIENT} -q " + SELECT count() + FROM system.asynchronous_insert_log + WHERE position(query, '${CLICKHOUSE_DATABASE}') > 0") + + [ "$count" -gt 0 ] && break + sleep 0.5 +done + +${CLICKHOUSE_CLIENT} -q " + SELECT + countIf(position(query, '[HIDDEN]') > 0) > 0 AS masked, + countIf(position(query, 'SUPER_SECRET_PASSWORD') > 0) AS leaked + FROM system.asynchronous_insert_log + WHERE position(query, '${CLICKHOUSE_DATABASE}') > 0" diff --git a/tests/queries/0_stateless/05182_json_shared_data_bucketed_map_compact_no_substream_marks.reference b/tests/queries/0_stateless/05182_json_shared_data_bucketed_map_compact_no_substream_marks.reference new file mode 100644 index 0000000000000..f037c389e431f --- /dev/null +++ b/tests/queries/0_stateless/05182_json_shared_data_bucketed_map_compact_no_substream_marks.reference @@ -0,0 +1,10 @@ +Compact +{"m":{"k":1},"t":{"k":{"a":2}}} +Map(String, UInt64) {'k':1} Map(String, Tuple(a UInt64)) {'k':(2)} +1 2 +{"m":{"k":1},"t":{"k":{"a":2}}} +{"m":{"k":1},"t":{"k":{"a":2}}} +Map(String, UInt64) {'k':1} Map(String, Tuple(a UInt64)) {'k':(2)} +Map(String, UInt64) {'k':1} Map(String, Tuple(a UInt64)) {'k':(2)} +1 2 +1 2 diff --git a/tests/queries/0_stateless/05182_json_shared_data_bucketed_map_compact_no_substream_marks.sql b/tests/queries/0_stateless/05182_json_shared_data_bucketed_map_compact_no_substream_marks.sql new file mode 100644 index 0000000000000..ec084ba7434c1 --- /dev/null +++ b/tests/queries/0_stateless/05182_json_shared_data_bucketed_map_compact_no_substream_marks.sql @@ -0,0 +1,27 @@ +DROP TABLE IF EXISTS json_shared_bucketed_map_compact; +CREATE TABLE json_shared_bucketed_map_compact (j JSON(max_dynamic_paths = 0)) +ENGINE = MergeTree ORDER BY tuple() +SETTINGS min_bytes_for_wide_part = 1000000000, min_rows_for_wide_part = 1000000000, + write_marks_for_substreams_in_compact_parts = 0, + object_shared_data_serialization_version = 'advanced', + map_serialization_version = 'with_buckets', + object_shared_data_serialization_version_for_zero_level_parts = 'advanced', + map_serialization_version_for_zero_level_parts = 'with_buckets', + propagate_types_serialization_versions_to_nested_types = 1; + +-- Preserve runtime `Map` types in shared data instead of parsing them as `JSON` objects. +INSERT INTO json_shared_bucketed_map_compact +SELECT * FROM format(RowBinary, 'j JSON(max_dynamic_paths = 0)', concat( + unhex('02016d'), formatRowNoNewline('RowBinary', CAST(map('k', toUInt64(1)) AS Dynamic)), + unhex('0174'), formatRowNoNewline('RowBinary', CAST(map('k', CAST(tuple(toUInt64(2)) AS Tuple(a UInt64))) AS Dynamic)))) +SETTINGS input_format_binary_read_json_as_string = 0; +SELECT part_type FROM system.parts WHERE database = currentDatabase() AND table = 'json_shared_bucketed_map_compact' AND active; +SELECT j FROM json_shared_bucketed_map_compact; +SELECT dynamicType(j.m), j.m, dynamicType(j.t), j.t FROM json_shared_bucketed_map_compact; +SELECT j.m.:`Map(String, UInt64)`['k'], j.t.:`Map(String, Tuple(a UInt64))`['k'].a FROM json_shared_bucketed_map_compact; +INSERT INTO json_shared_bucketed_map_compact SELECT j FROM json_shared_bucketed_map_compact; +OPTIMIZE TABLE json_shared_bucketed_map_compact FINAL; +SELECT j FROM json_shared_bucketed_map_compact; +SELECT dynamicType(j.m), j.m, dynamicType(j.t), j.t FROM json_shared_bucketed_map_compact; +SELECT j.m.:`Map(String, UInt64)`['k'], j.t.:`Map(String, Tuple(a UInt64))`['k'].a FROM json_shared_bucketed_map_compact; +DROP TABLE json_shared_bucketed_map_compact; diff --git a/tests/queries/0_stateless/05211_local_object_storage_path_embedded_nul.reference b/tests/queries/0_stateless/05211_local_object_storage_path_embedded_nul.reference new file mode 100644 index 0000000000000..9f845b0366429 --- /dev/null +++ b/tests/queries/0_stateless/05211_local_object_storage_path_embedded_nul.reference @@ -0,0 +1,3 @@ +-- a path with an embedded NUL is rejected +-- the same for icebergLocal +-- a path inside user_files is still accepted (and fails for its own reason) diff --git a/tests/queries/0_stateless/05211_local_object_storage_path_embedded_nul.sh b/tests/queries/0_stateless/05211_local_object_storage_path_embedded_nul.sh new file mode 100755 index 0000000000000..492ed2d68abdb --- /dev/null +++ b/tests/queries/0_stateless/05211_local_object_storage_path_embedded_nul.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-msan +# Tag no-fasttest: Requires DeltaLake +# Tag no-msan: DeltaKernel is not compiled with msan + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A path with an embedded NUL byte must not pass the `user_files` containment check: the check sees the +# whole string, while every syscall the path is later passed to stops at the NUL. The path below is +# normalized into `user_files` as a whole string, but addresses `/../probe_$CLICKHOUSE_DATABASE` +# once truncated at the NUL. + +TAIL="../$(basename "${USER_FILES_PATH}")/nul_${CLICKHOUSE_DATABASE}" + +echo "-- a path with an embedded NUL is rejected" +${CLICKHOUSE_CLIENT} --query " + SELECT * FROM deltaLakeLocal('${USER_FILES_PATH}/../probe_${CLICKHOUSE_DATABASE}\0/${TAIL}') -- { serverError PATH_ACCESS_DENIED } +" + +echo "-- the same for icebergLocal" +${CLICKHOUSE_CLIENT} --query " + SELECT * FROM icebergLocal('${USER_FILES_PATH}/../probe_${CLICKHOUSE_DATABASE}\0/${TAIL}') -- { serverError PATH_ACCESS_DENIED } +" + +echo "-- a path inside user_files is still accepted (and fails for its own reason)" +mkdir -p "${USER_FILES_PATH}/nul_${CLICKHOUSE_DATABASE}" +${CLICKHOUSE_CLIENT} --query " + SELECT * FROM deltaLakeLocal('${USER_FILES_PATH}/nul_${CLICKHOUSE_DATABASE}') -- { serverError DELTA_KERNEL_ERROR } +" +rmdir "${USER_FILES_PATH}/nul_${CLICKHOUSE_DATABASE}" diff --git a/tests/queries/0_stateless/05212_parallel_with_access_checks.reference b/tests/queries/0_stateless/05212_parallel_with_access_checks.reference new file mode 100644 index 0000000000000..10f1739ece85f --- /dev/null +++ b/tests/queries/0_stateless/05212_parallel_with_access_checks.reference @@ -0,0 +1,12 @@ +-- without the CREATE TABLE privilege, directly +ACCESS_DENIED +-- without the CREATE TABLE privilege, inside PARALLEL WITH +ACCESS_DENIED +-- CREATE DATABASE inside PARALLEL WITH is checked too +ACCESS_DENIED +-- nothing was created +0 +0 +-- with the CREATE TABLE privilege, PARALLEL WITH still works +t_parallel_1 +t_parallel_2 diff --git a/tests/queries/0_stateless/05212_parallel_with_access_checks.sh b/tests/queries/0_stateless/05212_parallel_with_access_checks.sh new file mode 100755 index 0000000000000..d66dceb563cf2 --- /dev/null +++ b/tests/queries/0_stateless/05212_parallel_with_access_checks.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `PARALLEL WITH` must not bypass access checks of its subqueries. + +user="user_${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" +${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED WITH plaintext_password BY 'password'" +${CLICKHOUSE_CLIENT} --query "GRANT SELECT ON ${CLICKHOUSE_DATABASE}.* TO ${user}" +# `TABLE ENGINE` is granted up front so that the only privilege missing below is `CREATE TABLE`. +# `access_control_improvements.table_engines_require_grant` is enabled in the test configuration, and +# the `TABLE ENGINE` check in `getTablePropertiesAndNormalizeCreateQuery` does not consult `internal` - +# without this grant a `CREATE TABLE` would be refused by that check even when its own access check is +# skipped, which would hide the very bypass this test is about. +${CLICKHOUSE_CLIENT} --query "GRANT TABLE ENGINE ON Memory TO ${user}" + +CLIENT_AS_USER="${CLICKHOUSE_CLIENT} --user ${user} --password password" + +# The test database may be reused across runs, so only the objects of this test are counted, and they +# are dropped both before and after the run. +tables="'t_direct', 't_parallel_1', 't_parallel_2'" +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.t_direct, ${CLICKHOUSE_DATABASE}.t_parallel_1, ${CLICKHOUSE_DATABASE}.t_parallel_2" +${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${CLICKHOUSE_DATABASE}_db_1" +${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${CLICKHOUSE_DATABASE}_db_2" + +echo "-- without the CREATE TABLE privilege, directly" +${CLIENT_AS_USER} --query " + CREATE TABLE ${CLICKHOUSE_DATABASE}.t_direct (x UInt8) ENGINE = Memory +" 2>&1 | grep -q "ACCESS_DENIED" && echo "ACCESS_DENIED" || echo "ALLOWED" + +echo "-- without the CREATE TABLE privilege, inside PARALLEL WITH" +${CLIENT_AS_USER} --query " + CREATE TABLE ${CLICKHOUSE_DATABASE}.t_parallel_1 (x UInt8) ENGINE = Memory + PARALLEL WITH + CREATE TABLE ${CLICKHOUSE_DATABASE}.t_parallel_2 (x UInt8) ENGINE = Memory +" 2>&1 | grep -q "ACCESS_DENIED" && echo "ACCESS_DENIED" || echo "ALLOWED" + +echo "-- CREATE DATABASE inside PARALLEL WITH is checked too" +${CLIENT_AS_USER} --query " + CREATE DATABASE ${CLICKHOUSE_DATABASE}_db_1 + PARALLEL WITH + CREATE DATABASE ${CLICKHOUSE_DATABASE}_db_2 +" 2>&1 | grep -q "ACCESS_DENIED" && echo "ACCESS_DENIED" || echo "ALLOWED" + +echo "-- nothing was created" +${CLICKHOUSE_CLIENT} --query "SELECT count() FROM system.tables WHERE database = '${CLICKHOUSE_DATABASE}' AND name IN (${tables})" +${CLICKHOUSE_CLIENT} --query "SELECT count() FROM system.databases WHERE name IN ('${CLICKHOUSE_DATABASE}_db_1', '${CLICKHOUSE_DATABASE}_db_2')" + +echo "-- with the CREATE TABLE privilege, PARALLEL WITH still works" +${CLICKHOUSE_CLIENT} --query "GRANT CREATE TABLE ON ${CLICKHOUSE_DATABASE}.* TO ${user}" +${CLIENT_AS_USER} --query " + CREATE TABLE ${CLICKHOUSE_DATABASE}.t_parallel_1 (x UInt8) ENGINE = Memory + PARALLEL WITH + CREATE TABLE ${CLICKHOUSE_DATABASE}.t_parallel_2 (x UInt8) ENGINE = Memory +" +${CLICKHOUSE_CLIENT} --query "SELECT name FROM system.tables WHERE database = '${CLICKHOUSE_DATABASE}' AND name IN (${tables}) ORDER BY name" + +${CLICKHOUSE_CLIENT} --query "DROP TABLE ${CLICKHOUSE_DATABASE}.t_parallel_1, ${CLICKHOUSE_DATABASE}.t_parallel_2" +${CLICKHOUSE_CLIENT} --query "DROP USER ${user}" diff --git a/tests/queries/0_stateless/05212_path_containment_embedded_nul.reference b/tests/queries/0_stateless/05212_path_containment_embedded_nul.reference new file mode 100644 index 0000000000000..056f0839d8ecf --- /dev/null +++ b/tests/queries/0_stateless/05212_path_containment_embedded_nul.reference @@ -0,0 +1,4 @@ +-- the file function checks with fileOrSymlinkPathStartsWith only +data +-- the Filesystem database engine checks with pathStartsWith only +data diff --git a/tests/queries/0_stateless/05212_path_containment_embedded_nul.sh b/tests/queries/0_stateless/05212_path_containment_embedded_nul.sh new file mode 100755 index 0000000000000..978a90a4ccba7 --- /dev/null +++ b/tests/queries/0_stateless/05212_path_containment_embedded_nul.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A path with an embedded NUL byte must be rejected by every filesystem containment check: the check sees +# the whole string, while every syscall the path is later passed to stops at the NUL. Each case below names +# an existing entry inside `user_files` followed by a NUL and a suffix, so that a check which ignored the +# NUL would let the truncated path through and the query would succeed. + +DIR="${USER_FILES_PATH}/nul_${CLICKHOUSE_DATABASE}" +mkdir -p "${DIR}" +printf 'data' > "${DIR}/file.csv" + +echo "-- the file function checks with fileOrSymlinkPathStartsWith only" +${CLICKHOUSE_CLIENT} --query "SELECT file('nul_${CLICKHOUSE_DATABASE}/file.csv')" +${CLICKHOUSE_CLIENT} --query "SELECT file('nul_${CLICKHOUSE_DATABASE}/file.csv\0suffix') -- { serverError DATABASE_ACCESS_DENIED }" +${CLICKHOUSE_CLIENT} --query "SELECT file('${DIR}/file.csv\0suffix') -- { serverError DATABASE_ACCESS_DENIED }" + +echo "-- the Filesystem database engine checks with pathStartsWith only" +${CLICKHOUSE_CLIENT} --query "CREATE DATABASE nul_${CLICKHOUSE_DATABASE} ENGINE = Filesystem('${DIR}\0suffix') -- { serverError BAD_ARGUMENTS }" +${CLICKHOUSE_CLIENT} --query "CREATE DATABASE nul_${CLICKHOUSE_DATABASE} ENGINE = Filesystem('nul_${CLICKHOUSE_DATABASE}\0suffix') -- { serverError BAD_ARGUMENTS }" +${CLICKHOUSE_CLIENT} --query "CREATE DATABASE nul_${CLICKHOUSE_DATABASE} ENGINE = Filesystem('${DIR}')" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM nul_${CLICKHOUSE_DATABASE}.\`file.csv\`" +${CLICKHOUSE_CLIENT} --query "DROP DATABASE nul_${CLICKHOUSE_DATABASE}" + +rm -r "${DIR}" diff --git a/tests/queries/0_stateless/05213_execute_as_access_checks.reference b/tests/queries/0_stateless/05213_execute_as_access_checks.reference new file mode 100644 index 0000000000000..6454eeb3e1620 --- /dev/null +++ b/tests/queries/0_stateless/05213_execute_as_access_checks.reference @@ -0,0 +1,11 @@ +-- target has no CREATE TABLE privilege +ACCESS_DENIED +-- target has no CREATE DATABASE privilege +ACCESS_DENIED +-- two layers of nesting: PARALLEL WITH inside EXECUTE AS is checked too +ACCESS_DENIED +-- nothing was created +0 +0 +-- once the target has the privilege, EXECUTE AS still works +t_execute_as diff --git a/tests/queries/0_stateless/05213_execute_as_access_checks.sh b/tests/queries/0_stateless/05213_execute_as_access_checks.sh new file mode 100755 index 0000000000000..8c36ae883e7a5 --- /dev/null +++ b/tests/queries/0_stateless/05213_execute_as_access_checks.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `EXECUTE AS` must not bypass access checks of the impersonated statement: the statement runs with +# the privileges of the target user, so a target without the `CREATE TABLE` privilege must be denied. + +caller="caller_${CLICKHOUSE_DATABASE}" +target="target_${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${caller}, ${target}" +${CLICKHOUSE_CLIENT} --query "CREATE USER ${caller} IDENTIFIED WITH plaintext_password BY 'password'" +${CLICKHOUSE_CLIENT} --query "CREATE USER ${target}" +${CLICKHOUSE_CLIENT} --query "GRANT SELECT ON ${CLICKHOUSE_DATABASE}.* TO ${caller}, ${target}" +${CLICKHOUSE_CLIENT} --query "GRANT IMPERSONATE ON ${target} TO ${caller}" +# See the note in `05212_parallel_with_access_checks.sh`: `TABLE ENGINE` is granted up front so that +# the only privilege missing below is `CREATE TABLE`, otherwise the `TABLE ENGINE` check - which does +# not consult `internal` - would hide the bypass this test is about. +${CLICKHOUSE_CLIENT} --query "GRANT TABLE ENGINE ON Memory TO ${target}" + +CLIENT_AS_CALLER="${CLICKHOUSE_CLIENT} --user ${caller} --password password" + +# The test database may be reused across runs, so only the objects of this test are counted, and they +# are dropped both before and after the run. +tables="'t_execute_as', 't_nested_1', 't_nested_2'" +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.t_execute_as, ${CLICKHOUSE_DATABASE}.t_nested_1, ${CLICKHOUSE_DATABASE}.t_nested_2" +${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${CLICKHOUSE_DATABASE}_db" + +echo "-- target has no CREATE TABLE privilege" +${CLIENT_AS_CALLER} --query " + EXECUTE AS ${target} CREATE TABLE ${CLICKHOUSE_DATABASE}.t_execute_as (x UInt8) ENGINE = Memory +" 2>&1 | grep -q "ACCESS_DENIED" && echo "ACCESS_DENIED" || echo "ALLOWED" + +echo "-- target has no CREATE DATABASE privilege" +${CLIENT_AS_CALLER} --query " + EXECUTE AS ${target} CREATE DATABASE ${CLICKHOUSE_DATABASE}_db +" 2>&1 | grep -q "ACCESS_DENIED" && echo "ACCESS_DENIED" || echo "ALLOWED" + +echo "-- two layers of nesting: PARALLEL WITH inside EXECUTE AS is checked too" +${CLIENT_AS_CALLER} --query " + EXECUTE AS ${target} + CREATE TABLE ${CLICKHOUSE_DATABASE}.t_nested_1 (x UInt8) ENGINE = Memory + PARALLEL WITH + CREATE TABLE ${CLICKHOUSE_DATABASE}.t_nested_2 (x UInt8) ENGINE = Memory +" 2>&1 | grep -q "ACCESS_DENIED" && echo "ACCESS_DENIED" || echo "ALLOWED" + +echo "-- nothing was created" +${CLICKHOUSE_CLIENT} --query "SELECT count() FROM system.tables WHERE database = '${CLICKHOUSE_DATABASE}' AND name IN (${tables})" +${CLICKHOUSE_CLIENT} --query "SELECT count() FROM system.databases WHERE name = '${CLICKHOUSE_DATABASE}_db'" + +echo "-- once the target has the privilege, EXECUTE AS still works" +${CLICKHOUSE_CLIENT} --query "GRANT CREATE TABLE ON ${CLICKHOUSE_DATABASE}.* TO ${target}" +${CLIENT_AS_CALLER} --query " + EXECUTE AS ${target} CREATE TABLE ${CLICKHOUSE_DATABASE}.t_execute_as (x UInt8) ENGINE = Memory +" +${CLICKHOUSE_CLIENT} --query "SELECT name FROM system.tables WHERE database = '${CLICKHOUSE_DATABASE}' AND name IN (${tables}) ORDER BY name" + +${CLICKHOUSE_CLIENT} --query "DROP TABLE ${CLICKHOUSE_DATABASE}.t_execute_as" +${CLICKHOUSE_CLIENT} --query "DROP USER ${caller}, ${target}" diff --git a/tests/queries/0_stateless/05213_http_multipart_form_data_total_size.reference b/tests/queries/0_stateless/05213_http_multipart_form_data_total_size.reference new file mode 100644 index 0000000000000..a9796c468d97d --- /dev/null +++ b/tests/queries/0_stateless/05213_http_multipart_form_data_total_size.reference @@ -0,0 +1,3 @@ +the maximum size of multipart/form-data +30 +50 diff --git a/tests/queries/0_stateless/05213_http_multipart_form_data_total_size.sh b/tests/queries/0_stateless/05213_http_multipart_form_data_total_size.sh new file mode 100755 index 0000000000000..4d497382264e5 --- /dev/null +++ b/tests/queries/0_stateless/05213_http_multipart_form_data_total_size.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `http_max_multipart_form_data_size` cannot be set through URL parameters, so it comes from a user. +user="u_${CLICKHOUSE_DATABASE}" +${CLICKHOUSE_CLIENT} -q "CREATE USER $user IDENTIFIED WITH plaintext_password BY 'pw' SETTINGS http_max_multipart_form_data_size = 100" +${CLICKHOUSE_CLIENT} -q "GRANT CREATE TEMPORARY TABLE, SELECT ON *.* TO $user" + +url="${CLICKHOUSE_URL}&user=${user}&password=pw" +part="${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}_part.tsv" +python3 -c 'import sys; sys.stdout.write("x\n" * 30)' > "$part" + +# Two parts of 60 bytes: each one fits the 100 byte limit on its own, the form as a whole does not. +${CLICKHOUSE_CURL} -sS -F "a=@$part" -F "b=@$part" \ + "${url}&query=SELECT+count()+FROM+a&a_structure=s+String&b_structure=s+String&a_format=TSV&b_format=TSV" \ + | grep -o -m1 'the maximum size of multipart/form-data' + +# One part of the same size stays under the limit. +${CLICKHOUSE_CURL} -sS -F "a=@$part" \ + "${url}&query=SELECT+count()+FROM+a&a_structure=s+String&a_format=TSV" + +# A part that ends exactly at the limit is not over it. +exact="${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}_exact.tsv" +python3 -c 'import sys; sys.stdout.write("x\n" * 50)' > "$exact" +${CLICKHOUSE_CURL} -sS -F "a=@$exact" \ + "${url}&query=SELECT+count()+FROM+a&a_structure=s+String&a_format=TSV" + +${CLICKHOUSE_CLIENT} -q "DROP USER $user" +rm -f "$part" "$exact" diff --git a/tests/queries/0_stateless/05213_storage_file_embedded_nul.reference b/tests/queries/0_stateless/05213_storage_file_embedded_nul.reference new file mode 100644 index 0000000000000..3fab3d2e8c89f --- /dev/null +++ b/tests/queries/0_stateless/05213_storage_file_embedded_nul.reference @@ -0,0 +1,5 @@ +-- the file table function +data +-- the File engine +data +data diff --git a/tests/queries/0_stateless/05213_storage_file_embedded_nul.sh b/tests/queries/0_stateless/05213_storage_file_embedded_nul.sh new file mode 100755 index 0000000000000..79adfea697447 --- /dev/null +++ b/tests/queries/0_stateless/05213_storage_file_embedded_nul.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The `file` table function and the `File` engine stat the path and expand its globs on the filesystem before +# the matched paths are checked for containment in `user_files`, and every syscall stops at the first NUL byte. +# A path with an embedded NUL must therefore be rejected up front, before anything touches the filesystem. +# Each case names an existing entry followed by a NUL and a suffix, so that the truncated path would succeed. + +DIR="${USER_FILES_PATH}/nul_${CLICKHOUSE_DATABASE}" +mkdir -p "${DIR}" +printf 'data\n' > "${DIR}/file.csv" + +echo "-- the file table function" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM file('nul_${CLICKHOUSE_DATABASE}/file.csv', CSV, 'x String')" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM file('nul_${CLICKHOUSE_DATABASE}/file.csv\0suffix', CSV, 'x String') -- { serverError BAD_ARGUMENTS }" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM file('${DIR}/file.csv\0suffix', CSV, 'x String') -- { serverError BAD_ARGUMENTS }" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM file('${DIR}/*\0suffix', CSV, 'x String') -- { serverError BAD_ARGUMENTS }" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM file('${DIR}/file.csv\0suffix.zip :: file.csv', CSV, 'x String') -- { serverError BAD_ARGUMENTS }" +${CLICKHOUSE_CLIENT} --query "INSERT INTO FUNCTION file('${DIR}/file.csv\0suffix', CSV, 'x String') VALUES ('more') -- { serverError BAD_ARGUMENTS }" + +echo "-- the File engine" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE nul_${CLICKHOUSE_DATABASE} (x String) ENGINE = File(CSV, 'nul_${CLICKHOUSE_DATABASE}/file.csv\0suffix') -- { serverError BAD_ARGUMENTS }" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE nul_${CLICKHOUSE_DATABASE} (x String) ENGINE = File(CSV, '${DIR}/file.csv\0suffix') -- { serverError BAD_ARGUMENTS }" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE nul_${CLICKHOUSE_DATABASE} (x String) ENGINE = File(CSV, 'nul_${CLICKHOUSE_DATABASE}/file.csv')" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM nul_${CLICKHOUSE_DATABASE}" +${CLICKHOUSE_CLIENT} --query "DROP TABLE nul_${CLICKHOUSE_DATABASE}" + +cat "${DIR}/file.csv" +rm -r "${DIR}" diff --git a/tests/queries/0_stateless/05215_external_table_second_block_schema.python b/tests/queries/0_stateless/05215_external_table_second_block_schema.python new file mode 100644 index 0000000000000..80cec21b7ea83 --- /dev/null +++ b/tests/queries/0_stateless/05215_external_table_second_block_schema.python @@ -0,0 +1,262 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +# Every Native-protocol block sent for an external table after the first one must match the schema +# the first block bound. Without that check the columns of the later block are written to the table +# as a `Chunk` (which carries no types) and labelled with the table header again, so the data ends up +# being read as a type the client did not send it as. + +import os +import socket +import uuid + +CLICKHOUSE_HOST = os.environ.get("CLICKHOUSE_HOST", "127.0.0.1") +CLICKHOUSE_PORT = int(os.environ.get("CLICKHOUSE_PORT_TCP", "9000")) +CLICKHOUSE_DATABASE = os.environ.get("CLICKHOUSE_DATABASE", "default") +CLIENT_NAME = "external table schema check" + +CLIENT_REVISION = 54449 + +INCORRECT_DATA = 117 + + +def writeVarUInt(x, ba): + for _ in range(0, 9): + byte = x & 0x7F + if x > 0x7F: + byte |= 0x80 + + ba.append(byte) + + x >>= 7 + if x == 0: + return + + +def writeStringBinary(s, ba): + b = bytes(s, "utf-8") + writeVarUInt(len(b), ba) + ba.extend(b) + + +def readStrict(s, size=1): + res = bytearray() + while size: + cur = s.recv(size) + if not cur: + raise RuntimeError("Socket is closed") + size -= len(cur) + res.extend(cur) + + return res + + +def readUInt(s, size=1): + res = readStrict(s, size) + val = 0 + for i in range(len(res)): + val += res[i] << (i * 8) + return val + + +def readUInt8(s): + return readUInt(s) + + +def readUInt32(s): + return readUInt(s, 4) + + +def readVarUInt(s): + x = 0 + for i in range(9): + byte = readStrict(s)[0] + x |= (byte & 0x7F) << (7 * i) + + if not byte & 0x80: + return x + + return x + + +def readStringBinary(s): + size = readVarUInt(s) + return bytes(readStrict(s, size)).decode("utf-8") + + +def sendHello(s): + ba = bytearray() + writeVarUInt(0, ba) # Hello + writeStringBinary(CLIENT_NAME, ba) + writeVarUInt(26, ba) + writeVarUInt(9, ba) + writeVarUInt(CLIENT_REVISION, ba) + writeStringBinary(CLICKHOUSE_DATABASE, ba) + writeStringBinary("default", ba) # user + writeStringBinary("", ba) # password + s.sendall(ba) + + +def receiveHello(s): + packet_type = readVarUInt(s) + assert packet_type == 0, packet_type + readStringBinary(s) # server name + readVarUInt(s) # version major + readVarUInt(s) # version minor + readVarUInt(s) # revision + readStringBinary(s) # timezone + readStringBinary(s) # display name + readVarUInt(s) # version patch + + +def serializeClientInfo(ba, query_id): + writeStringBinary("default", ba) # initial_user + writeStringBinary(query_id, ba) # initial_query_id + writeStringBinary("127.0.0.1:9000", ba) # initial_address + ba.extend([0] * 8) # initial_query_start_time_microseconds + ba.append(1) # TCP + writeStringBinary("os_user", ba) # os_user + writeStringBinary("client_hostname", ba) # client_hostname + writeStringBinary(CLIENT_NAME, ba) # client_name + writeVarUInt(26, ba) + writeVarUInt(9, ba) + writeVarUInt(CLIENT_REVISION, ba) + writeStringBinary("", ba) # quota_key + writeVarUInt(0, ba) # distributed_depth + writeVarUInt(1, ba) # client_version_patch + ba.append(0) # No telemetry + + +def sendQuery(s, query): + ba = bytearray() + query_id = uuid.uuid4().hex + writeVarUInt(1, ba) # Query + writeStringBinary(query_id, ba) + + ba.append(1) # INITIAL_QUERY + serializeClientInfo(ba, query_id) + + writeStringBinary("", ba) # No settings + writeStringBinary("", ba) # No interserver secret + writeVarUInt(2, ba) # Stage - Complete + ba.append(0) # No compression + writeStringBinary(query, ba) + s.sendall(ba) + + +def serializeBlockInfo(ba): + writeVarUInt(1, ba) + ba.append(0) # is_overflows + writeVarUInt(2, ba) + writeVarUInt(0, ba) + ba.extend([0] * 4) # bucket_num + + +def sendBlock(s, table_name, columns, rows, data=b""): + ba = bytearray() + writeVarUInt(2, ba) # Data + writeStringBinary(table_name, ba) + serializeBlockInfo(ba) + writeVarUInt(len(columns), ba) + writeVarUInt(rows, ba) + for name, type_name in columns: + writeStringBinary(name, ba) + writeStringBinary(type_name, ba) + ba.extend(data) + s.sendall(ba) + + +def sendEmptyBlock(s, table_name=""): + sendBlock(s, table_name, [], 0) + + +def readException(s): + code = readUInt32(s) + readStringBinary(s) # name + text = readStringBinary(s) + readStringBinary(s) # trace + readUInt8(s) # has_nested + return code, text.replace("DB::Exception:", "").strip() + + +def readResponse(s): + """Returns either `exception ` or `values `.""" + values = [] + while True: + packet_type = readVarUInt(s) + if packet_type == 2: # Exception + code, _ = readException(s) + if code == INCORRECT_DATA: + return "exception INCORRECT_DATA" + return "exception {}".format(code) + if packet_type == 1: # Data + readStringBinary(s) # external table name + # BlockInfo + readVarUInt(s) + readUInt8(s) # is_overflows + readVarUInt(s) + readUInt32(s) # bucket_num + readVarUInt(s) + columns = readVarUInt(s) + rows = readVarUInt(s) + for _ in range(columns): + readStringBinary(s) # name + readStringBinary(s) # type + for _ in range(rows): + values.append(int.from_bytes(readStrict(s, 8), "little")) + continue + if packet_type == 3: # Progress + for _ in range(5): + readVarUInt(s) + continue + if packet_type == 5: # EndOfStream + return "values {}".format(values) + if packet_type == 6: # ProfileInfo + readVarUInt(s) # rows + readVarUInt(s) # blocks + readVarUInt(s) # bytes + readUInt8(s) # applied_limit + readVarUInt(s) # rows_before_limit + readUInt8(s) # obsolete field + continue + raise RuntimeError("Unexpected packet type {}".format(packet_type)) + + +def connect(): + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(30) + s.connect((CLICKHOUSE_HOST, CLICKHOUSE_PORT)) + sendHello(s) + receiveHello(s) + return s + + +def sendMismatchingBlocks(): + with connect() as s: + sendQuery(s, "SELECT x FROM ext") + # A block with zero rows is not an empty block: it creates the external table and binds + # `x UInt64` as its schema. + sendBlock(s, "ext", [("x", "UInt64")], 0) + # The same table, one row, but the column is declared as a different type. + sendBlock(s, "ext", [("x", "String")], 1, b"\x05hello") + sendEmptyBlock(s) + print(readResponse(s)) + + +def sendMatchingBlocks(): + with connect() as s: + sendQuery(s, "SELECT sum(x) FROM ext") + sendBlock(s, "ext", [("x", "UInt64")], 0) + sendBlock(s, "ext", [("x", "UInt64")], 1, (1).to_bytes(8, "little")) + sendBlock(s, "ext", [("x", "UInt64")], 1, (2).to_bytes(8, "little")) + sendEmptyBlock(s) + print(readResponse(s)) + + +def main(): + sendMismatchingBlocks() + sendMatchingBlocks() + + +if __name__ == "__main__": + main() diff --git a/tests/queries/0_stateless/05215_external_table_second_block_schema.reference b/tests/queries/0_stateless/05215_external_table_second_block_schema.reference new file mode 100644 index 0000000000000..0634c4aad23bf --- /dev/null +++ b/tests/queries/0_stateless/05215_external_table_second_block_schema.reference @@ -0,0 +1,2 @@ +exception INCORRECT_DATA +values [3] diff --git a/tests/queries/0_stateless/05215_external_table_second_block_schema.sh b/tests/queries/0_stateless/05215_external_table_second_block_schema.sh new file mode 100755 index 0000000000000..21ec762b6c50a --- /dev/null +++ b/tests/queries/0_stateless/05215_external_table_second_block_schema.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +python3 "$CUR_DIR"/05215_external_table_second_block_schema.python diff --git a/tests/queries/0_stateless/05215_parallel_with_access_checks_replicated_database.reference b/tests/queries/0_stateless/05215_parallel_with_access_checks_replicated_database.reference new file mode 100644 index 0000000000000..8e3a9dbe8fd45 --- /dev/null +++ b/tests/queries/0_stateless/05215_parallel_with_access_checks_replicated_database.reference @@ -0,0 +1,9 @@ +-- without the CREATE TABLE privilege, directly +ACCESS_DENIED +-- without the CREATE TABLE privilege, inside PARALLEL WITH +ACCESS_DENIED +-- nothing was created +0 +-- with the CREATE TABLE privilege, PARALLEL WITH still works +t_parallel_1 +t_parallel_2 diff --git a/tests/queries/0_stateless/05215_parallel_with_access_checks_replicated_database.sh b/tests/queries/0_stateless/05215_parallel_with_access_checks_replicated_database.sh new file mode 100755 index 0000000000000..dff7b96ce9cb5 --- /dev/null +++ b/tests/queries/0_stateless/05215_parallel_with_access_checks_replicated_database.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Tags: zookeeper + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `PARALLEL WITH` must not bypass the access checks of its subqueries in a `Replicated` database either. +# There a `CREATE TABLE` is additionally replayed through the replicated DDL log, and the replay of a +# statement written by the user is subject to the same rules as the replay of a direct statement. + +# The user, the database and its ZooKeeper path carry a random suffix, and the leftovers of a previous +# run that died halfway are dropped up front, so that the test can be rerun on the same server. +run_id=$(random_str 8) +user="user_${CLICKHOUSE_DATABASE}_${run_id}" +db="rdb_${CLICKHOUSE_DATABASE}_${run_id}" + +${CLICKHOUSE_CLIENT} --query "DROP DATABASE IF EXISTS ${db} SYNC" +${CLICKHOUSE_CLIENT} --query "CREATE DATABASE ${db} ENGINE = Replicated('/test/${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}/rdb_${run_id}', '1', '1')" +${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" +${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED WITH plaintext_password BY 'password'" +${CLICKHOUSE_CLIENT} --query "GRANT SELECT ON ${db}.* TO ${user}" +# `TABLE ENGINE` is granted up front so that the only privilege missing below is `CREATE TABLE`, +# see `05212_parallel_with_access_checks.sh`. +${CLICKHOUSE_CLIENT} --query "GRANT TABLE ENGINE ON Memory TO ${user}" + +CLIENT_AS_USER="${CLICKHOUSE_CLIENT} --user ${user} --password password --distributed_ddl_output_mode none" + +echo "-- without the CREATE TABLE privilege, directly" +${CLIENT_AS_USER} --query " + CREATE TABLE ${db}.t_direct (x UInt8) ENGINE = Memory +" 2>&1 | grep -q "ACCESS_DENIED" && echo "ACCESS_DENIED" || echo "ALLOWED" + +echo "-- without the CREATE TABLE privilege, inside PARALLEL WITH" +${CLIENT_AS_USER} --query " + CREATE TABLE ${db}.t_parallel_1 (x UInt8) ENGINE = Memory + PARALLEL WITH + CREATE TABLE ${db}.t_parallel_2 (x UInt8) ENGINE = Memory +" 2>&1 | grep -q "ACCESS_DENIED" && echo "ACCESS_DENIED" || echo "ALLOWED" + +echo "-- nothing was created" +${CLICKHOUSE_CLIENT} --query " + SELECT count() FROM system.tables + WHERE database = '${db}' AND name IN ('t_direct', 't_parallel_1', 't_parallel_2') +" + +echo "-- with the CREATE TABLE privilege, PARALLEL WITH still works" +${CLICKHOUSE_CLIENT} --query "GRANT CREATE TABLE ON ${db}.* TO ${user}" +${CLIENT_AS_USER} --query " + CREATE TABLE ${db}.t_parallel_1 (x UInt8) ENGINE = Memory + PARALLEL WITH + CREATE TABLE ${db}.t_parallel_2 (x UInt8) ENGINE = Memory +" +${CLICKHOUSE_CLIENT} --query " + SELECT name FROM system.tables + WHERE database = '${db}' AND name IN ('t_direct', 't_parallel_1', 't_parallel_2') + ORDER BY name +" + +${CLICKHOUSE_CLIENT} --query "DROP DATABASE ${db} SYNC" +${CLICKHOUSE_CLIENT} --query "DROP USER ${user}" diff --git a/tests/queries/0_stateless/05215_uniq_theta_short_state.reference b/tests/queries/0_stateless/05215_uniq_theta_short_state.reference new file mode 100644 index 0000000000000..f5f31343b08b4 --- /dev/null +++ b/tests/queries/0_stateless/05215_uniq_theta_short_state.reference @@ -0,0 +1,16 @@ +char(8, 0, 1, 3, 0, 0, 0, 0, 0) -> at least 24 bytes expected, actual 8 +char(8, 0, 2, 3, 0, 0, 0, 0, 0) -> seed hash mismatch +char(8, 0, 3, 3, 0, 0, 0, 0, 0) -> at least 16 bytes expected, actual 8 +char(8, 0, 4, 3, 0, 0, 0, 0, 0) -> seed hash mismatch +char(8, 0, 5, 3, 0, 0, 0, 0, 0) -> unsupported serial version 5 +char(8, 2, 2, 3, 0, 0, 0, 204, 147) -> at least 16 bytes expected, actual 8 +char(8, 3, 2, 3, 0, 0, 0, 204, 147) -> at least 24 bytes expected, actual 8 +char(8, 2, 3, 3, 0, 0, 0, 204, 147) -> at least 16 bytes expected, actual 8 +char(8, 3, 3, 3, 0, 0, 0, 204, 147) -> at least 24 bytes expected, actual 8 +char(8, 2, 4, 3, 0, 0, 0, 204, 147) -> at least 16 bytes expected, actual 8 +1000 +1000 +100 +1 0 +17 1 +33 2 diff --git a/tests/queries/0_stateless/05215_uniq_theta_short_state.sh b/tests/queries/0_stateless/05215_uniq_theta_short_state.sh new file mode 100755 index 0000000000000..81eced6b8d120 --- /dev/null +++ b/tests/queries/0_stateless/05215_uniq_theta_short_state.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest -- compiled w/o datasketches + +# `compact_theta_sketch_parser::parse` verifies that the buffer holds 8 bytes and then reads header +# fields that lie beyond them - `num_entries` at offset 8, `theta` at offset 16 - before it checks the +# size again. A state shorter than that, which any `CAST` from a string can produce, was therefore read +# past its end (an ASan heap-buffer-overflow), and the out-of-bounds value decided the size the parser +# reported as missing: the same query answered `at least 86160 bytes expected` on one run and something +# else on the next. Every size below is now the one the parser really needs before it can validate +# anything, and the same on every run. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The outcome of one state, reduced to the part that used to come from outside the buffer. The client +# prints the message of the exception rather than only its code. +outcome() +{ + ${CLICKHOUSE_CLIENT} --query "SELECT finalizeAggregation(CAST($1, 'AggregateFunction(uniqTheta, UInt64)'))" 2>&1 \ + | grep -o -E "at least [0-9]+ bytes expected, actual [0-9]+|seed hash mismatch|unsupported serial version [0-9]+|^[0-9]+$" \ + | head -n 1 +} + +# The leading byte of each `char` is the length of the state that follows it. All of these states are +# eight bytes long - the most the parser checks for before it reads further out. +# +# The four states the fuzzer produced come first, and their seed hash is zero. Serial versions 2 and 4 +# compare that against the one they compute and stop at `check_seed_hash`; serial version 1 computes +# the seed hash itself instead of comparing it, and serial version 3 reads `num_entries` before it +# compares, so those two are refused for their size. Serial version 5 does not exist and is refused +# outright. +for STATE in \ + "char(8, 0, 1, 3, 0, 0, 0, 0, 0)" \ + "char(8, 0, 2, 3, 0, 0, 0, 0, 0)" \ + "char(8, 0, 3, 3, 0, 0, 0, 0, 0)" \ + "char(8, 0, 4, 3, 0, 0, 0, 0, 0)" \ + "char(8, 0, 5, 3, 0, 0, 0, 0, 0)" +do + echo "${STATE} -> $(outcome "${STATE}")" +done + +# `204, 147` at offset 6 is the seed hash of the default seed, so these states get past +# `check_seed_hash` and reach every one of the reads that happen before the size is validated: +# `num_entries` alone for two preamble longs, `num_entries` and `theta` for three. In particular a +# serial version 2 state with two preamble longs is refused rather than taken for an empty sketch on +# the strength of bytes it does not have. +for STATE in \ + "char(8, 2, 2, 3, 0, 0, 0, 204, 147)" \ + "char(8, 3, 2, 3, 0, 0, 0, 204, 147)" \ + "char(8, 2, 3, 3, 0, 0, 0, 204, 147)" \ + "char(8, 3, 3, 3, 0, 0, 0, 204, 147)" \ + "char(8, 2, 4, 3, 0, 0, 0, 204, 147)" +do + echo "${STATE} -> $(outcome "${STATE}")" +done + +# A state that a `uniqTheta` aggregation wrote still reads back. +${CLICKHOUSE_CLIENT} --query " + SELECT uniqTheta(number) FROM numbers(1000); + SELECT uniqThetaMerge(s) FROM (SELECT uniqThetaState(number) AS s FROM numbers(1000)); + SELECT finalizeAggregation(CAST(unhex(hex(uniqThetaState(number))), 'AggregateFunction(uniqTheta, UInt64)')) FROM numbers(100); +" + +# A short state cannot be refused for its length alone: a single-entry sketch is sixteen bytes long, +# below the twenty-four the parser reads for other headers before it validates anything. The length of +# each state is printed next to the value it reads back as - it counts the one-byte length prefix, so +# the sketch itself is one byte smaller. +${CLICKHOUSE_CLIENT} --query " + SELECT + intDiv(length(hex(s)), 2) AS serialized_size, + finalizeAggregation(CAST(unhex(hex(s)), 'AggregateFunction(uniqTheta, UInt64)')) + FROM + ( + SELECT rows, uniqThetaStateIf(number, number < rows) AS s + FROM numbers(2) AS n, (SELECT arrayJoin([0, 1, 2]) AS rows) AS r + GROUP BY rows + ORDER BY rows + )" diff --git a/tests/queries/0_stateless/05216_http_external_table_duplicate_part_schema.reference b/tests/queries/0_stateless/05216_http_external_table_duplicate_part_schema.reference new file mode 100644 index 0000000000000..c4c3eac5fa7f2 --- /dev/null +++ b/tests/queries/0_stateless/05216_http_external_table_duplicate_part_schema.reference @@ -0,0 +1,2 @@ +Code: 117 +3 diff --git a/tests/queries/0_stateless/05216_http_external_table_duplicate_part_schema.sh b/tests/queries/0_stateless/05216_http_external_table_duplicate_part_schema.sh new file mode 100755 index 0000000000000..7ea8ad289dead --- /dev/null +++ b/tests/queries/0_stateless/05216_http_external_table_duplicate_part_schema.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The schema of an HTTP external table is bound by the first multipart part that names it. The `_structure` and +# `_types` fields are looked up by the part name each time, and plain form fields are added to the parameter set +# as they are read, so a later part with the same name can declare a different schema. Its data must be rejected, +# not written to the already-bound table, where the table header would label it with the first schema. + +FIRST="${CLICKHOUSE_TMP}/05216_first.tsv" +SECOND="${CLICKHOUSE_TMP}/05216_second.tsv" +THIRD="${CLICKHOUSE_TMP}/05216_third.tsv" +printf '1\n' > "${FIRST}" +printf 'hello\n' > "${SECOND}" +printf '2\n' > "${THIRD}" + +# The first part binds `_1 UInt64`; the second one declares `_1 String` for the same table. +${CLICKHOUSE_CURL} -sS -F 'ext_types=UInt64' -F "ext=@${FIRST}" -F 'ext_structure=_1 String' -F "ext=@${SECOND}" \ + "${CLICKHOUSE_URL}&query=SELECT+_1+FROM+ext" | grep -o -m1 'Code: 117' + +# Two parts with the same name and the same schema are appended to one table. +${CLICKHOUSE_CURL} -sS -F 'ext_types=UInt64' -F "ext=@${FIRST}" -F "ext=@${THIRD}" \ + "${CLICKHOUSE_URL}&query=SELECT+sum(_1)+FROM+ext" + +rm -f "${FIRST}" "${SECOND}" "${THIRD}" diff --git a/tests/queries/0_stateless/05218_allow_executable_tables.reference b/tests/queries/0_stateless/05218_allow_executable_tables.reference new file mode 100644 index 0000000000000..604b153530121 --- /dev/null +++ b/tests/queries/0_stateless/05218_allow_executable_tables.reference @@ -0,0 +1 @@ +x UInt32 diff --git a/tests/queries/0_stateless/05218_allow_executable_tables.sql b/tests/queries/0_stateless/05218_allow_executable_tables.sql new file mode 100644 index 0000000000000..f56bfcf22caae --- /dev/null +++ b/tests/queries/0_stateless/05218_allow_executable_tables.sql @@ -0,0 +1,35 @@ +-- `allow_executable_tables` gates reading through the `executable` table function and from +-- `Executable` and `ExecutablePool` tables. + +SELECT * FROM executable('nonexist.sh', 'TSV', 'x UInt32'); -- { serverError UNSUPPORTED_METHOD } + +SET allow_executable_tables = 0; +SELECT * FROM executable('nonexist.sh', 'TSV', 'x UInt32'); -- { serverError SUPPORT_IS_DISABLED } +DESCRIBE executable('nonexist.sh', 'TSV', 'x UInt32'); + +-- A definition is still accepted and can be managed; only reading it is refused. +CREATE TABLE t_exec_gate (x UInt32) ENGINE = Executable('nonexist.sh', 'TSV'); +CREATE VIEW v_exec_gate AS SELECT * FROM t_exec_gate; +CREATE MATERIALIZED VIEW mv_exec_gate ENGINE = MergeTree ORDER BY x AS SELECT * FROM t_exec_gate; +CREATE TABLE t_exec_pool_gate (x UInt32) ENGINE = ExecutablePool('nonexist.sh', 'TSV'); +DETACH TABLE t_exec_gate; +ATTACH TABLE t_exec_gate; + +SELECT * FROM t_exec_gate; -- { serverError SUPPORT_IS_DISABLED } +SELECT * FROM v_exec_gate; -- { serverError SUPPORT_IS_DISABLED } +SELECT * FROM t_exec_pool_gate; -- { serverError SUPPORT_IS_DISABLED } + +-- The gate is evaluated on the node that runs the read, so a distributed wrapper does not lift it. +SELECT * FROM remote('127.0.0.1', executable('nonexist.sh', 'TSV', 'x UInt32')); -- { serverError SUPPORT_IS_DISABLED } + +-- A statement that reads as part of its own execution is refused too. +CREATE TABLE t_exec_gate_copy ENGINE = MergeTree ORDER BY x AS SELECT * FROM executable('nonexist.sh', 'TSV', 'x UInt32'); -- { serverError SUPPORT_IS_DISABLED } +CREATE MATERIALIZED VIEW mv_exec_gate_populate ENGINE = MergeTree ORDER BY x POPULATE AS SELECT * FROM t_exec_gate; -- { serverError SUPPORT_IS_DISABLED } + +SET allow_executable_tables = 1; +SELECT * FROM t_exec_gate; -- { serverError UNSUPPORTED_METHOD } + +SET allow_executable_tables = 0; +DROP VIEW mv_exec_gate; +DROP VIEW v_exec_gate; +DROP TABLE t_exec_gate; diff --git a/tests/queries/0_stateless/05218_database_backup_quoted_locator_parallel_with.reference b/tests/queries/0_stateless/05218_database_backup_quoted_locator_parallel_with.reference new file mode 100644 index 0000000000000..8b916e9e97123 --- /dev/null +++ b/tests/queries/0_stateless/05218_database_backup_quoted_locator_parallel_with.reference @@ -0,0 +1,5 @@ +refused +refused +refused +0 1 +10 diff --git a/tests/queries/0_stateless/05218_database_backup_quoted_locator_parallel_with.sh b/tests/queries/0_stateless/05218_database_backup_quoted_locator_parallel_with.sh new file mode 100755 index 0000000000000..454cce40ebff8 --- /dev/null +++ b/tests/queries/0_stateless/05218_database_backup_quoted_locator_parallel_with.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Tags: no-encrypted-storage + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +BACKUP_DATABASE_NAME=${CLICKHOUSE_TEST_UNIQUE_NAME}_backup +ATTACHED_DATABASE_NAME=${CLICKHOUSE_TEST_UNIQUE_NAME}_attached +OTHER_DATABASE_NAME=${CLICKHOUSE_TEST_UNIQUE_NAME}_other + +$CLICKHOUSE_CLIENT -q """ +DROP DATABASE IF EXISTS $BACKUP_DATABASE_NAME; +DROP DATABASE IF EXISTS $ATTACHED_DATABASE_NAME; +DROP DATABASE IF EXISTS $OTHER_DATABASE_NAME; +CREATE DATABASE $BACKUP_DATABASE_NAME; + +CREATE TABLE $BACKUP_DATABASE_NAME.test_table (id UInt64) ENGINE = MergeTree ORDER BY id; +INSERT INTO $BACKUP_DATABASE_NAME.test_table SELECT number FROM numbers(10); + +BACKUP DATABASE $BACKUP_DATABASE_NAME TO Disk('backups', '$BACKUP_DATABASE_NAME') FORMAT Null; +""" + +# `PARALLEL WITH` runs its statements as internal queries, and the full `ATTACH DATABASE ... ENGINE = ...` +# form runs in `ATTACH` mode - the same pair the server's own replay of stored metadata shows. The quoted +# locator is accepted only on that replay, never in a statement a user writes, however it is wrapped. +$CLICKHOUSE_CLIENT -q """ +CREATE DATABASE $OTHER_DATABASE_NAME +PARALLEL WITH +ATTACH DATABASE $ATTACHED_DATABASE_NAME ENGINE = Backup('$BACKUP_DATABASE_NAME', 'Disk(\\'backups\\', \\'$BACKUP_DATABASE_NAME\\')'); +""" 2>&1 | grep -q -F 'Expected function' && echo 'refused' + +# Each block below must start from the same state: whether the sibling `CREATE DATABASE` of a +# `PARALLEL WITH` whose other statement threw is committed or rolled back depends on how the +# statements were scheduled, and with `max_threads = 1` it is committed. Drop it in between, so the +# refusal under test is what the block observes, not a leftover "database already exists". +$CLICKHOUSE_CLIENT -q "DROP DATABASE IF EXISTS $OTHER_DATABASE_NAME" + +$CLICKHOUSE_CLIENT -q """ +CREATE DATABASE $OTHER_DATABASE_NAME +PARALLEL WITH +CREATE DATABASE $ATTACHED_DATABASE_NAME ENGINE = Backup('$BACKUP_DATABASE_NAME', 'Disk(\\'backups\\', \\'$BACKUP_DATABASE_NAME\\')'); +""" 2>&1 | grep -q -F 'Expected function' && echo 'refused' + +$CLICKHOUSE_CLIENT -q "DROP DATABASE IF EXISTS $OTHER_DATABASE_NAME" + +# A quoted locator can carry credentials, and `PARALLEL WITH` formats its statements before the engine +# refuses them, so the formatted text must hide it: neither the logged query text (the same text an +# `ON CLUSTER` statement puts into the distributed DDL payload) nor the refusal message may carry it. +$CLICKHOUSE_CLIENT -q """ +CREATE DATABASE $OTHER_DATABASE_NAME +PARALLEL WITH +ATTACH DATABASE $ATTACHED_DATABASE_NAME ENGINE = Backup('$BACKUP_DATABASE_NAME', 'S3(\\'http://localhost:11111/05218\\', \\'ak\\', \\'SEKRIT_05218\\')'); +""" 2>&1 | grep -q -F 'Expected function' && echo 'refused' + +$CLICKHOUSE_CLIENT -q "DROP DATABASE IF EXISTS $OTHER_DATABASE_NAME" + +$CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS query_log" +$CLICKHOUSE_CLIENT -q """ +SELECT countIf(query LIKE '%SEKRIT_05218%' OR exception LIKE '%SEKRIT_05218%'), countIf(query LIKE '%Backup(%[HIDDEN]%') > 0 +FROM system.query_log +WHERE current_database = currentDatabase() + AND type != 'QueryStart' + AND event_date >= yesterday() AND event_time > now() - INTERVAL 5 MINUTE; +""" + +# The function form goes through the same wrapper. +$CLICKHOUSE_CLIENT -q """ +CREATE DATABASE IF NOT EXISTS $OTHER_DATABASE_NAME +PARALLEL WITH +ATTACH DATABASE $ATTACHED_DATABASE_NAME ENGINE = Backup('$BACKUP_DATABASE_NAME', Disk('backups', '$BACKUP_DATABASE_NAME')); + +SELECT count() FROM $ATTACHED_DATABASE_NAME.test_table; +""" + +$CLICKHOUSE_CLIENT -q """ +DROP DATABASE IF EXISTS $ATTACHED_DATABASE_NAME; +DROP DATABASE IF EXISTS $OTHER_DATABASE_NAME; +DROP DATABASE $BACKUP_DATABASE_NAME; +""" diff --git a/tests/queries/0_stateless/05218_dictionary_invalidate_query_access_checks.reference b/tests/queries/0_stateless/05218_dictionary_invalidate_query_access_checks.reference new file mode 100644 index 0000000000000..97c9358c2bd09 --- /dev/null +++ b/tests/queries/0_stateless/05218_dictionary_invalidate_query_access_checks.reference @@ -0,0 +1,9 @@ +-- a non-SELECT invalidate query of a local source is rejected, so it does not run as internal +1 +-- reloaded after the invalidate query was evaluated +1 +1 +-- nothing was created +0 +-- a SELECT invalidate query keeps working +2 diff --git a/tests/queries/0_stateless/05218_dictionary_invalidate_query_access_checks.sh b/tests/queries/0_stateless/05218_dictionary_invalidate_query_access_checks.sh new file mode 100755 index 0000000000000..c3b179a57cbb0 --- /dev/null +++ b/tests/queries/0_stateless/05218_dictionary_invalidate_query_access_checks.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The `invalidate_query` of a local `ClickHouse` dictionary source is executed as an `internal` query +# on behalf of the source user. Its text comes from `CREATE DICTIONARY`, so a user who may create a +# dictionary but not a table must not be able to smuggle a `CREATE TABLE` through it: only a `SELECT` +# is accepted, the same way as for the main dictionary query. + +user="user_${CLICKHOUSE_DATABASE}" +db="${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS ${user}" +${CLICKHOUSE_CLIENT} --query "CREATE USER ${user} IDENTIFIED WITH plaintext_password BY 'password'" +${CLICKHOUSE_CLIENT} --query "GRANT SELECT, CREATE DICTIONARY, DROP DICTIONARY, dictGet ON ${db}.* TO ${user}" + +CLIENT_AS_USER="${CLICKHOUSE_CLIENT} --user ${user} --password password" + +# The test database may be reused across runs, so the objects of this test are dropped both before +# and after the run. +${CLICKHOUSE_CLIENT} --query "DROP DICTIONARY IF EXISTS ${db}.dict_invalidate_ddl, ${db}.dict_invalidate_select" +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${db}.t_smuggled" + +echo "-- a non-SELECT invalidate query of a local source is rejected, so it does not run as internal" +${CLIENT_AS_USER} --query " + CREATE DICTIONARY ${db}.dict_invalidate_ddl (id UInt64, value UInt64) PRIMARY KEY id + SOURCE(CLICKHOUSE(QUERY 'SELECT 1 AS id, 1 AS value' INVALIDATE_QUERY 'CREATE TABLE ${db}.t_smuggled (x UInt8) ENGINE = Memory')) + LAYOUT(FLAT()) LIFETIME(MIN 1 MAX 1) +" +${CLIENT_AS_USER} --query "SELECT dictGet(${db}.dict_invalidate_ddl, 'value', toUInt64(1))" + +# The first load does not run the invalidate query; it is run by the periodic check (every 5 seconds) +# once the lifetime has passed, and it decides whether the dictionary is reloaded. Wait for that +# check to happen: the rejected invalidate query counts as "modified", so the dictionary is reloaded +# and `last_successful_update_time` advances. +first_update=$(${CLICKHOUSE_CLIENT} --query "SELECT toUnixTimestamp(last_successful_update_time) FROM system.dictionaries WHERE database = '${db}' AND name = 'dict_invalidate_ddl'") +for _ in $(seq 1 120) +do + update=$(${CLICKHOUSE_CLIENT} --query "SELECT toUnixTimestamp(last_successful_update_time) FROM system.dictionaries WHERE database = '${db}' AND name = 'dict_invalidate_ddl'") + if [[ "${update}" -gt "${first_update}" ]] + then + break + fi + sleep 0.5 +done +echo "-- reloaded after the invalidate query was evaluated" +${CLICKHOUSE_CLIENT} --query "SELECT toUnixTimestamp(last_successful_update_time) > ${first_update} FROM system.dictionaries WHERE database = '${db}' AND name = 'dict_invalidate_ddl'" +${CLIENT_AS_USER} --query "SELECT dictGet(${db}.dict_invalidate_ddl, 'value', toUInt64(1))" + +echo "-- nothing was created" +${CLICKHOUSE_CLIENT} --query "SELECT count() FROM system.tables WHERE database = '${db}' AND name = 't_smuggled'" + +echo "-- a SELECT invalidate query keeps working" +${CLIENT_AS_USER} --query " + CREATE DICTIONARY ${db}.dict_invalidate_select (id UInt64, value UInt64) PRIMARY KEY id + SOURCE(CLICKHOUSE(QUERY 'SELECT 1 AS id, 2 AS value' INVALIDATE_QUERY 'SELECT 1')) + LAYOUT(FLAT()) LIFETIME(MIN 1 MAX 1) +" +${CLIENT_AS_USER} --query "SELECT dictGet(${db}.dict_invalidate_select, 'value', toUInt64(1))" + +${CLICKHOUSE_CLIENT} --query "DROP DICTIONARY ${db}.dict_invalidate_ddl, ${db}.dict_invalidate_select" +${CLICKHOUSE_CLIENT} --query "DROP USER ${user}" diff --git a/tests/queries/0_stateless/05218_dynamic_json_constant_deep_recursion.reference b/tests/queries/0_stateless/05218_dynamic_json_constant_deep_recursion.reference new file mode 100644 index 0000000000000..99b8764f122fa --- /dev/null +++ b/tests/queries/0_stateless/05218_dynamic_json_constant_deep_recursion.reference @@ -0,0 +1 @@ +{"a":{"a":{}}} diff --git a/tests/queries/0_stateless/05218_dynamic_json_constant_deep_recursion.sql b/tests/queries/0_stateless/05218_dynamic_json_constant_deep_recursion.sql new file mode 100644 index 0000000000000..6178d3e164508 --- /dev/null +++ b/tests/queries/0_stateless/05218_dynamic_json_constant_deep_recursion.sql @@ -0,0 +1,8 @@ +SELECT finalizeAggregation(CAST(unhex('0130008008200000000101613000800820000000010161300080082000000000') AS AggregateFunction(any, Dynamic))); + +-- Turning a Dynamic constant into a literal recurses once per JSON level, and the depth comes from the +-- value, so it must be reported rather than overflow the stack. +SELECT finalizeAggregation(CAST(unhex(concat('01', repeat('3000800820000000010161', 4000), '300080082000000000')) AS AggregateFunction(any, Dynamic))); -- { serverError TOO_DEEP_RECURSION } + +-- Constant folding reads the value as a Field, which walks the same nesting through the columns. +SELECT if(finalizeAggregation(CAST(unhex(concat('01', repeat('3000800820000000010161', 4000), '300080082000000000')) AS AggregateFunction(any, Dynamic))), 1, 2); -- { serverError TOO_DEEP_RECURSION } diff --git a/tests/queries/0_stateless/05218_prewhere_alias_column_access_check.reference b/tests/queries/0_stateless/05218_prewhere_alias_column_access_check.reference new file mode 100644 index 0000000000000..018a9a02fa76d --- /dev/null +++ b/tests/queries/0_stateless/05218_prewhere_alias_column_access_check.reference @@ -0,0 +1,60 @@ +-- user granted only pub +SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias +[0,1,2,3,4,5,6,7] +SELECT pub FROM t_prewhere_alias PREWHERE pub = 2 +2 +SELECT pub FROM t_prewhere_alias PREWHERE secret = 200 +ACCESS_DENIED +SELECT pub FROM t_prewhere_alias PREWHERE secret_alias = 200 +ACCESS_DENIED +SELECT count() FROM t_prewhere_alias PREWHERE secret_alias = 200 +ACCESS_DENIED +SELECT pub FROM t_prewhere_alias PREWHERE secret_alias_expression = 201 +ACCESS_DENIED +SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias WHERE indexHint(secret >= 500) +ACCESS_DENIED +SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias WHERE indexHint(secret_alias >= 500) +ACCESS_DENIED +SELECT count() FROM t_prewhere_alias WHERE pub = 5 AND indexHint(secret < 500) +ACCESS_DENIED +SELECT count() FROM t_prewhere_alias WHERE pub = 5 AND indexHint(pub < 5) +1 +-- user granted pub and the alias columns, but not their source +SELECT secret_alias FROM t_prewhere_alias PREWHERE pub = 2 +200 +SELECT pub FROM t_prewhere_alias WHERE secret_alias = 200 +2 +SELECT pub FROM t_prewhere_alias PREWHERE secret_alias = 200 +2 +SELECT pub FROM t_prewhere_alias WHERE secret_alias_expression = 201 +2 +SELECT pub FROM t_prewhere_alias PREWHERE secret_alias_expression = 201 +2 +SELECT pub FROM t_prewhere_alias PREWHERE secret_alias_expression + 0 = 201 +2 +SELECT pub FROM t_prewhere_alias PREWHERE secret = 200 +ACCESS_DENIED +SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias WHERE indexHint(secret >= 500) +ACCESS_DENIED +-- user granted pub and the source column, but not the alias columns +SELECT pub FROM t_prewhere_alias PREWHERE secret = 200 +2 +SELECT pub FROM t_prewhere_alias WHERE secret_alias = 200 +ACCESS_DENIED +SELECT pub FROM t_prewhere_alias PREWHERE secret_alias = 200 +ACCESS_DENIED +SELECT pub FROM t_prewhere_alias WHERE secret_alias_expression = 201 +ACCESS_DENIED +SELECT pub FROM t_prewhere_alias PREWHERE secret_alias_expression = 201 +ACCESS_DENIED +SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias WHERE indexHint(secret >= 500) +[4,5,6,7] +SELECT count() FROM t_prewhere_alias WHERE pub = 5 AND indexHint(secret < 500) +0 +SELECT count() FROM t_prewhere_alias WHERE pub = 5 AND indexHint(secret < 501) +1 +-- row policy over a column the user is not granted +SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias +[0,1,3,4,5,6,7] +SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias PREWHERE pub > 1 +[3,4,5,6,7] diff --git a/tests/queries/0_stateless/05218_prewhere_alias_column_access_check.sh b/tests/queries/0_stateless/05218_prewhere_alias_column_access_check.sh new file mode 100755 index 0000000000000..89efe05350649 --- /dev/null +++ b/tests/queries/0_stateless/05218_prewhere_alias_column_access_check.sh @@ -0,0 +1,129 @@ +#!/usr/bin/env bash +# Tags: no-old-analyzer, no-parallel-replicas +# no-parallel-replicas: reading an ALIAS column with a grant on only that ALIAS column is denied under +# parallel replicas. That is an unrelated pre-existing bug (reproducible on master without this test's +# subject), so exclude the setting rather than encode the wrong behaviour in the reference. + +# Column-level SELECT grants must be enforced for columns that the planner resolves away before the +# access check runs, so that PREWHERE cannot be used as an oracle over a column the user cannot read: +# +# 1. An ALIAS column referenced in PREWHERE is replaced by its expression, so it used to never reach +# the list of selected columns and was not access checked at all, while the same alias in SELECT or +# WHERE was correctly denied. +# 2. A column used only as an `indexHint` argument is never read, only used for index analysis, but +# which granules survive the analysis is observable in the result. +# +# In both cases the required privilege is the one on the name written in the query, exactly as in +# WHERE: referencing an ALIAS column requires a grant on the ALIAS column itself, and a grant on the +# physical columns its expression reads is neither sufficient nor required. Those source columns are +# still read from disk to compute the ALIAS - the administrator authored the ALIAS expression and so +# chose what it exposes - but their values never reach the user. + +# Most of the queries below are expected to be denied, and each denial would otherwise add an +# log line and a stack trace to the output, because the harness streams server-side logs to the client. +CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=none + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +USER_PUB="user_pub_${CLICKHOUSE_DATABASE}" +USER_ALIAS="user_alias_${CLICKHOUSE_DATABASE}" +USER_SOURCE="user_source_${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} --multiquery --query " + DROP USER IF EXISTS ${USER_PUB}, ${USER_ALIAS}, ${USER_SOURCE}; + + CREATE TABLE t_prewhere_alias + ( + secret Int32, + pub Int32, + secret_alias Int32 ALIAS secret, + secret_alias_expression Int32 ALIAS secret_alias + 1 + ) + ENGINE = MergeTree ORDER BY secret SETTINGS index_granularity = 1, min_bytes_for_wide_part = 0; + + INSERT INTO t_prewhere_alias (secret, pub) SELECT number * 100, number FROM numbers(8); + + CREATE USER ${USER_PUB}, ${USER_ALIAS}, ${USER_SOURCE}; + GRANT SELECT(pub) ON ${CLICKHOUSE_DATABASE}.t_prewhere_alias TO ${USER_PUB}; + GRANT SELECT(pub, secret_alias, secret_alias_expression) ON ${CLICKHOUSE_DATABASE}.t_prewhere_alias TO ${USER_ALIAS}; + GRANT SELECT(pub, secret) ON ${CLICKHOUSE_DATABASE}.t_prewhere_alias TO ${USER_SOURCE}; +" + +# Run all the queries of one user in a single session, echoing each query before its own result so +# that every line of the reference is attributable. --ignore-error keeps the session going past the +# ACCESS_DENIED answers that most of these queries are expected to produce; without it the first +# denial would end the session and silently drop every query after it. +# +# An exception is printed by the client as exactly three lines - the "Received exception" banner, the +# "Code: ..." message and the echoed query - of which only the error name is stable: the banner holds +# the server version and the message holds the user name, which embeds ${CLICKHOUSE_DATABASE}. Reduce +# the block to that name, keeping it distinct per error so that a change of error shows up as a diff +# rather than being masked. +run_all() +{ + local user=$1 + shift + + local sql="" + local query + for query in "$@" + do + # The echoed query is derived from the query itself, so the two cannot drift apart. + sql+="SELECT '${query//\'/\'\'}'; +${query}; +" + done + + ${CLICKHOUSE_CLIENT} --user "${user}" --multiquery --ignore-error --query "${sql}" 2>&1 \ + | sed -E '/^Received exception/d; /^\(query:/d; s/^Code: [0-9]+\..*\(([A-Z_]+)\)$/\1/' +} + +echo "-- user granted only pub" +run_all "${USER_PUB}" \ + "SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias" \ + "SELECT pub FROM t_prewhere_alias PREWHERE pub = 2" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret = 200" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret_alias = 200" \ + "SELECT count() FROM t_prewhere_alias PREWHERE secret_alias = 200" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret_alias_expression = 201" \ + "SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias WHERE indexHint(secret >= 500)" \ + "SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias WHERE indexHint(secret_alias >= 500)" \ + "SELECT count() FROM t_prewhere_alias WHERE pub = 5 AND indexHint(secret < 500)" \ + "SELECT count() FROM t_prewhere_alias WHERE pub = 5 AND indexHint(pub < 5)" + +echo "-- user granted pub and the alias columns, but not their source" +run_all "${USER_ALIAS}" \ + "SELECT secret_alias FROM t_prewhere_alias PREWHERE pub = 2" \ + "SELECT pub FROM t_prewhere_alias WHERE secret_alias = 200" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret_alias = 200" \ + "SELECT pub FROM t_prewhere_alias WHERE secret_alias_expression = 201" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret_alias_expression = 201" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret_alias_expression + 0 = 201" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret = 200" \ + "SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias WHERE indexHint(secret >= 500)" + +echo "-- user granted pub and the source column, but not the alias columns" +run_all "${USER_SOURCE}" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret = 200" \ + "SELECT pub FROM t_prewhere_alias WHERE secret_alias = 200" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret_alias = 200" \ + "SELECT pub FROM t_prewhere_alias WHERE secret_alias_expression = 201" \ + "SELECT pub FROM t_prewhere_alias PREWHERE secret_alias_expression = 201" \ + "SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias WHERE indexHint(secret >= 500)" \ + "SELECT count() FROM t_prewhere_alias WHERE pub = 5 AND indexHint(secret < 500)" \ + "SELECT count() FROM t_prewhere_alias WHERE pub = 5 AND indexHint(secret < 501)" + +# A row policy is defined by an administrator, so it may reference columns the user cannot read. +echo "-- row policy over a column the user is not granted" +${CLICKHOUSE_CLIENT} --query "CREATE ROW POLICY p_prewhere_alias ON ${CLICKHOUSE_DATABASE}.t_prewhere_alias USING secret_alias_expression != 201 TO ${USER_PUB}" +run_all "${USER_PUB}" \ + "SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias" \ + "SELECT arraySort(groupArray(pub)) FROM t_prewhere_alias PREWHERE pub > 1" + +${CLICKHOUSE_CLIENT} --multiquery --query " + DROP ROW POLICY p_prewhere_alias ON ${CLICKHOUSE_DATABASE}.t_prewhere_alias; + DROP TABLE t_prewhere_alias; + DROP USER ${USER_PUB}, ${USER_ALIAS}, ${USER_SOURCE}; +" diff --git a/tests/queries/0_stateless/05218_settings_constraints_set_profile_in_same_query.reference b/tests/queries/0_stateless/05218_settings_constraints_set_profile_in_same_query.reference new file mode 100644 index 0000000000000..17c209230693f --- /dev/null +++ b/tests/queries/0_stateless/05218_settings_constraints_set_profile_in_same_query.reference @@ -0,0 +1 @@ +7 8 diff --git a/tests/queries/0_stateless/05218_settings_constraints_set_profile_in_same_query.sh b/tests/queries/0_stateless/05218_settings_constraints_set_profile_in_same_query.sh new file mode 100755 index 0000000000000..e4d11a0e11bf4 --- /dev/null +++ b/tests/queries/0_stateless/05218_settings_constraints_set_profile_in_same_query.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# A `SET` statement that changes `profile` installs a new constraint set halfway through itself, and the +# assignments and resets after that point must pass it, as they would in a separate statement. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +CONST_PROFILE="profile_const_$CLICKHOUSE_DATABASE" +READONLY_PROFILE="profile_readonly_$CLICKHOUSE_DATABASE" +MAX_PROFILE="profile_max_$CLICKHOUSE_DATABASE" +RELAXED_PROFILE="profile_relaxed_$CLICKHOUSE_DATABASE" + +$CLICKHOUSE_CLIENT -m -q " +DROP SETTINGS PROFILE IF EXISTS $CONST_PROFILE, $READONLY_PROFILE, $MAX_PROFILE, $RELAXED_PROFILE; +CREATE SETTINGS PROFILE $CONST_PROFILE SETTINGS max_execution_time = 10 CONST, SQL_tenant_id = 1 CONST; +CREATE SETTINGS PROFILE $READONLY_PROFILE SETTINGS readonly = 1; +CREATE SETTINGS PROFILE $MAX_PROFILE SETTINGS max_memory_usage MAX 1000000; +CREATE SETTINGS PROFILE $RELAXED_PROFILE SETTINGS max_memory_usage MAX 1099511627776; +" + +# One session on purpose: a rejected statement must leave it untouched, which the statements after it prove. +$CLICKHOUSE_CLIENT -m -q " +SET profile = '$CONST_PROFILE', max_execution_time = 999; -- { serverError SETTING_CONSTRAINT_VIOLATION } +SET profile = '$CONST_PROFILE', max_execution_time = DEFAULT; -- { serverError SETTING_CONSTRAINT_VIOLATION } +SET profile = '$READONLY_PROFILE', max_memory_usage = 1099511627776; -- { serverError READONLY } +SET profile = '$MAX_PROFILE', max_memory_usage = 1099511627776; -- { serverError SETTING_CONSTRAINT_VIOLATION } +-- a later, looser profile does not lift the constraint in force at the assignment +SET profile = '$MAX_PROFILE', max_memory_usage = 5000000, profile = '$RELAXED_PROFILE'; -- { serverError SETTING_CONSTRAINT_VIOLATION } +-- none of the rejected statements applied its profile, which would have installed this setting +SELECT getSetting('SQL_tenant_id'); -- { serverError UNKNOWN_SETTING } +-- assigning the value the profile installs is a no-op and stays allowed +SET profile = '$CONST_PROFILE', max_execution_time = 10; +-- what is not constrained still applies, before and after the profile change +SET SQL_before = 7, profile = '$CONST_PROFILE', SQL_after = 8; +SELECT getSetting('SQL_before'), getSetting('SQL_after'); +" + +$CLICKHOUSE_CLIENT -q "DROP SETTINGS PROFILE $CONST_PROFILE, $READONLY_PROFILE, $MAX_PROFILE, $RELAXED_PROFILE" diff --git a/tests/queries/0_stateless/05219_explain_ast_hide_secrets.reference b/tests/queries/0_stateless/05219_explain_ast_hide_secrets.reference new file mode 100644 index 0000000000000..405f09ad316dd --- /dev/null +++ b/tests/queries/0_stateless/05219_explain_ast_hide_secrets.reference @@ -0,0 +1,101 @@ +-- the inlined view body hides the key +SelectWithUnionQuery (children 1) + ExpressionList (children 1) + SelectQuery (children 2) + ExpressionList (children 1) + Identifier encrypted_secret + TablesInSelectQuery (children 1) + TablesInSelectQueryElement (children 1) + TableExpression (children 1) + SelectWithUnionQuery (children 1) + ExpressionList (children 1) + SelectQuery (children 2) + ExpressionList (children 1) + Function hex (alias encrypted_secret) (children 1) + ExpressionList (children 1) + Function encrypt (children 1) + ExpressionList (children 3) + Literal \'aes-128-ecb\' + Identifier secret + Literal \'[HIDDEN]\' + TablesInSelectQuery (children 1) + TablesInSelectQueryElement (children 1) + TableExpression (children 1) + TableIdentifier default.private_plaintext +-- the session setting alone does not disclose it + Literal \'aes-128-ecb\' + Literal \'[HIDDEN]\' +-- the graph dump hides it too +Literal \'[HIDDEN]\' +-- a secret typed into the explained query itself is hidden as well +SelectWithUnionQuery (children 1) + ExpressionList (children 1) + SelectQuery (children 1) + ExpressionList (children 1) + Function encrypt (children 1) + ExpressionList (children 4) + Literal \'aes-128-ecb\' + Literal \'[HIDDEN]\' + Literal \'[HIDDEN]\' + Function leftPad (children 1) + ExpressionList (children 3) + Literal \'[HIDDEN]\' + Literal \'[HIDDEN]\' + Literal \'[HIDDEN]\' +-- a positional secret written as a comparison collapses to one node, not to 'Function equals' +SelectWithUnionQuery (children 1) + ExpressionList (children 1) + SelectQuery (children 1) + ExpressionList (children 1) + Function encrypt (children 1) + ExpressionList (children 3) + Literal \'aes-128-ecb\' + Literal \'[HIDDEN]\' + Literal \'[HIDDEN]\' +-- a url the finder cannot read collapses to one node, not to 'Function concat' +SelectWithUnionQuery (children 1) + ExpressionList (children 1) + SelectQuery (children 2) + ExpressionList (children 1) + Asterisk + TablesInSelectQuery (children 1) + TablesInSelectQueryElement (children 1) + TableExpression (children 1) + Function url (children 1) + ExpressionList (children 1) + Literal \'[HIDDEN]\' +-- a mongodb url the finder cannot read collapses to one node, not to 'Function concat' +SelectWithUnionQuery (children 1) + ExpressionList (children 1) + SelectQuery (children 2) + ExpressionList (children 1) + Asterisk + TablesInSelectQuery (children 1) + TablesInSelectQueryElement (children 1) + TableExpression (children 1) + Function mongodb (children 1) + ExpressionList (children 2) + Literal \'[HIDDEN]\' + Literal \'c\' +-- the same url as a named override keeps its key, the value collapses +SelectWithUnionQuery (children 1) + ExpressionList (children 1) + SelectQuery (children 2) + ExpressionList (children 1) + Asterisk + TablesInSelectQuery (children 1) + TablesInSelectQueryElement (children 1) + TableExpression (children 1) + Function url (children 1) + ExpressionList (children 2) + Identifier creds + Function equals (children 1) + ExpressionList (children 2) + Identifier url + Literal \'[HIDDEN]\' +-- a nested map keeps its keys and hides its values + Literal \'http://x/f\' + Literal \'Authorization\' + Literal \'[HIDDEN]\' +-- the view itself stays usable for the restricted user +1 diff --git a/tests/queries/0_stateless/05219_explain_ast_hide_secrets.sh b/tests/queries/0_stateless/05219_explain_ast_hide_secrets.sh new file mode 100755 index 0000000000000..dcc3ca50f8c08 --- /dev/null +++ b/tests/queries/0_stateless/05219_explain_ast_hide_secrets.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-replicated-database +# Tag no-fasttest: the encryption functions are not available in the fast test build +# Tag no-replicated-database: SQL SECURITY DEFINER views and users are set up per-test + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `EXPLAIN AST optimize = 1` inlines the body of a view the user may only SELECT from, and the dump +# prints every literal verbatim. The secret arguments must be hidden as `SHOW CREATE` hides them. +# The stateless test server keeps `display_secrets_in_show_and_select` off, so the gate always hides here. + +user="user_05219_${CLICKHOUSE_DATABASE}_$RANDOM" +db=${CLICKHOUSE_DATABASE} +key='Sixteen byte key' + +${CLICKHOUSE_CLIENT} <