diff --git a/.github/workflows/test_clickhouse_role.yml b/.github/workflows/test_clickhouse_role.yml new file mode 100644 index 00000000..26a57e5a --- /dev/null +++ b/.github/workflows/test_clickhouse_role.yml @@ -0,0 +1,44 @@ +# Tests the clickhouse role's SQL users, quotas and grants and its apt pin +# (ansible/roles/clickhouse/tests), and adding and removing a replica with oonidata_clickhouse +# (ansible/roles/oonidata_clickhouse/tests), when the roles, their vars or the schema change +name: Test ClickHouse roles +on: + pull_request: + paths: + - "ansible/roles/clickhouse/**" + - "ansible/roles/oonidata_clickhouse/**" + - "ansible/group_vars/**" + - "ansible/host_vars/data*" + - "scripts/cluster-migration/schema.sql" + - ".github/workflows/test_clickhouse_role.yml" + push: + branches: [main] + paths: + - "ansible/roles/clickhouse/**" + - "ansible/roles/oonidata_clickhouse/**" + - "ansible/group_vars/**" + - "ansible/host_vars/data*" + - "scripts/cluster-migration/schema.sql" + - ".github/workflows/test_clickhouse_role.yml" + +jobs: + sql-objects: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - run: pipx install ansible-core==2.17.7 && sudo apt-get update -qq && sudo apt-get install -y --no-install-recommends python3-yaml + - run: ansible/roles/clickhouse/tests/run.sh + - if: failure() + uses: actions/upload-artifact@v4 + with: {name: sql-objects-out, path: ansible/roles/clickhouse/tests/out/} + replicas: + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + - run: pipx install ansible-core==2.17.7 && sudo apt-get update -qq && sudo apt-get install -y --no-install-recommends python3-yaml + - run: ansible/roles/oonidata_clickhouse/tests/run.sh + - if: failure() + uses: actions/upload-artifact@v4 + with: {name: replicas-out, path: ansible/roles/oonidata_clickhouse/tests/out/} diff --git a/.github/workflows/test_clickhouse_upgrade.yml b/.github/workflows/test_clickhouse_upgrade.yml new file mode 100644 index 00000000..a80607da --- /dev/null +++ b/.github/workflows/test_clickhouse_upgrade.yml @@ -0,0 +1,70 @@ +# Rolls a ClickHouse version bump out with the clickhouse role, one host at a time +# (ansible/roles/clickhouse/tests/upgrade). Slow (it installs ClickHouse twice on three hosts), +# so it only runs when it has something to test: +# - a PR that changes clickhouse_version: from the base's version to the PR's; +# - a PR that changes the role (other than its README and other tests), the schema step or +# this workflow: from production's version to tests/upgrade/next_version; +# - by hand, with any two versions. +# A change to the vars alone, e.g. a user's settings, doesn't run it. +name: Test ClickHouse upgrade +on: + pull_request: + paths: + - "ansible/group_vars/clickhouse/vars.yml" + - "ansible/roles/clickhouse/**" + - "!ansible/roles/clickhouse/README.md" + - "!ansible/roles/clickhouse/tests/**" + - "ansible/roles/clickhouse/tests/upgrade/**" + - "ansible/roles/oonidata_clickhouse/tasks/schema.yml" + - ".github/workflows/test_clickhouse_upgrade.yml" + workflow_dispatch: + inputs: + from_version: {description: "Version to install first (default: production's)", required: false} + to_version: {description: "Version to upgrade to (default: tests/upgrade/next_version)", required: false} + +jobs: + upgrade: + runs-on: ubuntu-24.04 + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + with: {fetch-depth: 0} + - name: Choose the versions + id: versions + env: + EVENT: ${{ github.event_name }} + BASE: ${{ github.event.pull_request.base.sha }} + FROM_INPUT: ${{ inputs.from_version }} + TO_INPUT: ${{ inputs.to_version }} + run: | + # clickhouse_version's value, without quotes or a trailing comment + version() { sed -n 's/^clickhouse_version: *["'"'"']\{0,1\}\([0-9.]*\).*/\1/p'; } + run=true from=$FROM_INPUT to=$TO_INPUT + if [ "$EVENT" = pull_request ]; then + base=$(git show "$BASE:ansible/group_vars/clickhouse/vars.yml" | version) + head=$(version < ansible/group_vars/clickhouse/vars.yml) + if [ "$base" != "$head" ]; then + from=$base to=$head + else + # the files of `paths:` above, other than the vars + changed=$(git diff --name-only "$BASE" HEAD \ + | grep -E '^(ansible/roles/clickhouse/|ansible/roles/oonidata_clickhouse/tasks/schema\.yml$|\.github/workflows/test_clickhouse_upgrade\.yml$)' \ + | grep -v -E '^ansible/roles/clickhouse/(README\.md$|tests/)' ; \ + git diff --name-only "$BASE" HEAD | grep '^ansible/roles/clickhouse/tests/upgrade/' || true) + [ -n "$changed" ] || run=false # only the vars changed, not the version + fi + fi + echo "run=$run" >> "$GITHUB_OUTPUT" + echo "from=$from" >> "$GITHUB_OUTPUT" + echo "to=$to" >> "$GITHUB_OUTPUT" + echo "run=$run from=${from:-production} to=${to:-next_version}" + - if: steps.versions.outputs.run == 'true' + run: pipx install ansible-core==2.17.7 && sudo apt-get update -qq && sudo apt-get install -y --no-install-recommends python3-yaml + - if: steps.versions.outputs.run == 'true' + env: + FROM_VERSION: ${{ steps.versions.outputs.from }} + TO_VERSION: ${{ steps.versions.outputs.to }} + run: ansible/roles/clickhouse/tests/upgrade/run.sh + - if: failure() + uses: actions/upload-artifact@v4 + with: {name: upgrade-out, path: ansible/roles/clickhouse/tests/upgrade/out/} diff --git a/ansible/deploy-clickhouse.yml b/ansible/deploy-clickhouse.yml index 630d1736..b40882a6 100644 --- a/ansible/deploy-clickhouse.yml +++ b/ansible/deploy-clickhouse.yml @@ -1,11 +1,14 @@ --- - name: Deploy oonidata clickhouse hosts + # Not the notebook: its ClickHouse isn't part of the cluster. hosts: - - notebook.ooni.org - data1.htz-fsn.prod.ooni.nu - data2.htz-fsn.prod.ooni.nu - data3.htz-fsn.prod.ooni.nu become: true + # One host at a time: a config change restarts ClickHouse, and data1-3 are + # also the Keeper ensemble, so restarting them together loses quorum. + serial: 1 roles: - role: prometheus_node_exporter tags: [node_exporter] diff --git a/ansible/group_vars/clickhouse/vars.yml b/ansible/group_vars/clickhouse/vars.yml index 73275001..2367d054 100644 --- a/ansible/group_vars/clickhouse/vars.yml +++ b/ansible/group_vars/clickhouse/vars.yml @@ -164,7 +164,6 @@ clickhouse_distributed_ddl: cleanup_delay_period: 60 max_tasks_in_queue: 1000 -clickhouse_role_manage_settings_profiles: True clickhouse_default_profiles: default: readonly: 2 @@ -227,171 +226,107 @@ clickhouse_default_users: profile: write quota: default +# SQL-driven users, quotas and grants (see the clickhouse role's README.md). +# Each user carries its own settings and sets readonly = 0, since a user without +# it gets the default profile's readonly = 2; what it may do comes from +# clickhouse_custom_grants. clickhouse_role_manage_users: True +clickhouse_role_manage_quotas: True +clickhouse_role_manage_grants: True +# Off until each node has been checked for SQL users and quotas not listed here. +clickhouse_sql_objects_drop_unlisted: false + clickhouse_custom_users: - - user: - name: fastpath + - name: fastpath password_type: sha256_hash password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_fastpath_password', profile='oonidevops_user_prod') | hash('sha256') }}" - networks: - - "IP '0.0.0.0/0'" - profile: - - write - quota: "fastpath" - databases: [ooni, oonitest] + networks: ["IP '0.0.0.0/0'"] + settings: + - "readonly = 0" + - "max_memory_usage = 61001001000" # 61 GB - - user: - name: oonimeasurements + # Only SELECT through its grants. CONST stops queries from raising its limits; + # other settings, such as log_comment, can still be set per query. + - name: oonimeasurements password_type: sha256_hash password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonimeasurements_password', profile='oonidevops_user_prod') | hash('sha256') }}" - networks: - - "IP '0.0.0.0/0'" + networks: ["IP '0.0.0.0/0'"] settings: - # 1 GB - - "max_memory_usage = 1001001000" - # 60 seconds - - "max_execution_time = 30" - # 500 GB - - "max_bytes_to_read = 501001001000" - # 5 B - - "max_rows_to_read = 5001001000" - # 5s - - "timeout_before_checking_execution_speed = 5" - # 50k - - "max_result_rows = 51000" - profile: - - readonly - quota: oonimeasurements - databases: [ooni, oonitest] - - - user: - name: ooniprobe + - "readonly = 0" + - "max_memory_usage = 1001001000 CONST" # 1 GB + - "max_execution_time = 30 CONST" # seconds + - "max_bytes_to_read = 501001001000 CONST" # 500 GB + - "max_rows_to_read = 5001001000 CONST" # 5 B + - "timeout_before_checking_execution_speed = 5 CONST" # seconds + - "max_result_rows = 51000 CONST" + + - name: ooniprobe password_type: sha256_hash password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_ooniprobe_password', profile='oonidevops_user_prod') | hash('sha256') }}" - networks: - - "IP '0.0.0.0/0'" - profile: - - write - quota: ooniprobe - databases: [ooni, oonitest] + networks: ["IP '0.0.0.0/0'"] + settings: + - "readonly = 0" + - "max_memory_usage = 61001001000" # 61 GB - - user: - name: oonirun + - name: oonirun password_type: sha256_hash password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonirun_password', profile='oonidevops_user_prod') | hash('sha256') }}" - networks: - - "IP '0.0.0.0/0'" - profile: - - write - quota: oonirun - databases: [ooni, oonitest] + networks: ["IP '0.0.0.0/0'"] + settings: + - "readonly = 0" + - "max_memory_usage = 61001001000" # 61 GB - - user: - name: oonitestlists + - name: oonitestlists password_type: sha256_hash password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_oonitestlists_password', profile='oonidevops_user_prod') | hash('sha256') }}" - networks: - - "IP '0.0.0.0/0'" - profile: - - write - quota: oonitestlists - databases: [ooni, oonitest] + networks: ["IP '0.0.0.0/0'"] + settings: + - "readonly = 0" + - "max_memory_usage = 61001001000" # 61 GB - - user: - name: notebook + - name: notebook password_type: sha256_hash password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_notebook_password', profile='oonidevops_user_prod') | hash('sha256') }}" - networks: - - "IP '0.0.0.0/0'" - profile: - - write - quota: notebook - databases: [ooni] + networks: ["IP '0.0.0.0/0'"] + settings: + - "readonly = 0" + - "max_memory_usage = 61001001000" # 61 GB -clickhouse_role_manage_quotas: True clickhouse_custom_quotas: - - quota: - name: oonimeasurements - duration: 600 - queries: 12000 - errors: 1000 - result_rows: 0 - read_rows: 0 - execution_time: 1000 - - # no limits set - - quota: - name: ooniprobe - duration: 0 - queries: 0 - errors: 0 - result_rows: 0 - read_rows: 0 - execution_time: 0 - - # no limits set - - quota: - name: oonirun - duration: 0 - queries: 0 - errors: 0 - result_rows: 0 - read_rows: 0 - execution_time: 0 - - - quota: - name: oonitestlists - duration: 600 - queries: 12000 - errors: 1000 - result_rows: 0 - read_rows: 0 - execution_time: 1000 - - # no limits set - - quota: - name: notebook - duration: 0 - queries: 0 - errors: 0 - result_rows: 0 - read_rows: 0 - execution_time: 0 + # Not assigned to the user (no `to`), so each node keeps its current assignment: + # the API's node, data2, has never had a quota, and these limits were never + # checked against its traffic. + - name: oonimeasurements + intervals: + - "INTERVAL 600 second MAX queries = 12000, errors = 1000, execution_time = 1000" + # No limits: these exist on the nodes already, and are listed so that they aren't + # dropped if clickhouse_sql_objects_drop_unlisted is turned on + - name: ooniprobe + - name: oonirun + - name: notebook + # The test lists editor only inserts url_priorities rows, so a small quota + - name: oonitestlists + intervals: + - "INTERVAL 3600 second MAX queries = 1000, errors = 100, execution_time = 600" + to: [oonitestlists] -clickhouse_role_manage_grants: True clickhouse_custom_grants: -- on: - databases: [ooni, oonitest] - tables: ["*"] - privileges: [SELECT] - to: [ooniprobe, oonimeasurements, oonirun, fastpath, notebook] - -- on: - databases: [ooni, oonitest] - tables: [url_priorities] - privileges: [INSERT] - to: [oonitestlists] - -- on: - databases: [ooni, oonitest] - tables: [faulty_measurements] - privileges: [INSERT] - to: [ooniprobe] - -- on: - databases: [ooni, oonitest] - tables: [fastpath, obs_web, obs_openvpn, jsonl, new_jsonl] - privileges: [INSERT] - to: [fastpath] - -clickhouse_custom_grant_roles: - - roles: [oonimeasurements] - to: [oonimeasurements] - -clickhouse_role_manage_roles: True -clickhouse_custom_roles: - - role: - name: oonimeasurements + - databases: [ooni, oonitest] + tables: ["*"] + privileges: [SELECT] + to: [ooniprobe, oonimeasurements, oonirun, fastpath, notebook] + - databases: [ooni, oonitest] + tables: [url_priorities] + privileges: [INSERT] + to: [oonitestlists] + - databases: [ooni, oonitest] + tables: [faulty_measurements] + privileges: [INSERT] + to: [ooniprobe] + - databases: [ooni, oonitest] + tables: [fastpath, obs_web, obs_openvpn, jsonl, new_jsonl] + privileges: [INSERT] + to: [fastpath] clickhouse_default_quotas: - quota: diff --git a/ansible/host_vars/notebook1.htz-fsn.prod.ooni.nu b/ansible/host_vars/notebook1.htz-fsn.prod.ooni.nu index 3e31da67..35dcb69c 100644 --- a/ansible/host_vars/notebook1.htz-fsn.prod.ooni.nu +++ b/ansible/host_vars/notebook1.htz-fsn.prod.ooni.nu @@ -152,6 +152,7 @@ clickhouse_listen_hosts: # this only applies to SQL managed users clickhouse_role_manage_users: false +clickhouse_role_manage_quotas: false clickhouse_role_manage_grants: false # these are handled via config files clickhouse_default_users: diff --git a/ansible/requirements/ansible-galaxy.yml b/ansible/requirements/ansible-galaxy.yml index 5172cbec..6019e101 100644 --- a/ansible/requirements/ansible-galaxy.yml +++ b/ansible/requirements/ansible-galaxy.yml @@ -3,10 +3,7 @@ roles: version: 0.26.0 - src: geerlingguy.certbot version: 5.2.0 - - src: https://github.com/idealista/clickhouse_role - scm: git - version: 3.5.1 - name: idealista.clickhouse_role + # The clickhouse role is vendored from idealista/clickhouse_role in ansible/roles/clickhouse - src: https://github.com/ooni/airflow-role.git scm: git name: ooni.airflow_role diff --git a/ansible/roles/clickhouse/LICENSE b/ansible/roles/clickhouse/LICENSE new file mode 100644 index 00000000..f2ecd366 --- /dev/null +++ b/ansible/roles/clickhouse/LICENSE @@ -0,0 +1,67 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. + +"Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of this License; and + +You must cause any modified files to carry prominent notices stating that You changed the files; and + +You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and + +If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. +You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. +5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + + Copyright 2020 Idealista S.A.U. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/ansible/roles/clickhouse/README.md b/ansible/roles/clickhouse/README.md new file mode 100644 index 00000000..cee1d937 --- /dev/null +++ b/ansible/roles/clickhouse/README.md @@ -0,0 +1,90 @@ +# clickhouse + +Installs and configures ClickHouse on Debian, and manages SQL-driven users, +quotas and grants. + +Vendored from [idealista/clickhouse_role](https://github.com/idealista/clickhouse_role) +by Idealista S.A.U. and contributors, tag `3.5.2` +(`5ca82f11c0bd554c4b52045a57cb0e0bc6addb61`, the last upstream release, December 2024), +under the Apache License 2.0, kept unchanged in `LICENSE`. OONI has modified the files +in this directory: the changes are listed below and in this repo's git history. Edit it +here like any other role in this repo. + +## What it does + +1. Installs ClickHouse `clickhouse_version` from packages.clickhouse.com, with the + repository key pinned in `files/clickhouse-repo-keyring.asc`, and pins the + `clickhouse-*` packages to that version in `/etc/apt/preferences.d/clickhouse`, so + `apt upgrade` can't move them (`tasks/install-Debian.yml`). To upgrade or + downgrade, change `clickhouse_version` and rerun the role. +2. Writes `config.xml` and `users.xml` from `templates/` (`tasks/config/clickhouse.yml`) + and the systemd unit (`tasks/service/clickhouse.yml`). New packages, config or unit + restart ClickHouse right away, not at the end of the play; the role then fails unless the + server runs `clickhouse_version`, and waits until none of its replicated tables is + read-only, so that with `serial: 1` or `--limit` a host is back before the next one. +3. Applies `clickhouse_custom_users`, `clickhouse_custom_quotas` and + `clickhouse_custom_grants` as SQL (`tasks/config/{users,quotas,grants}.yml`, + `templates/DDL/`), each behind its `clickhouse_role_manage_*` toggle: + - users and quotas are created if missing and `ALTER`ed if they exist, so changes + in vars apply to existing objects too; `ALTER` keeps grants and assignments; + - each grantee's privileges are replaced in one `GRANT ... WITH REPLACE OPTION`, + so privileges it keeps are never revoked, even briefly; roles granted by hand are + left alone; + - users and quotas that aren't in vars are dropped when + `clickhouse_sql_objects_drop_unlisted` is true. + + The SQL, and the checks after a restart, run through `clickhouse-client --config-file + {{ clickhouse_admin_client_config }}`, a root-only file with `clickhouse_admin_user`'s + credentials, and the statements go in on stdin, so task output isn't hidden. Only at + `-vvv` does ansible show the statements, and with them the users' password hashes. + +## Changes from upstream + +- Removed what this repo doesn't use: the JDBC bridge, clickhouse-copier, SQL + databases, SQL roles and settings profiles, config/users files copied from the + playbook, the `clickhouse_install_db` toggle, LDAP/Kerberos/`GRANTEES`/`DEFAULT + ROLE` in `CREATE USER`, Galaxy metadata and the standalone repo's scaffolding. +- Existing SQL objects were never updated: upstream only ran `CREATE ... IF NOT + EXISTS`, and listed existing objects by `storage = 'local directory'` while + ClickHouse 24.8 says `local_directory`, so it never saw them. +- Upstream started its grants with `REVOKE ALL ON *.* FROM ALL EXCEPT `, + so every SQL-driven user lost its privileges on every run until they were granted + again. +- Upstream put `--password` on every command line and hid all SQL task output with + `no_log`. +- Upstream didn't restart ClickHouse after installing new packages, and apt doesn't: a + version bump left the old version running until something else restarted it. +- Upstream installed `clickhouse_packages` at `clickhouse_version` but didn't pin them, + so `apt upgrade` would install the newest release in the repo (26.8.36.3 instead of + 24.8.6.70, checked 2026-10-27). +- Debian 12+ apt keyring (`signed-by`, from upstream PR #80) with the key pinned; the + entry upstream added without it is removed, as apt refuses both. + +## Vars for SQL-driven objects + +```yaml +clickhouse_custom_users: + - name: user1 + password_type: sha256_hash + password: "" + networks: ["IP '10.0.0.0/8'"] # HOST clause, ANY when omitted + settings: # SETTINGS clause, NONE when omitted + - "readonly = 0" + - "max_execution_time = 30 CONST" + +clickhouse_custom_quotas: + - name: user1 + intervals: # FOR clause, limits kept when omitted + - "INTERVAL 600 second MAX queries = 12000" + to: [user1] # TO clause, assignment kept when omitted + +clickhouse_custom_grants: + - databases: [db1] + tables: ["*"] + privileges: [SELECT] + to: [user1] +``` + +XML users, profiles and quotas (`clickhouse_default_users`, `clickhouse_admin_users`, +`clickhouse_default_profiles`, `clickhouse_default_quotas`) go into `users.xml`; see +`templates/users.xml.j2`. diff --git a/ansible/roles/clickhouse/defaults/main.yml b/ansible/roles/clickhouse/defaults/main.yml new file mode 100644 index 00000000..07ed5ee3 --- /dev/null +++ b/ansible/roles/clickhouse/defaults/main.yml @@ -0,0 +1,732 @@ +--- + +## General +# Version +clickhouse_version: 23.8.9.54 +clickhouse_release_type: lts # Values should be: stable, lts or testing + +# SQL-driven users, quotas and grants (tasks/config/{users,quotas,grants}.yml) +clickhouse_role_manage_users: False +clickhouse_role_manage_quotas: False +clickhouse_role_manage_grants: False +# Drop SQL-driven users and quotas that exist on the server but aren't in vars +clickhouse_sql_objects_drop_unlisted: False +# clickhouse-client config holding the admin credentials for those tasks +clickhouse_admin_client_config: /etc/clickhouse-client/ansible-admin.xml + +## Service options +# Documentation +clickhouse_documentation_link: https://clickhouse.tech/docs/en/ +clickhouse_environment_file: /etc/default/clickhouse + +# Handler for a changed config.xml or users.xml: Restart-clickhouse, or Reload-clickhouse to +# have the server reload them without a restart (not every setting applies on a reload) +clickhouse_handler_on_config_change: "Restart-clickhouse" + +# Owner +clickhouse_user: clickhouse +clickhouse_group: clickhouse + +# Maximum number of files limit +clickhouse_max_files: 32768 + +# Start on boot +clickhouse_service_enabled: True +# Current state: started, stopped +clickhouse_service_state: started + +# Logs +# If wanted to output the logs to a file define the following variable +# More information at https://www.freedesktop.org/software/systemd/man/systemd.exec.html#StandardOutput= +#clickhouse_log_file: +clickhouse_log_output: "{% if clickhouse_log_file is defined %} file:{{ clickhouse_log_file }} {% else %} journal {% endif %}" + +clickhouse_private_tmp: true +clickhouse_limit_no_file: 500000 + +# DEBIAN - Clickhouse +clickhouse_prerequisite_repos: + - apt-transport-https + - ca-certificates + - dirmngr +clickhouse_service: "clickhouse-server" +clickhouse_deb_repo: "deb [signed-by={{ clickhouse_deb_keyring_path }}] https://packages.clickhouse.com/deb {{ clickhouse_release_type }} main" +# The repo signing key is pinned (files/clickhouse-repo-keyring.asc), not fetched on +# every run; rotating it takes a reviewed commit. Verified 2026-09-22: fingerprint +# 3A9E A119 3A97 B548 BE14 57D4 8919 F6BD 2B48 D754, "ClickHouse Inc. Repositories Key +# ". +clickhouse_deb_keyring_path: /etc/apt/keyrings/clickhouse.asc +clickhouse_packages: + - procps + - "clickhouse-client={{ clickhouse_version }}" + - "clickhouse-common-static={{ clickhouse_version }}" + - "clickhouse-server={{ clickhouse_version }}" +# clickhouse_packages_extra: +# - clickhouse-common-static-dbg + +# Config templates +clickhouse_service_template_path: clickhouse-server.service.j2 +clickhouse_config_file_name: config.xml +clickhouse_users_file_name: users.xml + +# Directories +clickhouse_base_path: "/var/lib/clickhouse" +clickhouse_base_directory_mode: 0700 +clickhouse_config_directory: "/etc/clickhouse-server" +clickhouse_config_directory_mode: 0700 +clickhouse_config_directory_config_d: "{{ clickhouse_config_directory }}/config.d" +clickhouse_config_directory_users_d: "{{ clickhouse_config_directory }}/users.d" +clickhouse_log_directory: "/var/log/clickhouse-server" +clickhouse_log_directory_mode: 0740 +clickhouse_data_directory: "{{ clickhouse_base_path }}/data" +clickhouse_data_directory_mode: 0700 +clickhouse_tmp_directory: "{{ clickhouse_base_path }}/tmp" +clickhouse_tmp_directory_mode: 0755 +clickhouse_user_files_directory: "{{ clickhouse_base_path }}/user_files" +clickhouse_access_control_directory: "{{ clickhouse_base_path }}/access/" + +# Logs +clickhouse_logger: + level: trace + log: "{{ clickhouse_log_directory }}/clickhouse-server.log" + errorlog: "{{ clickhouse_log_directory }}/clickhouse-server.err.log" + size: 1000M + count: 10 + console: autodetection + +# clickhouse_logger_overrides_legacy: > +# +# none +# + +# clickhouse_logger_overrides: > +# +# +# ContextAccess (default) +# none +# +# +# DatabaseOrdinary (test) +# none +# +# + +# CORS +# clickhouse_custom_headers_response: > +# +#
+# Access-Control-Allow-Origin +# * +#
+#
+# Access-Control-Allow-Headers +# origin, x-requested-with +#
+#
+# Access-Control-Allow-Methods +# POST, GET, OPTIONS +#
+#
+# Access-Control-Max-Age +# 86400 +#
+# + +# Network +clickhouse_http_port: 8123 +clickhouse_tcp_port: 9000 +clickhouse_mysql_port: 9004 +clickhouse_postgresql_port: 9005 +# You must define the next two vars if you need to enable https/tcp secured +# clickhouse_https_port: 8443 +# clickhouse_tcp_port_secure: 9440 +# clickhouse_tcp_with_proxy_port: 9011 +clickhouse_interserver_http_port: 9009 +# clickhouse_interserver_https_port: 9010 +# clickhouse_interserver_http_host: +# clikhouse_interserver_http_credentials: +# user: interserver +# password: "{{ interserver_http_credentials_password }}" + + +# Configuration +clickhouse_config: + max_connections: 4096 + keep_alive_timeout: 3 + max_concurrent_queries: 100 + max_server_memory_usage: 0 + max_thread_pool_size: 10000 + max_server_memory_usage_to_ram_ratio: 0.9 + total_memory_profiler_step: 4194304 + total_memory_tracker_sample_probability: 0 + uncompressed_cache_size: 8589934592 + mark_cache_size: 5368709120 + # max_open_files: 262144 + mmap_cache_size: 1000 + compiled_expression_cache_size: 134217728 + compiled_expression_cache_elements_size: 10000 + # tmp_policy: tmp + default_profile: default + custom_settings_prefixes: "" + # system_profile: default + # buffer_profile: default + default_database: default + # timezone: + # umask: 027 + mlock_executable: true + remap_executable: true + builtin_dictionaries_reload_interval: 3600 + max_session_timeout: 3600 + default_session_timeout: 60 + # regions_hierarchy_file: /opt/geo/regions_hierarchy.txt + # regions_names_files_path: /opt/geo/ + # top_level_domains_path: /var/lib/clickhouse/top_level_domains/ + # top_level_domains: # Path to the list is under top_level_domains_path + # - domain: + # name: example_name + # path: /path/to/example_name.dat + dictionaries_config: "*_dictionary.xml" + user_defined_executable_functions_config: "*_function.xml" + # max_table_size_to_drop: 0 + # max_partition_size_to_drop: 0 + format_schema_path: /var/lib/clickhouse/format_schemas/ + # disable_internal_dns_cache: 1 + +clickhouse_default_networks: + - "::1" + - "127.0.0.1" + +# clickhouse_listen_hosts: +# - "::1" +# - "127.0.0.1" +# clickhouse_listen_try: 0 +# clickhouse_listen_reuse_port: 0 +# clickhouse_listen_backlog: 4096 +# clickhouse_remote_url_allow_hosts: + +clickhouse_http_server_default_response:
]]> + +# GRPC +# clickhouse_grpc_port: 9100 +clickhouse_enable_ssl: false +clickhouse_ssl_cert_file: /path/to/ssl_cert_file +clickhouse_ssl_key_file: /path/to/ssl_key_file +clickhouse_ssl_require_client_auth: false +clickhouse_ssl_ca_cert_file: /path/to/ssl_ca_cert_file +clickhouse_grpc_compression: deflate +clickhouse_grpc_compression_level: medium +clickhouse_max_send_message_size: -1 +clickhouse_max_receive_message_size: -1 +clickhouse_verbose_logs: false + +# SSL +# Server +clickhouse_certificatefile: /etc/clickhouse-server/server.crt +clickhouse_privatekeyfile: /etc/clickhouse-server/server.key +clickhouse_dhparamsfile: /etc/clickhouse-server/dhparam.pem +clickhouse_verificationmode: none +clickhouse_loaddefaultcafile_server: true +clickhouse_cachesessions_server: true +clickhouse_disableprotocols_server: sslv2,sslv3 +clickhouse_preferserverciphers_server: true +# Client +clickhouse_loaddefaultcafile_client: true +clickhouse_cachesessions_client: true +clickhouse_disableprotocols_client: sslv2,sslv3 +clickhouse_preferserverciphers_client: true +clickhouse_invalidcertificatehandler: + name: RejectCertificateHandler # Use for self-signed: AcceptCertificateHandler + +# Specify backend authenticacion: AD / LDAP (By default, AD) +clickhouse_auth_backend: AD + +# LDAP +# clickhouse_ldap: +# - serverame: +# host: +# port: +# bind_dn: +# base_dn: +# scope: +# search_filter: +# verification_cooldown: +# enable_tls: +# tls_minimum_protocol_version: +# tls_require_cert: +# tls_cert_file: +# tls_key_file: +# tls_ca_cert_file: +# tls_ca_cert_dir: +# tls_cipher_suite: + +# clickhouse_user_directories_ldap: +# server: +# roles: [] +# role_mappings: +# - base_dn: +# scope: +# search_filter: +# attribute: +# prefix: + +# Kerberos +# clickhouse_kerberos: +# principal: +# realm: + + +# clickhouse_remote_servers: +# - server: +# servername: +# # secret: +# shards: +# - shard: +# # internal_replication: +# # weight: +# replicas: +# - replica: +# host: +# port: +# # priority: +# # secure: + +clickhouse_remote_servers: + - server: + servername: test_shard_localhost + shards: + - shard: + replicas: + - replica: + host: localhost + port: 9000 + - server: + servername: test_cluster_two_shards_localhost + shards: + - shard: + replicas: + - replica: + host: localhost + port: 9000 + - shard: + replicas: + - replica: + host: localhost + port: 9000 + - server: + servername: test_cluster_two_shards + shards: + - shard: + replicas: + - replica: + host: 127.0.0.1 + port: 9000 + - shard: + replicas: + - replica: + host: 127.0.0.1 + port: 9000 + - server: + servername: test_cluster_two_shards_internal_replication + shards: + - shard: + internal_replication: true + replicas: + - replica: + host: 127.0.0.1 + port: 9000 + - shard: + internal_replication: true + replicas: + - replica: + host: 127.0.0.1 + port: 9000 + - server: + servername: test_shard_localhost_secure + shards: + - shard: + replicas: + - replica: + host: localhost + port: 9440 + secure: 1 + - server: + servername: test_unavailable_shard + shards: + - shard: + replicas: + - replica: + host: localhost + port: 9000 + - shard: + replicas: + - replica: + host: localhost + port: 1 + +# Clickhouse Keeper +# clickhouse_keeper: +# tcp_port: +# log_storage_path: +# snapshot_storage_path: +# coordination_settings: +# operation_timeout_ms: +# session_timeout_ms: +# raft_logs_level: +# keeper_servers: +# - keeper_server: +# server: +# id: +# hostname: +# port: + +# Zookeeper +# clickhouse_zookeeper: +# - node: +# host: +# port: + +# clickhouse_replicated_tables_macros: # Var deprecated use clickhouse_macros +# - server: servername +# macro: | +# 1 +# 1 + +# Macros +# clickhouse_macros: +# - macro: | +# 1 +# 1 +# server: +# - clickhouse1 +# - clickhouse2 +# - macro: | +# default + +# Graphite +# clickhouse_graphite: +# - node: +# host: # required if defined clickhouse_graphite +# port: # required if defined clickhouse_graphite +# timeout: # required if defined clickhouse_graphite +# interval: +# root_path: +# hostname_in_path: +# metrics: +# events: +# events_cumulative: +# asynchronous_metrics: + +# Prometheus +# clickhouse_prometheus: +# endpoint: "/metrics" +# port: 9363 +# metrics: true +# events: true +# asynchronous_metrics: true +# status_info: true + +# Query tables +clickhouse_tables_log: + # Query log (Used only for queries with setting log_queries = 1.) + query_log: + database: system + table: query_log + ttl: event_date + INTERVAL 30 DAY DELETE + partition_by: toYYYYMM(event_date) + # engine: ENGINE = MergeTree PARTITION BY toYYYYMM(event_date) ORDER BY (event_date, event_time) SETTINGS index_granularity = 1024 # Instead of partition_by, you can provide full engine expression + flush_interval_milliseconds: 7500 + # Trace log + trace_log: + database: system + table: trace_log + ttl: event_date + INTERVAL 30 DAY DELETE + partition_by: toYYYYMM(event_date) + flush_interval_milliseconds: 7500 + # Thread log + thread_log: + database: system + table: query_thread_log + ttl: event_date + INTERVAL 30 DAY DELETE + partition_by: toYYYYMM(event_date) + flush_interval_milliseconds: 7500 + # Views log + views_log: + database: system + table: query_views_log + ttl: event_date + INTERVAL 30 DAY DELETE + partition_by: toYYYYMM(event_date) + flush_interval_milliseconds: 7500 + # Part log + part_log: + database: system + table: part_log + ttl: event_date + INTERVAL 30 DAY DELETE + partition_by: toYYYYMM(event_date) + flush_interval_milliseconds: 7500 + # Text log + # Uncomment to write text log into table. Text log contains all information from usual server log but stores it in structured and efficient way. The level of the messages that goes to the table can be limited (), if not specified all messages will go to the table. + # text_log: + # database: system + # table: text_log + # flush_interval_milliseconds: 7500 + # level: + # Metric log + metric_log: + database: system + table: metric_log + ttl: event_date + INTERVAL 30 DAY DELETE + flush_interval_milliseconds: 7500 + collect_interval_milliseconds: 1000 + # Asynchronous metric log: + asynchronous_metric_log: + database: system + table: asynchronous_metric_log + ttl: event_date + INTERVAL 30 DAY DELETE + flush_interval_milliseconds: 7000 + # OpenTelemetry log + opentelemetry_span_log: + engine: > + engine MergeTree + partition by toYYYYMM(finish_date) + order by (finish_date, finish_time_us, trace_id) + database: system + table: opentelemetry_span_log + flush_interval_milliseconds: 7500 + # Crash log + crash_log: + database: system + table: crash_log + ttl: event_date + INTERVAL 30 DAY DELETE + partition_by: + flush_interval_milliseconds: 1000 + # Session log + session_log: + database: system + table: session_log + ttl: event_date + INTERVAL 30 DAY DELETE + partition_by: toYYYYMM(event_date) + flush_interval_milliseconds: 7500 + +# Compression +# clickhouse_compression: +# - case: +# min_part_size: +# min_part_size_ratio: +# method: +# level: + +# Encryption +# clickhouse_encryption: +# encryption: +# current_key_id: +# nonce: +# elements: +# - key: +# type: # key or keyhex +# id: +# # Key from a value / file or from ENV, use only one option. +# value: +# # from_env: ENV + +# Default replica params +# clickhouse_default_replica_path: +# clickhouse_default_replica_name: + +# Distributed DDL +# clickhouse_distributed_ddl: +# path: +# profile: +# pool_size: +# task_max_lifetime: +# cleanup_delay_period: +# max_tasks_in_queue: + +# MergeTree settings. See See documentation in clickhouse source code, in MergeTreeSettings.h +# clickhouse_mergetree_settings: +# min_compress_block_size: +# max_compress_block_size: +# index_granularity: + +clickhouse_graphitemergetree_settings: + name: graphite_rollup_example + path_column_name: Path + time_column_name: Time + value_column_name: Value + version_column_name: Version + pattern: + regexp: click_cost + function: any + retention: + - age: 0 + precision: 3600 + - age: 86400 + precision: 60 + default: + function: max + retention: + - age: 0 + precision: 60 + - age: 3600 + precision: 300 + - age: 86400 + precision: 3600 + +clickhouse_query_masking_rules: + - rule: + name: hide encrypt/decrypt arguments + regexp: ((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\) # or more secure, but also more invasive: (aes_\w+)\s*\(.*\) + replace: \1(???) + +# clickhouse_http_handlers: +# - rule: +# url: / +# methods: POST,GET +# headers: no-cache +# handler: +# type: dynamic_query_handler +# query_param_name: query +# - rule: +# url: /predefined_query +# methods: POST,GET +# handler: +# type: predefined_query_handler +# query: SELECT * FROM system.settings +# - rule: +# handler: +# type: static +# status: 200 +# content_type: text/plain; charset=UTF-8 +# response_content: config://http_server_default_response + +clickhouse_send_crash_reports: + enabled: false + anonymize: false + endpoint: https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277 + +# clickhouse_rocksdb: +# db_options: +# max_background_jobs: 8 +# cf_options: +# num_levels: 2 +# table_options: +# - table: +# name: TABLE +# options: +# max_background_jobs: 8 +# cf_options: +# num_levels: 2 + +# For additional config, for example: when you need to define catboost library paths +# clickhouse_config_xml_extra_config: | +# /home/catboost/data/libcatboostmodel.so +# /home/catboost/models/*_model.xml + +# users.xml + +# Profiles +clickhouse_default_profiles: + # Default settings + default: + max_memory_usage: 10000000000 + use_uncompressed_cache: 0 + load_balancing: random + max_partitions_per_insert_block: 100 + # Profile that allows only read queries + readonly: + readonly: 1 + +# clickhouse_custom_profiles: [] + +# Users +clickhouse_default_users: + - user: + name: default + password: + networks: "{{ clickhouse_default_networks }}" + profile: default + quota: default + - user: + name: readonly + password: readonly + networks: "{{ clickhouse_default_networks }}" + profile: readonly + quota: default + +# clickhouse_admin_users: +# - user: +# name: "{{ clickhouse_admin_user }}" +# password: "{{ clickhouse_admin_password }}" +# networks: "{{ clickhouse_default_networks }}" +# profile: "default" +# quota: "default" +# access_management: 1 +# named_collection_control: 1 +# show_named_collections: 1 +# show_named_collections_secrets: 1 + +# Users examples +# clickhouse_admin_user: admin +# clickhouse_admin_password: admin + +## See templates/users.xml.j2 for other possible passwords format (plain text is not recommended). + +# clickhouse_custom_users_xml: +# - user: +# name: "{{ clickhouse_admin_user }}" +# password: "{{ clickhouse_admin_password }}" +# networks: "{{ clickhouse_default_networks }}" +# profile: "default" +# quota: "default" +# access_management: 1 + +# - user: +# name: "user1" +# password: "user1" +# networks: "{{ clickhouse_default_networks }}" +# profile: "default" +# quota: "default" +# databases: [test1] + +clickhouse_custom_users: [] +# clickhouse_custom_users: +# - name: user1 +# password_type: sha256_hash +# password: "" +# networks: ["IP '10.0.0.0/8'"] # HOST clause; ANY when omitted +# settings: # SETTINGS clause; NONE when omitted +# - "readonly = 0" +# - "max_execution_time = 30 CONST" + +# clickhouse_custom_grants: +# - databases: [db1] +# tables: ["*"] +# privileges: [SELECT] +# to: [user1] + +# Quotas +clickhouse_default_quotas: + - quota: + name: default + duration: 3600 + queries: 0 + errors: 0 + result_rows: 0 + read_rows: 0 + execution_time: 0 + +clickhouse_custom_quotas: [] +# clickhouse_custom_quotas: +# - name: user1 +# intervals: # FOR clause; limits are kept as they are when omitted +# - "INTERVAL 600 second MAX queries = 12000" +# to: [user1] # TO clause; assignment is kept as it is when omitted + + +# For additional config, like clickhouse_config_xml_extra_config but in users.xml +# clickhouse_user_xml_extra_config: | +# value +# +# +# value +# +# diff --git a/ansible/roles/clickhouse/files/clickhouse-repo-keyring.asc b/ansible/roles/clickhouse/files/clickhouse-repo-keyring.asc new file mode 100644 index 00000000..c31c3f7d --- /dev/null +++ b/ansible/roles/clickhouse/files/clickhouse-repo-keyring.asc @@ -0,0 +1,93 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQINBGGbfsABEAC+01IxYaykEHIBW4dom/YipkmjPGbkU/oDibqsHE8VgDLC10Xt +Glto0H9q1fsTIi1nZ9S4lWUuuEAFgcPBt82qnT3E11ZCDC5khjFvPb7d0iK5Wg5T +a1hHNLxuqvRn4sOCyMp7LZJkgLFavvGnXWsHOmB3TzQRaQouc+Y21hqANXFCZXCD +be3Wbdy86ZHOezoWEdHaU1868ZlCPMq10lpBFcLzvdeEfMDAYhOE43c5X0tSbxNo +//kingPLPQ7nxbgsGDKNIsghazmKyMaJA+qpzMmu8qo1ioPZfW8GcYC06cBBAklQ +2dQmGqNvpdJeItBeis9kfJvfkWYCbQ5+ebmm/uJDGCXosIf3/OliMykJhiJzILp5 +1AJqCBD6muL5aP7UorGtVrCQeW2DKB6vtw73qoWmIAmMdW80Nr2lkFu8sDM8VeCI +YUULVa2mkuAet8B3TEtFs6dRZZZNlw6fNny6gmN1dbh+aDebilkHn7eOm8A3qUO3 +PgYNP8swIuGBDQ37lEKZaN6glL7+h0TY9Pz0UsBlCOOh0f3jrjVR2eseQfMtm5TU +/cqY7/+eTzycRpHRHl+xiSiUT1XwyzCf0sAEuMX6d1Um0KFXWWHYdpe4/P4yQmLw +Ex3JaYQHM7EBbLsj9vK7J9qAnrtH0kdyz1RBVE1SV0APS7qNec2GBUXh5QARAQAB +tDpDbGlja0hvdXNlIEluYy4gUmVwb3NpdG9yaWVzIEtleSA8cGFja2FnZXNAY2xp +Y2tob3VzZS5jb20+iQJOBBMBCAA4FiEEOp6hGTqXtUi+FFfUiRn2vStI11QFAmGb +fsACGwMFCwkIBwIGFQoJCAsCBBYCAwECHgECF4AACgkQiRn2vStI11R0OA/9GHe5 +Gfco4zWrPSX1cCM1sZZeLZUXeRu4iJlJPahi1sjP2J0rcuv5Xp7paW23el0ksXf4 +P+xknzzp8SlCndQKFWsb404vsNK5VtPB1RJ0Sje0dFM2nO1/Kt8pVX+r1WFJ1gur +BSugsAs3erTCy/ZIB6nhRcFBGsJfvZV24bVxFQKUW6ARigia3bqOZlM4ksdvjrLL +HZ84WiLY++W3wTMIh34KzLClVokl4au8L9Z1g+24EqJMRV7722FVm3GZDdVhmzNy +gKXQvopFvrfQY8Uk9ApZOVSsx9zH65UDP0u13ki7UTlyCi9ucGgT4w6cek1DUcXN +W0FU8498J80fADcWG4bsPjiJ7WiKI29saVked4gfUqx0G5kUCP+l5oo+okm/BFrt +66sO0cEbG9KZCmgdTa2KoNUtsampIBXDG0+9c2SWp3pa1j5sErngR2dyV6L8Muxx +dQJbVR4RxXkM8LkAUMN+YAatknMKEPa7gXalt73fiuvFXpY7NOxeUmlrRXucnCY8 +vSnTNPcuQYAeNyg7z2gyrpqp81Kd+XKUIdKYuydPpvRHzhFOBIf68NUc3MR1UeZx +eBF+bnVkkyL5nRhU+1eLZ0YyW2/PBCmegUahCHEWLpXGzeNBwdnt3HrJEQg54Fu8 +VPcKFa15kJ8QghTlMwYSQkXUPZeBOYaHtva+aSqJATMEEAEIAB0WIQSdgk/zlCIK +KzIjZgsAGciEq/hW0AUCYeaqyAAKCRAAGciEq/hW0CKVB/42Pla8EDBNTrki3CsV +hycpjrEDKnsxQgkN+DowtZPgRdYRaiQcIgk/xEgsud0RdMV16LYiHhj7YLvuTFay +H/QLhppatmQcEzyCZEvodklP8qX8fwls8QX9dOBDjfzfReV0ESrAs8SFcmFlLarb +/9LBX9izD0ouQqxdkkRnGS68SOu8WCwxffrJjI9qqdAsx5KWqqjuoRWkDIAQXOqX +sPynaqFFG6Pqhj99MT/+p+TxUhcPtqysII/2TdXm5yE0cnkX5xcog9bLi6Lp5G0+ +2hv/sLxrtdrobtuZMNetfyb/g7zclTWbAzGAxPm/Y5kFqgqUgztsZRV4n80SOcM6 +G4OouQINBGGbfsABEADXZ0mAMCsf7Y2yJ2aSBLCVPUEZESeB+kDAciksAguHeMVp +7owI2mfAfw4Z8TEQE6aOb+cqbDbqneQ6nuaRxMNvlDhhtrKztt+UxWrvgPMvJ59/ +O1ujY7+VZ+3lXJ01r0u5Xdk1zUZ1uYYl8+Lt3/WmJmmlosX22EwQdsAvuRuOmuM2 +jPTMVN81TaZ2R0Om5vT2L8/SQY7csxc5OylBLskuDQmU1PNlkwTltCMqHl7g08LP +ADLGi/3C9I/0P92A+jZ9RgDCI8BIOS5Ny7CStT96dqrje6TLbWRxIWBjn1xsVUTx +YQY4FXYfw1p7Rm5kzZGImzI4Z7wNcVHCty+X9La+7yKwABW2BeTqP0rCXpKmy4nN +OgwTDVRaA5CL8mg2doK8HOoa03SgziPGEeHHFyS7I2DJAEw+h9K8jXVy9Tgd7ZSX +mO4Z99ovdHksX4fmD7iOWU2WRp3b2yhpO8b/TgU2r9AQaNMAOsudN+0Q9NGkEYhW +RdWsGapJpa3Q+v1s88lodPhjcIkqGUCH82T/+MhTDdotssj8a6ueg3mcwVD38690 +j8ngoBCPmLCNtgZDqhtfy+905uO3ksonF/X2lFc/YG7sqBiOHoDKyN347pPyK8Kb +ukiMoOasDIQUsryBUeAM0M0Xzftc66BaPgH75KTPZqkOO8aTBCRxl/AHTejV5wAR +AQABiQI2BBgBCAAgFiEEOp6hGTqXtUi+FFfUiRn2vStI11QFAmGbfsACGwwACgkQ +iRn2vStI11RimQ//a0dHoSekuMLDPEYpQYEEr5JYYWFdFPA/ixeMHZ566ankWMqK +HynhbqqRXVhqKd0pveV+ALhgrUU1nBB9Ma5P51VNEGC0rThiLBVkCTrV0UrUXxxg +A5cTJBVZe7yyysqdCHcuuNNrlQbuzhkJWBySgeWrbmd4VUbDdL0iGMFSAtbrYjSW +reBDBgWBYd8uRiPHtc6ACBIhzGGOulw0k4NnMXHDJyCwEv7EAxESY/V8kLK3p7DX +c6tpmd1vZ38X52CnVRM6aLQl6YWKXhA+vp9cPJehfEbK+ZPErbCB4jF6AC4BFwnt +E0YSod6TVM+wJBuPplnWr4nUUhsk/FeulWGF+AhUG2XJiIn90ZHIcxlct8Nt1zBg +oquxWYmqt/z+s7TOBXnvotOMJggjgsPozFlHjoy47Xc14VwzxnU+NknXYqDAsTfj +gXiw7NRQdJ8BUd4TSf8iINf2uCNzp4QDJ37VfN7/BiaXBGjEEN+kOKpxkse6qm06 +GFd+bXGSHh6H7z2d973k2QIfW03opWAwS+AdH8xmUUaAyyhEql5mlXZVkQRS8rEO +4IV5HQV8FD5iV9bdNihaydlKP09/D9ZDlie0dJZXOIHQDuwDt4Ab3reSs1T5utCl +QE0FS8ZZjPuitq/l+TsT83TASVynvEmwvNiEbpOsNjvB1u++w4niqCHQv3a5Ag0E +Yyw+bgEQAOECkzewc6GdlGA65PIqEPL4JORnsYr0fHNHYGJ+WCHC9HuQEcFJ8Q0R +7nSNbMmi6KjnTqM4QdkLmyEQs9TKK5brFBa/ObbL4JVyBJTi4LdlCg95YfEQKTCl +b7roaLwKH+TykBJ617R+QUvFp65YCwzNiS2aKu6AC6zsD2rsrJulq5LIDKvWwLWf +FtwRwOHMgviuGlj/97omXUhuPIScJ5rB+QzICaiKriF/qVVO7obTV0OcUYrTOkNK +o8H/q/fw0FSvX3eznz6fK46gi468P34VO5ZLZs4UQih7EzDqdGz4Lo8GnIrtThWH +5rLyFhm7PdB6yLwf8bQeqDviI73Zb0w9ookYJQpBdglnDyLdhVjoAA0Hp3Q5XL6y +vZVZz06YL3k1VWHI2H/AWeceTFlrExmF2u1KousrwD8/583T042oRcGW97YV5Q9U +w7iffKvJiLp5VBZa319FsvF9hxM8bZVgVqBksBI/ScIvzMgG5FHeGflJ2xEIWWv2 +uDiAs498kmm5wI6HdcfsYuTT+uPsKSNd5G+Ts/39+hirXhVygtx6pFeCRVTdrfWc +OfVIVilZm7I+qDNrap2w6dT23zJT1DvZFCXymU9dxPEoLOm7cn9oxkvjs6c/YOIe +yU5CSdI5ZdO99G/8TbFwLKdSTajkRTVfIa99j/8P0jPMetnk7PoHABEBAAGJBGwE +GAEIACAWIQQ6nqEZOpe1SL4UV9SJGfa9K0jXVAUCYyw+bgIbAgJACRCJGfa9K0jX +VMF0IAQZAQgAHRYhBIheK9z5awtFq/BYRT5K1HGd3po4BQJjLD5uAAoJED5K1HGd +3po4CjIQALM1EjN2zCtQip9m5IuPnHA6d/YFIuNAXjAnZcg6HQSi+ERFu1w3mL2y +a+ll3EYOhFMVMEnja0h0e+ezt+WMJt1R2eH1PMZEeJ2uMXgIOl0QsTUXWemhweTQ +EmsgqksQMqsvFuogxNP/Co+cKJCiBpMIU9FJfLrSoGGidnUqr48QjGRVx69FPPth +lNOj6/KkhksmJFTT7YhMG8V7K8yxiYIcQQvRfINLZfmXF2cvGlZU21Il43nCmNb4 +iFUnrUOouQ7k/Oe/OzZqggmcfPGp+6GoSJB3exr3NonjuEhmYR1VPe+WcgsWSl5R +WVEaasN5C7dcekqWsqlAUHpvIqDfzaL3s4HBp19uxxdiy42wXotDhT02P7DJVjVJ +f9XWgQbmaanzLdPnrUxwUYcLEQM17VJ37j9DyW9zPBB/pKLRwDr89xpmAKMtRzeX +ImnBuUJHctp/c9dFvbhJGgxFgLHhBCUdTsEs7phLAB32n+oP2Mas6yRgG98k46ic ++ZNdJIzplfXT96n/5u+aeyQOkY3rZh6Jbhzwpb+r5Tj1rAXIJta6prkd8d3+1Rg3 +7x9HkEAHn3ZPDk1RwbVOVCU0UjMOPU/6Y3FsQq30jSuNhr70p5H/okwpkS7YX/7l +Y0068XEpWoIyJQIHcBM0HZCV904mwdqfRDujziIoQ/xEm3+zIVtzdgYP/3kX4kUu +p2DEUEY5ND3oKc5CAjyFUGU7thzILHtt76WrrwXqqfxjJKl9+acmLlrBLj5FQwFE +hThEeWunm2T5hhgr4FfF4MXneQqVRhUslm5+sRIm/K1ye5q4NLfgiwZD/nTM5def +yI4cynrby4fy1R9SCBLolzvPLpvEraegxIQ3Hk1kT5HmnXi79hZVbZxIkt6wNPG1 +rTS0TaJ2M37nmhm4GcNjDtKnhy4Aq569DO8Sd0U6bYKIaFFoi7X5KYnKWY37pYFl +0ERU0ffQp7Mg9ruHRvHhkA41iQ5kF/O1fJHRUfXcYnfIfvN1C77TdXLQryrA/bRN +S2dlulIu4CYH+lsfhlIjBQrRoEOWGu6PzKBwPIXKcVFfB4BcG5si/3vZD3Q1xfUj +3MV2sQcgVNbd/t7jJda3uhX9DiEPyqdgl382Rq+LoPV4az1igDLDUe7yPrCpD1rw +JMyKJyStiP4i1jiIejVavPiCpXH1oxCQeR8KqjdrdNbSVgDOapPauZEcwJL464Zl +WkdxKX09sN6GaXFTHYTEXkxrzhlYZaysWEO86+iBUcHnRtc1D3dMJg7hi/Du1Iou +TwnBsYIwd9Fz/wHpDkiu0Z3gCmYEg4JQj4iwAtNOQfhJzhEH4kHLom8j8XI/TlgJ +e/vtNUUUDp3i8qczTjuaVywRbE3+9xjjsSiI +=c48w +-----END PGP PUBLIC KEY BLOCK----- diff --git a/ansible/roles/clickhouse/handlers/main.yml b/ansible/roles/clickhouse/handlers/main.yml new file mode 100644 index 00000000..67409399 --- /dev/null +++ b/ansible/roles/clickhouse/handlers/main.yml @@ -0,0 +1,11 @@ +--- + +- name: Restart-clickhouse + systemd: + name: "{{ clickhouse_service }}" + state: restarted + +- name: Reload-clickhouse + systemd: + name: "{{ clickhouse_service }}" + state: reloaded diff --git a/ansible/roles/clickhouse/tasks/config/clickhouse.yml b/ansible/roles/clickhouse/tasks/config/clickhouse.yml new file mode 100644 index 00000000..2e6f4109 --- /dev/null +++ b/ansible/roles/clickhouse/tasks/config/clickhouse.yml @@ -0,0 +1,19 @@ +--- + +- name: CLICKHOUSE | Configure server + ansible.builtin.template: + src: "{{ clickhouse_config_file_name }}.j2" + dest: "{{ clickhouse_config_directory }}/{{ clickhouse_config_file_name }}" + mode: 0644 + notify: "{{ clickhouse_handler_on_config_change }}" + tags: clickhouse_configure + +- name: CLICKHOUSE | Configure users (from vars) + ansible.builtin.template: + src: "{{ clickhouse_users_file_name }}.j2" + dest: "{{ clickhouse_config_directory }}/{{ clickhouse_users_file_name }}" + mode: 0644 + notify: "{{ clickhouse_handler_on_config_change }}" + tags: + - clickhouse_configure + - clickhouse_users diff --git a/ansible/roles/clickhouse/tasks/config/grants.yml b/ansible/roles/clickhouse/tasks/config/grants.yml new file mode 100644 index 00000000..3a8af4ee --- /dev/null +++ b/ansible/roles/clickhouse/tasks/config/grants.yml @@ -0,0 +1,12 @@ +--- + +# Each grantee's privileges are replaced in one GRANT ... WITH REPLACE OPTION +# statement (templates/DDL/GRANTS.j2): privileges it keeps are never revoked, not +# even for a moment, and ones missing from vars go in the same statement. Reports changed +# on every run, as the statements are always sent. + +- name: CLICKHOUSE | Replace grants + ansible.builtin.command: + cmd: clickhouse-client --config-file {{ clickhouse_admin_client_config }} --multiquery + stdin: "{{ lookup('template', 'DDL/GRANTS.j2') }}" + tags: [clickhouse_grants] diff --git a/ansible/roles/clickhouse/tasks/config/quotas.yml b/ansible/roles/clickhouse/tasks/config/quotas.yml new file mode 100644 index 00000000..28011a34 --- /dev/null +++ b/ansible/roles/clickhouse/tasks/config/quotas.yml @@ -0,0 +1,33 @@ +--- + +# SQL-driven quotas (clickhouse_custom_quotas): created if missing, altered if they +# already exist. Without `to`, ALTER QUOTA leaves the quota's assignment as it is. + +- name: CLICKHOUSE | List SQL quotas + ansible.builtin.command: + cmd: >- + clickhouse-client --config-file {{ clickhouse_admin_client_config }} + --query "SELECT name FROM system.quotas WHERE storage = 'local_directory'" + register: clickhouse_existing_quotas + changed_when: false + check_mode: false # read-only, so that --check can show what would change + tags: [clickhouse_quotas] + +- name: CLICKHOUSE | Drop SQL quotas missing from vars + ansible.builtin.command: + cmd: >- + clickhouse-client --config-file {{ clickhouse_admin_client_config }} + --query "DROP QUOTA {{ item }}" + loop: "{{ clickhouse_existing_quotas.stdout_lines | difference(clickhouse_custom_quotas | map(attribute='name')) }}" + when: clickhouse_sql_objects_drop_unlisted + tags: [clickhouse_quotas] + +# Reports changed on every run, as the statements are always sent; they only change +# the server when the vars have changed. +- name: CLICKHOUSE | Create or update SQL quotas + ansible.builtin.command: + cmd: clickhouse-client --config-file {{ clickhouse_admin_client_config }} --multiquery + stdin: "{{ lookup('template', 'DDL/QUOTAS.j2') }}" + vars: + existing: "{{ clickhouse_existing_quotas.stdout_lines }}" + tags: [clickhouse_quotas] diff --git a/ansible/roles/clickhouse/tasks/config/users.yml b/ansible/roles/clickhouse/tasks/config/users.yml new file mode 100644 index 00000000..6893f504 --- /dev/null +++ b/ansible/roles/clickhouse/tasks/config/users.yml @@ -0,0 +1,34 @@ +--- + +# SQL-driven users (clickhouse_custom_users): created if missing, altered if they +# already exist, so that changes in vars always apply. ALTER keeps a user's grants +# and quota assignments. + +- name: CLICKHOUSE | List SQL users + ansible.builtin.command: + cmd: >- + clickhouse-client --config-file {{ clickhouse_admin_client_config }} + --query "SELECT name FROM system.users WHERE storage = 'local_directory'" + register: clickhouse_existing_users + changed_when: false + check_mode: false # read-only, so that --check can show what would change + tags: [clickhouse_users] + +- name: CLICKHOUSE | Drop SQL users missing from vars + ansible.builtin.command: + cmd: >- + clickhouse-client --config-file {{ clickhouse_admin_client_config }} + --query "DROP USER {{ item }}" + loop: "{{ clickhouse_existing_users.stdout_lines | difference(clickhouse_custom_users | map(attribute='name')) }}" + when: clickhouse_sql_objects_drop_unlisted + tags: [clickhouse_users] + +# Reports changed on every run, as the statements are always sent; they only change +# the server when the vars have changed. +- name: CLICKHOUSE | Create or update SQL users + ansible.builtin.command: + cmd: clickhouse-client --config-file {{ clickhouse_admin_client_config }} --multiquery + stdin: "{{ lookup('template', 'DDL/USERS.j2') }}" + vars: + existing: "{{ clickhouse_existing_users.stdout_lines }}" + tags: [clickhouse_users] diff --git a/ansible/roles/clickhouse/tasks/install-Debian.yml b/ansible/roles/clickhouse/tasks/install-Debian.yml new file mode 100644 index 00000000..95cdd1b2 --- /dev/null +++ b/ansible/roles/clickhouse/tasks/install-Debian.yml @@ -0,0 +1,143 @@ +--- + +- name: CLICKHOUSE | Ensure clickhouse group + ansible.builtin.group: + name: "{{ clickhouse_group }}" + system: yes + state: present + tags: [clickhouse_install] + +- name: CLICKHOUSE | Ensure clickhouse user + ansible.builtin.user: + name: "{{ clickhouse_user }}" + group: "{{ clickhouse_group }}" + system: yes + shell: /usr/sbin/nologin + createhome: no + tags: [clickhouse_install] + +- name: CLICKHOUSE | Ensure base directory exist + ansible.builtin.file: + state: directory + dest: "{{ clickhouse_base_path }}" + owner: "{{ clickhouse_user }}" + group: "{{ clickhouse_group }}" + mode: "{{ clickhouse_base_directory_mode }}" + become: true + tags: [clickhouse_install] + +- name: CLICKHOUSE | Update repositories cache and install apt-transport-https, dirmngr ansible.builtin.packages + apt: + name: "{{ clickhouse_prerequisite_repos }}" + state: present + tags: [clickhouse_install] + +- name: CLICKHOUSE | Ensure keyrings directory exists + ansible.builtin.file: + path: "{{ clickhouse_deb_keyring_path | dirname }}" + state: directory + mode: "0755" + tags: [clickhouse_install] + +- name: CLICKHOUSE | Install pinned apt repository key + ansible.builtin.copy: + src: clickhouse-repo-keyring.asc + dest: "{{ clickhouse_deb_keyring_path }}" + mode: "0644" + tags: [clickhouse_install] + +# The role used to add this repository without signed-by, in a file named after its URL. +# apt refuses the same repository with two different keys, so remove that entry first. +- name: CLICKHOUSE | Remove the apt repository entry without signed-by + ansible.builtin.apt_repository: + repo: "deb https://packages.clickhouse.com/deb {{ clickhouse_release_type }} main" + state: absent + update_cache: false + tags: [clickhouse_install] + +- name: CLICKHOUSE | Add clickhouse apt repository (deb) + ansible.builtin.apt_repository: + repo: "{{ clickhouse_deb_repo }}" + state: present + filename: clickhouse + tags: [clickhouse_install] + +# Without a pin, `apt upgrade` (or unattended-upgrades) moves ClickHouse to the +# newest release in the repo. To upgrade or downgrade, change clickhouse_version +# and rerun the role. +- name: CLICKHOUSE | Pin clickhouse packages to clickhouse_version + ansible.builtin.copy: + dest: /etc/apt/preferences.d/clickhouse + content: | + # Managed by ansible (clickhouse role): pins ClickHouse to clickhouse_version + Package: clickhouse-* + Pin: version {{ clickhouse_version }} + Pin-Priority: 1000 + mode: "0644" + tags: [clickhouse_install] + +# A package upgrade doesn't restart the running server: restart it, or it keeps running +# the old version until something else restarts it. +- name: CLICKHOUSE | Be sure clickhouse packages are installed (apt) + ansible.builtin.apt: + name: "{{ clickhouse_packages }}" + state: present + update_cache: true + allow_downgrade: true + notify: Restart-clickhouse + tags: [clickhouse_install] + +- name: CLICKHOUSE | Ensure user files and access control directories exist + ansible.builtin.file: + state: directory + dest: "{{ item }}" + owner: "{{ clickhouse_user }}" + group: "{{ clickhouse_group }}" + mode: 0750 + with_items: + - "{{ clickhouse_access_control_directory }}" + - "{{ clickhouse_user_files_directory }}" + become: true + tags: [clickhouse_install] + +- name: CLICKHOUSE | Ensure needed config directories exist + ansible.builtin.file: + state: directory + dest: "{{ item }}" + owner: "{{ clickhouse_user }}" + group: "{{ clickhouse_group }}" + mode: "{{ clickhouse_config_directory_mode }}" + with_items: + - "{{ clickhouse_config_directory }}" + - "{{ clickhouse_config_directory_config_d }}" + - "{{ clickhouse_config_directory_users_d }}" + become: true + tags: [clickhouse_install] + +- name: CLICKHOUSE | Ensure needed tmp directories exist + ansible.builtin.file: + state: directory + dest: "{{ clickhouse_tmp_directory }}" + owner: "{{ clickhouse_user }}" + group: "{{ clickhouse_group }}" + mode: "{{ clickhouse_tmp_directory_mode }}" + become: true + tags: [clickhouse_install] + +- name: CLICKHOUSE | Ensure needed log directory exist + ansible.builtin.file: + state: directory + dest: "{{ clickhouse_log_directory }}" + owner: "{{ clickhouse_user }}" + group: "{{ clickhouse_group }}" + mode: "{{ clickhouse_log_directory_mode }}" + become: true + tags: [clickhouse_install] + +- name: CLICKHOUSE | Clickhouse extra packages are installed (apt) + ansible.builtin.apt: + name: "{{ clickhouse_packages_extra }}" + state: present + update_cache: true + when: clickhouse_packages_extra is defined + tags: [clickhouse_install] diff --git a/ansible/roles/clickhouse/tasks/main.yml b/ansible/roles/clickhouse/tasks/main.yml new file mode 100644 index 00000000..8c365be3 --- /dev/null +++ b/ansible/roles/clickhouse/tasks/main.yml @@ -0,0 +1,53 @@ +--- + +- name: CLICKHOUSE | Install + ansible.builtin.include_tasks: "install-{{ ansible_os_family | capitalize }}.yml" + tags: + - clickhouse_install + +- name: CLICKHOUSE | Configure clickhouse + ansible.builtin.include_tasks: config/clickhouse.yml + tags: + - clickhouse_configure + +# clickhouse-client runs as the admin user, for the checks after a restart and the +# SQL-driven users, quotas and grants, reading its credentials from this file, so +# that no password is on a command line or in task output. +- name: CLICKHOUSE | Admin client configuration + ansible.builtin.template: + src: admin-client.xml.j2 + dest: "{{ clickhouse_admin_client_config }}" + owner: root + group: root + mode: "0600" + no_log: true + diff: false + check_mode: false # the checks below read it, in --check too + tags: + - clickhouse_service + - clickhouse_quotas + - clickhouse_users + - clickhouse_grants + +- name: CLICKHOUSE | ClickHouse Service + ansible.builtin.include_tasks: service/clickhouse.yml + tags: + - clickhouse_service + +- name: CLICKHOUSE | Configure users + ansible.builtin.include_tasks: config/users.yml + tags: + - clickhouse_users + when: clickhouse_role_manage_users + +- name: CLICKHOUSE | Configure quotas + ansible.builtin.include_tasks: config/quotas.yml + tags: + - clickhouse_quotas + when: clickhouse_role_manage_quotas + +- name: CLICKHOUSE | Configure grants + ansible.builtin.include_tasks: config/grants.yml + tags: + - clickhouse_grants + when: clickhouse_role_manage_grants diff --git a/ansible/roles/clickhouse/tasks/service/clickhouse.yml b/ansible/roles/clickhouse/tasks/service/clickhouse.yml new file mode 100644 index 00000000..7d7c86d7 --- /dev/null +++ b/ansible/roles/clickhouse/tasks/service/clickhouse.yml @@ -0,0 +1,66 @@ +--- + +- name: CLICKHOUSE | Copy systemd unit configuration file + ansible.builtin.template: + src: "{{ clickhouse_service_template_path }}" + dest: "/etc/systemd/system/{{ clickhouse_service }}.service" + mode: 0644 + owner: root + group: root + notify: Restart-clickhouse # a changed unit only applies on a restart + +- name: CLICKHOUSE | Configuring service + ansible.builtin.systemd: + name: "{{ clickhouse_service }}" + state: "{{ clickhouse_service_state }}" + enabled: "{{ clickhouse_service_enabled }}" + daemon_reload: yes + +# Restart now (after daemon-reload) for new packages, config or unit, not at the end of +# the play, so that the checks below see the restarted server before the next host. +- name: CLICKHOUSE | Restart for changed packages, config or unit + ansible.builtin.meta: flush_handlers + +- name: CLICKHOUSE | Wait for service listening + ansible.builtin.wait_for: + host: localhost + port: "{{ clickhouse_http_port }}" + state: "{{ clickhouse_service_state }}" + delay: 5 + timeout: 300 + when: + - clickhouse_service_enabled + - clickhouse_service_state == "started" + +- name: CLICKHOUSE | Check that the server runs clickhouse_version + ansible.builtin.command: + cmd: clickhouse-client --config-file {{ clickhouse_admin_client_config }} --query "SELECT version()" + register: clickhouse_running_version + until: clickhouse_running_version.rc == 0 + retries: 60 + delay: 5 + changed_when: false + check_mode: false # read-only + failed_when: clickhouse_running_version.rc != 0 or clickhouse_running_version.stdout != clickhouse_version + when: + - clickhouse_service_enabled + - clickhouse_service_state == "started" + - not ansible_check_mode # the new version only runs after a real restart + +# After a restart, replicated tables are read-only until the server is back in Keeper: +# wait for that, so that the next host isn't restarted while this one is catching up. +# A new host has no replicated tables yet, and passes at once. +- name: CLICKHOUSE | Wait until no replicated table is read-only + ansible.builtin.command: + cmd: >- + clickhouse-client --config-file {{ clickhouse_admin_client_config }} + --query "SELECT count() FROM system.replicas WHERE is_readonly OR is_session_expired" + register: clickhouse_readonly_replicas + until: clickhouse_readonly_replicas.rc == 0 and clickhouse_readonly_replicas.stdout == "0" + retries: 60 + delay: 5 + changed_when: false + check_mode: false # read-only + when: + - clickhouse_service_enabled + - clickhouse_service_state == "started" diff --git a/ansible/roles/clickhouse/templates/DDL/GRANTS.j2 b/ansible/roles/clickhouse/templates/DDL/GRANTS.j2 new file mode 100644 index 00000000..c50bf42f --- /dev/null +++ b/ansible/roles/clickhouse/templates/DDL/GRANTS.j2 @@ -0,0 +1,16 @@ +{# One statement per grantee, with every privilege the clickhouse_custom_grants + entries give it ("PRIVILEGE ON database.table"), replacing all it had before. #} +{% set grantees = clickhouse_custom_grants | map(attribute='to') | flatten | unique %} +{% for grantee in grantees %} +{% set privileges = [] %} +{% for g in clickhouse_custom_grants if grantee in g.to %} +{% for database in g.databases %}{% for table in g.tables %}{% for privilege in g.privileges %} +{% set _ = privileges.append(privilege ~ ' ON ' ~ database ~ '.' ~ table) %} +{% endfor %}{% endfor %}{% endfor %} +{% endfor %} +GRANT {{ privileges | unique | join(', ') }} TO {{ grantee }} WITH REPLACE OPTION; +{% endfor %} +{# Users in vars that no entry grants anything to should have no privileges. #} +{% for user in clickhouse_custom_users | map(attribute='name') | reject('in', grantees) %} +REVOKE ALL ON *.* FROM {{ user }}; +{% endfor %} diff --git a/ansible/roles/clickhouse/templates/DDL/QUOTAS.j2 b/ansible/roles/clickhouse/templates/DDL/QUOTAS.j2 new file mode 100644 index 00000000..8c518816 --- /dev/null +++ b/ansible/roles/clickhouse/templates/DDL/QUOTAS.j2 @@ -0,0 +1,5 @@ +{# For each quota: CREATE if missing, otherwise ALTER. Without `intervals` or `to`, + ALTER leaves the quota's limits or assignment as they are. #} +{% for quota in clickhouse_custom_quotas %} +{{ 'ALTER' if quota.name in existing else 'CREATE' }} QUOTA {{ quota.name }}{% if quota.intervals is defined %} FOR {{ quota.intervals | join(', ') }}{% endif %}{% if quota.to is defined %} TO {{ quota.to | join(', ') }}{% endif %}; +{% endfor %} diff --git a/ansible/roles/clickhouse/templates/DDL/USERS.j2 b/ansible/roles/clickhouse/templates/DDL/USERS.j2 new file mode 100644 index 00000000..f402fa44 --- /dev/null +++ b/ansible/roles/clickhouse/templates/DDL/USERS.j2 @@ -0,0 +1,7 @@ +{# For each user: CREATE if missing, otherwise ALTER, which keeps its grants and quota + assignments. The password is set in a statement of its own, so that a mistake in the + other clauses fails without ClickHouse quoting the password hash in its error. #} +{% for user in clickhouse_custom_users %} +{{ 'ALTER' if user.name in existing else 'CREATE' }} USER {{ user.name }} IDENTIFIED WITH {{ user.password_type }} BY '{{ user.password }}'; +ALTER USER {{ user.name }} HOST {{ user.networks | join(', ') if user.networks is defined else 'ANY' }} SETTINGS {{ user.settings | join(', ') if user.settings is defined else 'NONE' }}; +{% endfor %} diff --git a/ansible/roles/clickhouse/templates/admin-client.xml.j2 b/ansible/roles/clickhouse/templates/admin-client.xml.j2 new file mode 100644 index 00000000..ccd02b6d --- /dev/null +++ b/ansible/roles/clickhouse/templates/admin-client.xml.j2 @@ -0,0 +1,5 @@ + + + {{ clickhouse_admin_user }} + {{ clickhouse_admin_password }} + diff --git a/ansible/roles/clickhouse/templates/clickhouse-server.service.j2 b/ansible/roles/clickhouse/templates/clickhouse-server.service.j2 new file mode 100644 index 00000000..7d366bed --- /dev/null +++ b/ansible/roles/clickhouse/templates/clickhouse-server.service.j2 @@ -0,0 +1,32 @@ +{{ ansible_managed | comment }} +[Unit] +Description=ClickHouse Server (analytic DBMS for big data) +Documentation= {{ clickhouse_documentation_link }} +Requires=network-online.target +# NOTE: that After/Wants=time-sync.target is not enough, you need to ensure +# that the time was adjusted already, if you use systemd-timesyncd you are +# safe, but if you use ntp or some other daemon, you should configure it +# additionaly. +After=time-sync.target network-online.target +Wants=time-sync.target + +[Service] +Type=simple +User={{ clickhouse_user }} +Group={{ clickhouse_group }} +Restart=always +RestartSec=30 +PrivateTmp= {{ clickhouse_private_tmp }} +RuntimeDirectory={{ clickhouse_service }} +ExecStart=/usr/bin/{{ clickhouse_service }} --config={{ clickhouse_config_directory }}/{{ clickhouse_config_file_name }} --pid-file=/run/clickhouse-server/{{ clickhouse_service }}.pid +ExecReload=/bin/kill -HUP $MAINPID +EnvironmentFile=-{{ clickhouse_environment_file }} +LimitCORE=infinity +LimitNOFILE={{ clickhouse_limit_no_file }} +CapabilityBoundingSet=CAP_NET_ADMIN CAP_IPC_LOCK CAP_SYS_NICE CAP_NET_BIND_SERVICE + +StandardOutput={{ clickhouse_log_output }} +StandardError={{ clickhouse_log_output }} + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/clickhouse/templates/config.xml.j2 b/ansible/roles/clickhouse/templates/config.xml.j2 new file mode 100644 index 00000000..ed6f950b --- /dev/null +++ b/ansible/roles/clickhouse/templates/config.xml.j2 @@ -0,0 +1,1605 @@ + + + + + + {{ clickhouse_logger.level }} + {{ clickhouse_logger.log }} + {{ clickhouse_logger.errorlog }} + + {{ clickhouse_logger.size }} + {{ clickhouse_logger.count }} + {% if clickhouse_logger.console is defined and clickhouse_logger.console != 'autodetection' -%} + {{ clickhouse_logger.console }} + {% else -%} + + {% endif -%} + + + + + {% if clickhouse_logger_overrides_legacy is defined %}{{ clickhouse_logger_overrides_legacy }}{% endif %} + + + + {% if clickhouse_logger_overrides is defined %}{{ clickhouse_logger_overrides }}{% endif %} + + + + + + {% if clickhouse_custom_headers_response is defined %}{{ clickhouse_custom_headers_response }}{% endif %} + + + + + + {{ clickhouse_http_port }} + + + {{ clickhouse_tcp_port }} + + + {{ clickhouse_mysql_port }} + + + {{ clickhouse_postgresql_port }} + + + + {% if clickhouse_https_port is defined -%} + {{ clickhouse_https_port }} + {% endif -%} + + + + {% if clickhouse_tcp_port_secure is defined -%} + {{ clickhouse_tcp_port_secure }} + {% endif -%} + + + + {% if clickhouse_tcp_with_proxy_port is defined -%} + {{ clickhouse_tcp_with_proxy_port }} + {% endif -%} + + + {{ clickhouse_interserver_http_port }} + + + + {% if clickhouse_interserver_https_port is defined -%} + {{ clickhouse_interserver_https_port }} + {% endif -%} + + + + {% if clickhouse_interserver_http_host is defined -%} + {{ clickhouse_interserver_http_host }} + {% endif -%} + + + + {% if clikhouse_interserver_http_credentials is defined -%} + + {{ clikhouse_interserver_http_credentials.user }} + {{ clikhouse_interserver_http_credentials.password }} + + {% endif -%} + + + + + + + + + + {% if clickhouse_listen_hosts is defined -%} + {% for host in clickhouse_listen_hosts -%} + {{ host }} + {% endfor -%} + {% endif -%} + + + + {% if clickhouse_listen_try is defined -%} + {{ clickhouse_listen_try }} + {% endif -%} + + + + {% if clickhouse_listen_reuse_port is defined -%} + {{ clickhouse_listen_reuse_port }} + {% endif -%} + + + {% if clickhouse_listen_backlog is defined -%} + {{ clickhouse_listen_backlog }} + {% endif -%} + + {{ clickhouse_config.max_connections }} + + + {{ clickhouse_config.keep_alive_timeout }} + + + + {% if clickhouse_grpc_port is defined -%} + {{ clickhouse_grpc_port }} + {% endif -%} + + {% if clickhouse_grpc_port is defined -%} + {{ clickhouse_grpc_port }} + {% endif -%} + {{ clickhouse_enable_ssl }} + + + {% if clickhouse_enable_ssl is true -%} + {{ clickhouse_ssl_cert_file }} + {{ clickhouse_ssl_key_file }} + {% endif -%} + + + {{ clickhouse_ssl_require_client_auth }} + + + {{ clickhouse_ssl_ca_cert_file }} + + + {{ clickhouse_grpc_compression }} + + + {{ clickhouse_grpc_compression_level }} + + + {{ clickhouse_max_send_message_size }} + {{ clickhouse_max_receive_message_size }} + + + {{ clickhouse_verbose_logs }} + + + + + + + {{ clickhouse_certificatefile }} + {{ clickhouse_privatekeyfile }} + + {{ clickhouse_dhparamsfile }} + {{ clickhouse_verificationmode }} + {{ clickhouse_loaddefaultcafile_server }} + {{ clickhouse_cachesessions_server }} + {{ clickhouse_disableprotocols_server }} + {{ clickhouse_preferserverciphers_server }} + + + + {{ clickhouse_loaddefaultcafile_client }} + {{ clickhouse_cachesessions_client }} + {{ clickhouse_disableprotocols_client }} + {{ clickhouse_preferserverciphers_client }} + + + + {{ clickhouse_invalidcertificatehandler.name }} + + + + + + + {% if clickhouse_http_server_default_response is defined -%} + {{ clickhouse_http_server_default_response }} + {% endif -%} + + + {{ clickhouse_config.max_concurrent_queries }} + + + {{ clickhouse_config.max_server_memory_usage }} + + + {{ clickhouse_config.max_thread_pool_size }} + + + + + + {{ clickhouse_config.max_server_memory_usage_to_ram_ratio }} + + + {{ clickhouse_config.total_memory_profiler_step }} + + + {{ clickhouse_config.total_memory_tracker_sample_probability }} + + + + {% if clickhouse_config.max_open_files is defined -%} + {{ clickhouse_config.max_open_files }} + {% endif -%} + + + {{ clickhouse_config.uncompressed_cache_size }} + + + {{ clickhouse_config.mark_cache_size }} + + + + {{ clickhouse_config.mmap_cache_size }} + + + {{ clickhouse_config.compiled_expression_cache_size }} + + + {{ clickhouse_config.compiled_expression_cache_elements_size }} + + + {{ clickhouse_base_path }} + + + {{ clickhouse_tmp_directory }} + + + + {% if clickhouse_config.tmp_policy is defined -%} + {{ clickhouse_config.tmp_policy }} + {% endif -%} + + + {{ clickhouse_user_files_directory }} + + + + + {% if clickhouse_ldap is defined -%} + {% for ldap_server in clickhouse_ldap %} + <{{ ldap_server.servername }}> + {% if ldap_server.host is defined -%}{{ ldap_server.host }}{% endif -%} + {% if ldap_server.port is defined -%}{{ ldap_server.port }}{% endif -%} + {% if ldap_server.bind_dn is defined -%}{{ ldap_server.bind_dn }}{% endif -%} + {% if ldap_server.base_dn is defined or ldap_server.scope or ldap_server.search_filter -%} + {% if clickhouse_auth_backend == 'AD' -%}{% endif -%} + {% if ldap_server.base_dn is defined -%}{{ ldap_server.base_dn }}{% endif -%} + {% if ldap_server.scope is defined -%}{{ ldap_server.scope }}{% endif -%} + {% if ldap_server.search_filter is defined -%}{{ ldap_server.search_filter }}{% endif -%} + {% if clickhouse_auth_backend == 'AD' -%}{% endif -%} + {% endif -%} + {% if ldap_server.verification_cooldown is defined -%}{{ ldap_server.verification_cooldown }}{% endif -%} + {% if ldap_server.enable_tls is defined -%}{{ ldap_server.enable_tls }}{% endif -%} + {% if ldap_server.tls_minimum_protocol_version is defined -%}{{ ldap_server.tls_minimum_protocol_version }}{% endif -%} + {% if ldap_server.tls_require_cert is defined -%}{{ ldap_server.tls_require_cert }}{% endif -%} + {% if ldap_server.tls_cert_file is defined -%}{{ ldap_server.tls_cert_file }}{% endif -%} + {% if ldap_server.tls_key_file is defined -%}{{ ldap_server.tls_key_file }}{% endif -%} + {% if ldap_server.tls_ca_cert_file is defined -%}{{ ldap_server.tls_ca_cert_file }}{% endif -%} + {% if ldap_server.tls_ca_cert_dir is defined -%}{{ ldap_server.tls_ca_cert_dir }}{% endif -%} + {% if ldap_server.tls_cipher_suite is defined -%}{{ ldap_server.tls_cipher_suite }}{% endif -%} + + {% endfor %} + {% endif -%} + + + + {% if clickhouse_kerberos is defined -%} + + {% if clickhouse_kerberos.principal is defined%}{{ clickhouse_kerberos.principal }}{% endif -%} + {% if clickhouse_kerberos.realm is defined%}{{ clickhouse_kerberos.realm }}{% endif -%} + + {% endif -%} + + + + + + {{ clickhouse_users_file_name }} + + + + {{ clickhouse_access_control_directory }} + + + + {% if clickhouse_user_directories_ldap is defined -%} + {% for user_directory in clickhouse_user_directories_ldap -%} + + {{ user_directory.server }} + {% if user_directory.roles is defined -%} + + {% for role in user_directory.roles -%} + <{{ role }} /> + {% endfor -%} + + {% endif -%} + {% if user_directory.role_mappings is defined -%} + {% for role_mapping in user_directory.role_mappings -%} + + {{ role_mapping.base_dn }} + {{ role_mapping.scope }} + {{ role_mapping.search_filter }} + {{ role_mapping.attribute }} + {{ role_mapping.prefix }} + + {% endfor -%} + {% endif -%} + + {% endfor -%} + {% endif -%} + + + + {{ clickhouse_config.default_profile }} + + + {{ clickhouse_config.custom_settings_prefixes }} + + + + {% if clickhouse_config.system_profile is defined -%} + {{ clickhouse_config.system_profile }} + {% endif -%} + + + + {% if clickhouse_config.buffer_profile is defined -%} + {{ clickhouse_config.buffer_profile }} + {% endif -%} + + + {{ clickhouse_config.default_database }} + + + + {% if clickhouse_config.timezone is defined -%} + {{ clickhouse_config.timezone }} + {% endif -%} + + + + {% if clickhouse_config.umask is defined -%} + {{ clickhouse_config.umask }} + {% endif -%} + + {{ clickhouse_config.mlock_executable }} + + + {{ clickhouse_config.remap_executable }} + + + {% if clickhouse_jdbc_bridge is defined -%} + + {{ clickhouse_jdbc_bridge.host }} + {{ clickhouse_jdbc_bridge.port }} + + {% endif -%} + + +{% if clickhouse_remote_servers is defined %} + +{% for server in clickhouse_remote_servers %} + <{{ server.servername }}> +{% if server.secret is defined %} + {{ server.secret }} +{% endif %} +{% for shard in server.shards %} + +{% if shard.internal_replication is defined %} + {{ shard.internal_replication }} +{% endif %} +{% if shard.weight is defined %} + {{ shard.weight }} +{% endif %} +{% for replica in shard.replicas %} + + {{ replica.host }} + {{ replica.port }} +{% if replica.user is defined %} + {{ replica.user }} +{% endif %} +{% if replica.password is defined %} + {{ replica.password }} +{% endif %} +{% if replica.priority is defined %} + {{ replica.priority }} +{% endif %} +{% if replica.secure is defined %} + {{ replica.secure }} +{% endif %} +{% if replica.compression is defined %} + {{ replica.compression }} +{% endif %} + +{% endfor %} + +{% endfor %} + +{% endfor %} + +{% endif %} + + + + + + + +{% if clickhouse_remote_url_allow_hosts is defined %} + +{% for host in clickhouse_remote_url_allow_hosts %} + {{ host }} +{% endfor %} + +{% endif %} + + + + +{% if clickhouse_keeper is defined %} +{% for keeper_server in clickhouse_keeper.keeper_servers %} +{% if keeper_server.server is defined and inventory_hostname == keeper_server.server %} + + + {{ clickhouse_keeper.tcp_port }} + {{ keeper_server.id }} + {{ clickhouse_keeper.log_storage_path }} + {{ clickhouse_keeper.snapshot_storage_path }} + + + {{ clickhouse_keeper.coordination_settings.operation_timeout_ms }} + {{ clickhouse_keeper.coordination_settings.session_timeout_ms }} + {{ clickhouse_keeper.coordination_settings.raft_logs_level }} + + + +{% for keeper_server in clickhouse_keeper.keeper_servers %} + + {{ keeper_server.id }} + {{ keeper_server.hostname }} + {{ keeper_server.port }} + +{% endfor %} + + +{% endif %} +{% endfor %} +{% endif %} + + + + +{% if clickhouse_zookeeper is defined %} + +{% for node in clickhouse_zookeeper %} + + {{ node.host }} + {{ node.port }} + +{% endfor %} + +{% endif %} + + + + +{% if clickhouse_macros is defined %} + +{% for macros in clickhouse_macros %} +{% if macros.server is defined and inventory_hostname in macros.server %}{{ macros.macro|indent(4, true) }}{% endif %} +{% if macros.server is not defined %}{{ macros.macro|indent(4, true) }}{% endif %} +{% endfor %} + +{# Deprecated, please use clickhouse_macros instead of clickhouse_replicated_tables_macros #} +{% elif clickhouse_replicated_tables_macros is defined %} + +{% for macros in clickhouse_replicated_tables_macros %} +{% if macros.server == inventory_hostname %}{{ macros.macro|indent(4, true) }}{% endif %} +{% endfor %} + +{% endif %} + + + {{ clickhouse_config.builtin_dictionaries_reload_interval }} + + + {{ clickhouse_config.max_session_timeout }} + + + {{ clickhouse_config.default_session_timeout }} + + + + +{% if clickhouse_graphite is defined %} +{% for graphite in clickhouse_graphite %} + + {{ graphite.host }} + {{ graphite.port }} + {{ graphite.timeout }} +{% if graphite.interval is defined %} + {{ graphite.interval }} +{% endif %} +{% if graphite.root_path is defined %} + {{ graphite.root_path }} +{% endif %} +{% if graphite.hostname_in_path is defined %} + {{ graphite.hostname_in_path }} +{% endif %} +{% if graphite.metrics is defined %} + {{ graphite.metrics }} +{% endif %} +{% if graphite.events is defined %} + {{ graphite.events }} +{% endif %} +{% if graphite.events_cumulative is defined %} + {{ graphite.events_cumulative }} +{% endif %} +{% if graphite.asynchronous_metrics is defined %} + {{ graphite.asynchronous_metrics }} +{% endif %} + +{% endfor %} +{% endif %} + + + + +{% if clickhouse_prometheus is defined %} + + {{ clickhouse_prometheus.endpoint }} + {{ clickhouse_prometheus.port }} + {{ clickhouse_prometheus.metrics }} + {{ clickhouse_prometheus.events }} + {{ clickhouse_prometheus.asynchronous_metrics }} + {{ clickhouse_prometheus.status_info }} + +{% endif %} + + + + + {{ clickhouse_tables_log.query_log.database }} + {{ clickhouse_tables_log.query_log.table }}
+ +{% if clickhouse_tables_log.query_log.partition_by is defined %} + {{ clickhouse_tables_log.query_log.partition_by }} +{% endif %} + +{% if clickhouse_tables_log.query_log.ttl is defined %} + {{ clickhouse_tables_log.query_log.ttl }} +{% endif %} + + +{% if clickhouse_tables_log.query_log.engine is defined %} + {{ clickhouse_tables_log.query_log.engine }} +{% endif %} + + + {{ clickhouse_tables_log.query_log.flush_interval_milliseconds }} + + + + + {{ clickhouse_tables_log.trace_log.database }} + {{ clickhouse_tables_log.trace_log.table }}
+ {{ clickhouse_tables_log.trace_log.partition_by }} + {{ clickhouse_tables_log.trace_log.flush_interval_milliseconds }} +{% if clickhouse_tables_log.trace_log.ttl is defined %} + {{ clickhouse_tables_log.trace_log.ttl }} +{% endif %} + + + + + {{ clickhouse_tables_log.thread_log.database }} + {{ clickhouse_tables_log.thread_log.table }}
+ {{ clickhouse_tables_log.thread_log.partition_by }} + {{ clickhouse_tables_log.thread_log.flush_interval_milliseconds }} +{% if clickhouse_tables_log.thread_log.ttl is defined %} + {{ clickhouse_tables_log.thread_log.ttl }} +{% endif %} + + + + + {{ clickhouse_tables_log.views_log.database }} + {{ clickhouse_tables_log.views_log.table }}
+ {{ clickhouse_tables_log.views_log.partition_by }} + {{ clickhouse_tables_log.views_log.flush_interval_milliseconds }} +{% if clickhouse_tables_log.views_log.ttl is defined %} + {{ clickhouse_tables_log.views_log.ttl }} +{% endif %} + + + + + {{ clickhouse_tables_log.part_log.database }} + {{ clickhouse_tables_log.part_log.table }}
+ {{ clickhouse_tables_log.part_log.partition_by }} + {{ clickhouse_tables_log.part_log.flush_interval_milliseconds }} +{% if clickhouse_tables_log.part_log.ttl is defined %} + {{ clickhouse_tables_log.part_log.ttl }} +{% endif %} + + + +{% if clickhouse_tables_log.text_log is defined %} + + {{ clickhouse_tables_log.text_log.database }} + {{ clickhouse_tables_log.text_log.table }}
+ {{ clickhouse_tables_log.text_log.flush_interval_milliseconds }} +{% if clickhouse_tables_log.text_log.ttl is defined %} + {{ clickhouse_tables_log.text_log.ttl }} +{% endif %} +{% if clickhouse_tables_log.text_log.level is defined %} + clickhouse_tables_log.text_log.level +{% endif %} + +{% endif %} + + + + {{ clickhouse_tables_log.metric_log.database }} + {{ clickhouse_tables_log.metric_log.table }}
+ {{ clickhouse_tables_log.metric_log.flush_interval_milliseconds }} +{% if clickhouse_tables_log.metric_log.ttl is defined %} + {{ clickhouse_tables_log.metric_log.ttl }} +{% endif %} + {{ clickhouse_tables_log.metric_log.collect_interval_milliseconds }} + + + + + {{ clickhouse_tables_log.asynchronous_metric_log.database }} + {{ clickhouse_tables_log.asynchronous_metric_log.table }}
+ + {{ clickhouse_tables_log.asynchronous_metric_log.flush_interval_milliseconds }} +{% if clickhouse_tables_log.asynchronous_metric_log.ttl is defined %} + {{ clickhouse_tables_log.asynchronous_metric_log.ttl }} +{% endif %} + + + + + + + {{ clickhouse_tables_log.opentelemetry_span_log.engine }} + + {{ clickhouse_tables_log.opentelemetry_span_log.database }} + {{ clickhouse_tables_log.opentelemetry_span_log.table }}
+ {{ clickhouse_tables_log.opentelemetry_span_log.flush_interval_milliseconds }} + + + + + + {{ clickhouse_tables_log.crash_log.database }} + {{ clickhouse_tables_log.crash_log.table }}
+ {{ clickhouse_tables_log.crash_log.partition_by }} + {{ clickhouse_tables_log.crash_log.flush_interval_milliseconds }} +{% if clickhouse_tables_log.crash_log.ttl is defined %} + {{ clickhouse_tables_log.crash_log.ttl }} +{% endif %} + + + + + {{ clickhouse_tables_log.session_log.database }} + {{ clickhouse_tables_log.session_log.table }}
+ {{ clickhouse_tables_log.session_log.partition_by }} + {{ clickhouse_tables_log.session_log.flush_interval_milliseconds }} +{% if clickhouse_tables_log.session_log.ttl is defined %} + {{ clickhouse_tables_log.session_log.ttl }} +{% endif %} + + + + + + +{% if clickhouse_config.regions_hierarchy_file is defined %} + {{ clickhouse_config.regions_hierarchy_file }} +{% endif %} + + + +{% if clickhouse_config.regions_names_files_path is defined %} + {{ clickhouse_config.regions_names_files_path }} +{% endif %} + + +{% if clickhouse_config.top_level_domains_path is defined %} + {{ clickhouse_config.top_level_domains_path }} +{% endif %} + + + +{% if clickhouse_config.top_level_domains is defined %} +{% for domain in clickhouse_config.top_level_domains %} + <{{ domain.name }}>{{ domain.path }} +{% endfor %} +{% endif %} + + + + {{ clickhouse_config.dictionaries_config }} + + + {{ clickhouse_config.user_defined_executable_functions_config }} + + + +{% if clickhouse_compression is defined %} + +{% for case in clickhouse_compression %} + +{% if case.min_part_size is defined %} + {{ case.min_part_size }} +{% endif %} +{% if case.min_part_size_ratio is defined %} + {{ case.min_part_size_ratio }} +{% endif %} +{% if case.method is defined %} + {{ case.method }} +{% endif %} +{% if case.level is defined %} + {{ case.level }} +{% endif %} + +{% endfor %} + +{% endif %} + + + + + + + + + + + + + + + + + + + + + + + +{% if clickhouse_encryption is defined %} + <{{ clickhouse_encryption.encryption }}> +{% for key in clickhouse_encryption.elements %} + <{{ key.type }}{% if key.id is defined %} id={{ key.id }}{% endif %}{% if key.from_env is defined %} from_env={{ key.from_env }}{% endif %}>{% if key.value is defined %}{{ key.value }}{% endif %} +{% endfor %} +{% if clickhouse_encryption.current_key_id is defined %} + {{ clickhouse_encryption.current_key_id }} +{% endif %} +{% if clickhouse_encryption.nonce is defined %} + {{ clickhouse_encryption.nonce }} +{% endif %} + +{% endif %} + + + +{% if clickhouse_default_replica_path is defined %} + {{ clickhouse_default_replica_path }} +{% endif %} +{% if clickhouse_default_replica_name is defined %} + {{ clickhouse_default_replica_name }} +{% endif %} + + + +{% if clickhouse_zookeeper is defined and clickhouse_distributed_ddl is defined %} + + + {{ clickhouse_distributed_ddl.path }} + + + +{% if clickhouse_distributed_ddl.profile is defined %} + {{ clickhouse_distributed_ddl.profile }} +{% endif %} + + + +{% if clickhouse_distributed_ddl.pool_size is defined %} + {{ clickhouse_distributed_ddl.pool_size }} +{% endif %} + + + + + +{% if clickhouse_distributed_ddl.task_max_lifetime %} + {{ clickhouse_distributed_ddl.task_max_lifetime }} +{% endif %} + + + +{% if clickhouse_distributed_ddl.cleanup_delay_period %} + {{ clickhouse_distributed_ddl.cleanup_delay_period }} +{% endif %} + + + +{% if clickhouse_distributed_ddl.max_tasks_in_queue %} + {{ clickhouse_distributed_ddl.max_tasks_in_queue }} +{% endif %} + +{% endif %} + + + +{% if clickhouse_mergetree_settings is defined %} + +{% for setting in clickhouse_mergetree_settings|dict2items %} + <{{ setting.key }}>{{ setting.value }} +{% endfor %} + +{% endif %} + + + + +{% if clickhouse_config.max_table_size_to_drop is defined %} + {{ clickhouse_config.max_table_size_to_drop }} +{% endif %} +{% if clickhouse_config.max_partition_size_to_drop is defined %} + {{ clickhouse_config.max_partition_size_to_drop }} +{% endif %} + + +{% if clickhouse_graphitemergetree_settings is defined %} + <{{ clickhouse_graphitemergetree_settings.name }}> +{% if clickhouse_graphitemergetree_settings.pattern is defined %} + + {{ clickhouse_graphitemergetree_settings.pattern.regexp }} +{% if clickhouse_graphitemergetree_settings.pattern.function is defined %} + {{ clickhouse_graphitemergetree_settings.pattern.function }} +{% endif %} +{% for retention in clickhouse_graphitemergetree_settings.pattern.retention %} + + {{ retention.age }} + {{ retention.precision }} + +{% endfor %} + +{% endif %} + +{% if clickhouse_graphitemergetree_settings.default.function is defined %} + {{ clickhouse_graphitemergetree_settings.default.function }} +{% endif %} +{% for retention in clickhouse_graphitemergetree_settings.default.retention %} + + {{ retention.age }} + {{ retention.precision }} + +{% endfor %} + + +{% endif %} + + + {{ clickhouse_config.format_schema_path }} + + +{% if clickhouse_query_masking_rules is defined %} + +{% for rule in clickhouse_query_masking_rules %} + + {{ rule.name }} + {{ rule.regexp }} + {{ rule.replace }} + +{% endfor %} + +{% endif %} + + +{% if clickhouse_http_handlers is defined %} + +{% for rule in clickhouse_http_handlers %} + +{% if rule.url is defined %} + {{ rule.url }} +{% endif %} +{% if rule.methods is defined %} + {{ rule.methods }} +{% endif %} +{% if rule.headers is defined %} + {{ rule.headers }} +{% endif %} + +{% if rule.handler.type is defined %} + {{ rule.handler.type }} +{% endif %} +{% if rule.handler.type == 'predefined_query_handler' %} + {{ rule.handler.query }} +{% elif rule.handler.type == 'dynamic_query_handler' %} + {{ rule.handler.query_param_name }} +{% endif %} +{% if rule.handler.status is defined %} + {{ rule.handler.status }} +{% endif %} +{% if rule.handler.content_type is defined %} + {{ rule.handler.content_type }} +{% endif %} +{% if rule.handler.response_content is defined %} + {{ rule.handler.response_content }} +{% endif %} + + +{% endfor %} + +{% endif %} + + + + + + {{ clickhouse_send_crash_reports.enabled }} + + {{ clickhouse_send_crash_reports.anonymize }} + + + {{ clickhouse_send_crash_reports.endpoint }} + + + + +{% if clickhouse_config.disable_internal_dns_cache is defined %} + {{ clickhouse_config.disable_internal_dns_cache }} +{% endif %} + + + +{% if clickhouse_rocksdb is defined %} + +{% if clickhouse_rocksdb.db_options is defined %} + +{% for option in clickhouse_rocksdb.db_options|dict2items %} + <{{ option.key }}>{{ option.value }} +{% endfor %} + +{% endif %} +{% if clickhouse_rocksdb.db_options is defined %} + +{% for option in clickhouse_rocksdb.cf_options|dict2items %} + <{{ option.key }}>{{ option.value }} +{% endfor %} + +{% endif %} +{% if clickhouse_rocksdb.table_options is defined %} + +{% for table in clickhouse_rocksdb.table_options %} + + {{ table.name }} + +{% for option in table.options|dict2items %} + <{{ option.key }}>{{ option.value }} +{% endfor %} + + +{% for option in table.cf_options|dict2items %} + <{{ option.key }}>{{ option.value }} +{% endfor %} + +
+{% endfor %} + +{% endif %} + +{% endif %} + +{% if clickhouse_config_xml_extra_config is defined %} +{{ clickhouse_config_xml_extra_config|indent(2, true) }} +{% endif %} + diff --git a/ansible/roles/clickhouse/templates/users.xml.j2 b/ansible/roles/clickhouse/templates/users.xml.j2 new file mode 100644 index 00000000..ccdb0ba6 --- /dev/null +++ b/ansible/roles/clickhouse/templates/users.xml.j2 @@ -0,0 +1,190 @@ + + + + + + + +{% if clickhouse_default_profiles is defined %} +{% for profile in clickhouse_default_profiles %} + <{{ profile }}> +{% for item in clickhouse_default_profiles[profile] %} + <{{ item }}>{{ clickhouse_default_profiles[profile][item] }} +{% endfor %} + +{% endfor %} +{% endif %} + +{% if clickhouse_custom_profiles is defined %} +{% for profile in clickhouse_custom_profiles %} + <{{ profile }}> +{% for item in clickhouse_custom_profiles[profile] %} + <{{ item }}>{{ clickhouse_custom_profiles[profile][item] }} +{% endfor %} + +{% endfor %} +{% endif %} + + + + + +{% if clickhouse_default_users is defined %} +{% for user in clickhouse_default_users %} + <{{ user.name }}> +{% if user.password is defined %} + {{ user.password }} +{% elif user.password_sha256_hex is defined %} + {{ user.password_sha256_hex }} +{% elif user.password_double_sha1_hex is defined %} + {{ user.password_double_sha1_hex }} +{% endif %} + +{% for network in user.networks %} + {{ network }} +{% endfor %} + + {{ user.profile }} +{% if user.databases is defined %} + +{% for database in user.databases %} + {{ database }} +{% endfor %} + +{% endif %} + {{ user.quota }} +{% if user.access_management is defined%} + {{ user.access_management }} +{% endif %} +{% if user.named_collection_control is defined%} + {{ user.named_collection_control }} +{% endif %} +{% if user.show_named_collections is defined%} + {{ user.show_named_collections }} +{% endif %} +{% if user.show_named_collections_secrets is defined%} + {{ user.show_named_collections_secrets }} +{% endif %} + +{% endfor %} +{% endif %} + +{% if clickhouse_admin_users is defined %} +{% for user in clickhouse_admin_users %} + <{{ user.name }}> +{% if user.password is defined %} + {{ user.password }} +{% elif user.password_sha256_hex is defined %} + {{ user.password_sha256_hex }} +{% elif user.password_double_sha1_hex is defined %} + {{ user.password_double_sha1_hex }} +{% endif %} + +{% for network in user.networks %} + {{ network }} +{% endfor %} + + {{ user.profile }} +{% if user.databases is defined %} + +{% for database in user.databases %} + {{ database }} +{% endfor %} + +{% endif %} + {{ user.quota }} +{% if user.access_management is defined%} + {{ user.access_management }} +{% endif %} +{% if user.named_collection_control is defined%} + {{ user.named_collection_control }} +{% endif %} +{% if user.show_named_collections is defined%} + {{ user.show_named_collections }} +{% endif %} +{% if user.show_named_collections_secrets is defined%} + {{ user.show_named_collections_secrets }} +{% endif %} + +{% endfor %} +{% endif %} + +{% if clickhouse_custom_users_xml is defined %} +{% for user in clickhouse_custom_users_xml %} + <{{ user.name }}> +{% if user.password is defined %} + {{ user.password }} +{% elif user.password_sha256_hex is defined %} + {{ user.password_sha256_hex }} +{% elif user.password_double_sha1_hex is defined %} + {{ user.password_double_sha1_hex }} +{% endif %} + +{% for network in user.networks %} + {{ network }} +{% endfor %} + + {{ user.profile }} + {{ user.quota }} +{% if user.databases is defined %} + +{% for database in user.databases %} + {{ database }} +{% endfor %} + +{% endif %} +{% if user.access_management is defined%} + {{ user.access_management }} +{% endif %} +{% if user.named_collection_control is defined%} + {{ user.named_collection_control }} +{% endif %} +{% if user.show_named_collections is defined%} + {{ user.show_named_collections }} +{% endif %} +{% if user.show_named_collections_secrets is defined%} + {{ user.show_named_collections_secrets }} +{% endif %} + +{% endfor %} +{% endif %} + + + + + +{% if clickhouse_default_quotas is defined %} +{% for quota in clickhouse_default_quotas %} + <{{ quota.name }}> + + {{ quota.duration }} + {{ quota.queries }} + {{ quota.errors }} + {{ quota.result_rows }} + {{ quota.read_rows }} + {{ quota.execution_time }} + + +{% endfor %} +{% endif %} +{% if clickhouse_custom_quotas_xml is defined %} + +{% for quota in clickhouse_custom_quotas_xml %} + <{{ quota.name }}> + + {{ quota.duration }} + {{ quota.queries }} + {{ quota.errors }} + {{ quota.result_rows }} + {{ quota.read_rows }} + {{ quota.execution_time }} + + +{% endfor %} +{% endif %} + + +{% if clickhouse_user_xml_extra_config is defined %} +{{ clickhouse_user_xml_extra_config|indent(2, true) }} +{% endif %} + diff --git a/ansible/roles/clickhouse/tests/.gitignore b/ansible/roles/clickhouse/tests/.gitignore new file mode 100644 index 00000000..89f9ac04 --- /dev/null +++ b/ansible/roles/clickhouse/tests/.gitignore @@ -0,0 +1 @@ +out/ diff --git a/ansible/roles/clickhouse/tests/README.md b/ansible/roles/clickhouse/tests/README.md new file mode 100644 index 00000000..c2f008c9 --- /dev/null +++ b/ansible/roles/clickhouse/tests/README.md @@ -0,0 +1,21 @@ +# Tests + +`run.sh` checks what the role's README says it does, against ClickHouse 24.8.6.70 +(production's version) in containers, using the real +`ansible/group_vars/clickhouse/vars.yml` with its secrets replaced by test values: + +- `test_sql_objects.py` runs the role's SQL tasks against an empty node and three + nodes seeded like production's data1-3 (`seed/`), and checks that every node ends up + like the empty one; that each user logs in with its password from vars, can do what + its grants allow and nothing else, and has its settings; that ids, quota assignments, + roles granted by hand and objects not in vars are kept, and the objects dropped with + `clickhouse_sql_objects_drop_unlisted`; + that a second run changes nothing; that queries running during the role never fail; + and that task output shows errors but no password or hash. +- `apt_pin.sh` checks, on Debian 12 against packages.clickhouse.com, that with the + role's pin file apt installs `clickhouse_version` and not the newest release. + +Needs docker compose (or rootless podman with +`DOCKER_HOST=unix:///run/user/$(id -u)/podman/podman.sock`), ansible-playbook and PyYAML. +`KEEP=1` leaves the containers running; `SKIP_APT_PIN=1` skips the apt test. The +ansible output of each run is in `out/`. diff --git a/ansible/roles/clickhouse/tests/apt_pin.sh b/ansible/roles/clickhouse/tests/apt_pin.sh new file mode 100755 index 00000000..8245c838 --- /dev/null +++ b/ansible/roles/clickhouse/tests/apt_pin.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# With the role's pin file, apt's candidate for the ClickHouse packages from +# packages.clickhouse.com is the pinned version, not the newest release in the repo. When +# clickhouse_version is the newest, a pin to an older release is checked too, as the pin +# can't show otherwise. +set -euo pipefail +cd "$(dirname "$0")" +vars=../../../group_vars/clickhouse/vars.yml +version=$(sed -n 's/^clickhouse_version: *//p' $vars) +release=$(sed -n 's/^clickhouse_release_type: *//p' $vars); release=${release:-lts} +# the pin file as tasks/install-Debian.yml writes it, for a version given later +pin=$(python3 -c ' +import yaml +task = next(t for t in yaml.safe_load(open("../tasks/install-Debian.yml")) if t.get("ansible.builtin.copy", {}).get("dest") == "/etc/apt/preferences.d/clickhouse") +print(task["ansible.builtin.copy"]["content"], end="") +') +docker run --rm -i -v "$PWD/..:/role:ro,z" -e VERSION="$version" -e RELEASE="$release" -e PIN="$pin" docker.io/library/debian:12 bash -s <<'SH' +set -euo pipefail +apt-get update -qq >/dev/null && apt-get install -y -qq ca-certificates >/dev/null +mkdir -p /etc/apt/keyrings && cp /role/files/clickhouse-repo-keyring.asc /etc/apt/keyrings/clickhouse.asc +echo "deb [signed-by=/etc/apt/keyrings/clickhouse.asc] https://packages.clickhouse.com/deb $RELEASE main" > /etc/apt/sources.list.d/clickhouse.list +apt-get update -qq >/dev/null +newest=$(apt-cache policy clickhouse-server | sed -n 's/^ Candidate: //p') +versions=("$VERSION") +if [ "$newest" = "$VERSION" ]; then + versions+=("$(apt-cache madison clickhouse-server | awk '{print $3}' | sed -n 2p)") +fi +fail=0 +for v in "${versions[@]}"; do + printf "%s" "${PIN//\{\{ clickhouse_version \}\}/$v}" > /etc/apt/preferences.d/clickhouse + for p in clickhouse-server clickhouse-client clickhouse-common-static; do + c=$(apt-cache policy $p | sed -n 's/^ Candidate: //p') + if [ "$c" = "$v" ]; then echo "ok pinned to $v, $p: candidate $c (newest in the repo: $newest)"; else echo "FAIL pinned to $v, $p: candidate $c"; fail=1; fi + done +done +exit $fail +SH diff --git a/ansible/roles/clickhouse/tests/bin/clickhouse-client b/ansible/roles/clickhouse/tests/bin/clickhouse-client new file mode 100755 index 00000000..f1238dd7 --- /dev/null +++ b/ansible/roles/clickhouse/tests/bin/clickhouse-client @@ -0,0 +1,7 @@ +#!/bin/bash +# The role runs clickhouse-client on the ClickHouse host; run it in this test host's +# container instead (CH_NODE), where tests/out/ is mounted at /out. +set -euo pipefail +args=() +for a in "$@"; do args+=("${a/#$TESTS\/out\//\/out\/}"); done +exec $COMPOSE -f "$TESTS/compose.yml" exec -T "$CH_NODE" clickhouse-client "${args[@]}" diff --git a/ansible/roles/clickhouse/tests/compose.yml b/ansible/roles/clickhouse/tests/compose.yml new file mode 100644 index 00000000..67c76fbc --- /dev/null +++ b/ansible/roles/clickhouse/tests/compose.yml @@ -0,0 +1,16 @@ +# ClickHouse at production's version: `fresh` starts empty, data1-3 are seeded like the +# production nodes (seed/). users.xml is rendered from the role's template (render.yml); +# out/ also holds the admin client configs the role's tasks write. +x-node: &node + image: docker.io/clickhouse/clickhouse-server:24.8.6.70 + environment: {CLICKHOUSE_SKIP_USER_SETUP: "1"} + volumes: + - ./out/users.xml:/etc/clickhouse-server/users.xml:ro,z + - ./out:/out:ro,z + ulimits: {nofile: 262144} + mem_limit: 1500m # ClickHouse caps itself at 90% of it; at 700m, queries during the role ran out on CI +services: + fresh: {<<: *node, ports: ["127.0.0.1:18120:8123"]} + data1: {<<: *node, ports: ["127.0.0.1:18121:8123"]} + data2: {<<: *node, ports: ["127.0.0.1:18122:8123"]} + data3: {<<: *node, ports: ["127.0.0.1:18123:8123"]} diff --git a/ansible/roles/clickhouse/tests/play.yml b/ansible/roles/clickhouse/tests/play.yml new file mode 100644 index 00000000..9bdf27a9 --- /dev/null +++ b/ansible/roles/clickhouse/tests/play.yml @@ -0,0 +1,18 @@ +# The role's SQL-driven tasks, in tasks/main.yml's order, against the test containers +- hosts: all + connection: local + gather_facts: false + vars: + clickhouse_admin_client_config: "{{ lookup('env', 'TESTS') }}/out/admin-{{ inventory_hostname }}.xml" + environment: + CH_NODE: "{{ inventory_hostname }}" + PATH: "{{ lookup('env', 'TESTS') }}/bin:{{ lookup('env', 'PATH') }}" + tasks: + # tasks/main.yml's task, without owner root + - name: Admin client configuration + ansible.builtin.template: {src: ../templates/admin-client.xml.j2, dest: "{{ clickhouse_admin_client_config }}", mode: "0644"} + no_log: true + # users first, as in tasks/main.yml: a quota's TO names users + - {name: users, include_role: {name: clickhouse, tasks_from: config/users.yml}, when: clickhouse_role_manage_users} + - {name: quotas, include_role: {name: clickhouse, tasks_from: config/quotas.yml}, when: clickhouse_role_manage_quotas} + - {name: grants, include_role: {name: clickhouse, tasks_from: config/grants.yml}, when: clickhouse_role_manage_grants} diff --git a/ansible/roles/clickhouse/tests/render.yml b/ansible/roles/clickhouse/tests/render.yml new file mode 100644 index 00000000..db3f8eb6 --- /dev/null +++ b/ansible/roles/clickhouse/tests/render.yml @@ -0,0 +1,7 @@ +# Renders users.xml from the role's template with the test vars, for compose.yml +- hosts: localhost + connection: local + gather_facts: false + vars_files: [../defaults/main.yml, out/vars.yml] + tasks: + - ansible.builtin.template: {src: ../templates/users.xml.j2, dest: out/users.xml, mode: "0644"} diff --git a/ansible/roles/clickhouse/tests/run.sh b/ansible/roles/clickhouse/tests/run.sh new file mode 100755 index 00000000..023d4d8c --- /dev/null +++ b/ansible/roles/clickhouse/tests/run.sh @@ -0,0 +1,16 @@ +#!/bin/bash +# Tests the role's SQL-driven users, quotas and grants (test_sql_objects.py) and its apt pin +# (apt_pin.sh). Needs docker compose, ansible-playbook and PyYAML. With rootless podman, +# point DOCKER_HOST at its socket. KEEP=1 leaves the containers running. +set -euo pipefail +cd "$(dirname "$0")" +export COMPOSE=${COMPOSE:-docker compose} +compose() { $COMPOSE -f compose.yml "$@"; } +trap 'rc=$?; [ $rc = 0 ] || compose logs --no-color --tail=50; [ -n "${KEEP:-}" ] || compose down -v --remove-orphans >/dev/null 2>&1' EXIT +compose down -v --remove-orphans >/dev/null 2>&1 # left from a run with KEEP=1 +rm -rf out && mkdir out +python3 test_sql_objects.py render +ANSIBLE_NOCOLOR=1 ansible-playbook -i localhost, render.yml > out/ansible-render.log || { tail -40 out/ansible-render.log; exit 1; } +compose up -d --quiet-pull +python3 test_sql_objects.py +[ -n "${SKIP_APT_PIN:-}" ] || ./apt_pin.sh diff --git a/ansible/roles/clickhouse/tests/seed/data1.sql b/ansible/roles/clickhouse/tests/seed/data1.sql new file mode 100644 index 00000000..83d29ffa --- /dev/null +++ b/ansible/roles/clickhouse/tests/seed/data1.sql @@ -0,0 +1,22 @@ +-- data1 as on 2026-10-26: oonimeasurements created by hand (old limits, no profile, so +-- readonly 2 from the default profile), the writers on the XML write profile, an empty +-- role granted to the user of the same name, the oonimeasurements quota set up and +-- assigned by hand, grants the role doesn't list, and objects that aren't in vars. +-- {password:} is the hash of the user's password from vars. +CREATE USER oonimeasurements IDENTIFIED WITH sha256_hash BY '{password:oonimeasurements}' SETTINGS max_memory_usage = 501001000, max_execution_time = 60; +CREATE USER fastpath IDENTIFIED WITH sha256_hash BY '{password:fastpath}' SETTINGS PROFILE 'write'; +CREATE USER ooniprobe IDENTIFIED WITH sha256_hash BY '{password:ooniprobe}' SETTINGS PROFILE 'write'; +CREATE USER oonirun IDENTIFIED WITH sha256_hash BY '{password:oonirun}' SETTINGS PROFILE 'write'; +CREATE USER oonitestlists IDENTIFIED WITH sha256_hash BY '{password:oonitestlists}' SETTINGS PROFILE 'write'; +CREATE USER notebook IDENTIFIED WITH sha256_hash BY '{password:notebook}' SETTINGS PROFILE 'write'; +CREATE ROLE oonimeasurements; +GRANT oonimeasurements TO oonimeasurements; +GRANT SELECT ON ooni.* TO oonimeasurements, fastpath, ooniprobe, oonirun, notebook; +GRANT INSERT ON ooni.fastpath TO fastpath; +CREATE QUOTA oonimeasurements FOR INTERVAL 600 second MAX queries = 12000, errors = 1000, execution_time = 1000 TO oonimeasurements; +CREATE QUOTA ooniprobe; +GRANT ALTER UPDATE ON ooni.* TO oonimeasurements; +GRANT INSERT ON ooni.fastpath TO oonimeasurements; +GRANT SELECT ON extra.* TO ooniprobe; +CREATE USER manual_user IDENTIFIED WITH plaintext_password BY 'manual'; +CREATE QUOTA manual_quota; diff --git a/ansible/roles/clickhouse/tests/seed/data2.sql b/ansible/roles/clickhouse/tests/seed/data2.sql new file mode 100644 index 00000000..9e91ee41 --- /dev/null +++ b/ansible/roles/clickhouse/tests/seed/data2.sql @@ -0,0 +1,10 @@ +-- data2 as on 2026-10-26: oonimeasurements created by hand with only a memory limit, and +-- quotas that exist but are empty and assigned to nobody. +CREATE USER oonimeasurements IDENTIFIED WITH sha256_hash BY '{password:oonimeasurements}' SETTINGS max_memory_usage = 4294967296; +CREATE USER fastpath IDENTIFIED WITH sha256_hash BY '{password:fastpath}' SETTINGS PROFILE 'write'; +GRANT SELECT ON ooni.* TO oonimeasurements, fastpath; +CREATE QUOTA oonimeasurements; +CREATE QUOTA ooniprobe; +CREATE QUOTA oonirun; +CREATE QUOTA oonitestlists; +CREATE QUOTA notebook; diff --git a/ansible/roles/clickhouse/tests/seed/data3.sql b/ansible/roles/clickhouse/tests/seed/data3.sql new file mode 100644 index 00000000..babecd2f --- /dev/null +++ b/ansible/roles/clickhouse/tests/seed/data3.sql @@ -0,0 +1,9 @@ +-- data3 as on 2026-10-26: the users as upstream's role created them from older vars, with +-- the XML readonly profile (readonly 1, which refuses log_comment) and old passwords. +-- {old_password:} is the hash of a password that isn't the one in vars. +CREATE USER oonimeasurements IDENTIFIED WITH sha256_hash BY '{old_password:oonimeasurements}' SETTINGS max_memory_usage = 1001001000, max_execution_time = 30, max_bytes_to_read = 501001001000, max_rows_to_read = 5001001000, timeout_before_checking_execution_speed = 5, max_result_rows = 51000, PROFILE 'readonly'; +CREATE USER fastpath IDENTIFIED WITH sha256_hash BY '{old_password:fastpath}' SETTINGS PROFILE 'write'; +CREATE USER ooniprobe IDENTIFIED WITH sha256_hash BY '{old_password:ooniprobe}' SETTINGS PROFILE 'write'; +GRANT SELECT ON ooni.* TO oonimeasurements, fastpath, ooniprobe; +GRANT INSERT ON ooni.fastpath TO fastpath; +CREATE QUOTA oonimeasurements; diff --git a/ansible/roles/clickhouse/tests/test_sql_objects.py b/ansible/roles/clickhouse/tests/test_sql_objects.py new file mode 100755 index 00000000..faac59c0 --- /dev/null +++ b/ansible/roles/clickhouse/tests/test_sql_objects.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python3 +""" +Runs the role's SQL-driven tasks (users, quotas, grants) with the real +ansible/group_vars/clickhouse/vars.yml against ClickHouse containers (compose.yml): +`fresh` starts empty, data1-3 are seeded like the production nodes (seed/), and checks +that: + +- every node ends up with the users, privileges and quota limits of an empty node that + only ever had the role, keeping user ids, quota assignments, roles granted by hand and + objects not in vars; +- every user logs in with its password from vars (and not with an old one), can do what + its grants allow and nothing else, and has the settings from vars, CONST ones fixed; +- a second run changes nothing; +- queries running during the role runs never lose their privileges; +- task output never shows a password or its hash, and errors are visible; +- with clickhouse_sql_objects_drop_unlisted, objects not in vars are dropped. + +Called by run.sh; needs ansible-playbook and PyYAML. +""" +import base64, hashlib, json, os, re, subprocess, sys, threading, time, urllib.request + +TESTS = os.path.dirname(os.path.abspath(__file__)) +ROLES = os.path.normpath(os.path.join(TESTS, "../..")) +VARS = os.path.normpath(os.path.join(ROLES, "../group_vars/clickhouse/vars.yml")) +OUT = os.path.join(TESTS, "out") +COMPOSE = os.environ.get("COMPOSE", "docker compose").split() + ["-f", os.path.join(TESTS, "compose.yml")] +NODES = {"fresh": 18120, "data1": 18121, "data2": 18122, "data3": 18123} +SEEDED = ["data1", "data2", "data3"] +failures = [] + + +def check(ok, what): + print(("ok " if ok else "FAIL ") + what) + if not ok: + failures.append(what) + + +# --- test vars: production's vars.yml, with the AWS lookups replaced by test secrets ------- + +def write_test_vars(): + text = open(VARS).read() + text = re.sub(r"lookup\('amazon\.aws\.aws_(?:ssm|secret)', '([^']+)'[^)]*\)", + lambda m: "'test-" + m.group(1).rsplit("/", 1)[-1] + "'", text) + os.makedirs(OUT, exist_ok=True) + open(os.path.join(OUT, "vars.yml"), "w").write(text) + import yaml + v = yaml.safe_load(text) + # each user's password is "{{ 'test-...' | hash('sha256') }}" + passwords = {u["name"]: re.search(r"'([^']+)' \| hash\('sha256'\)", u["password"]).group(1) + for u in v["clickhouse_custom_users"]} + admin = re.fullmatch(r"\{\{ '([^']+)' \}\}", v["clickhouse_admin_password"]).group(1) + return v, passwords, admin + + +def sha256(s): + return hashlib.sha256(s.encode()).hexdigest() + + +# --- ClickHouse access --------------------------------------------------------------- + +def admin_sql(node, sql): + """As admin, inside the node's container (admin may only connect from localhost).""" + r = subprocess.run(COMPOSE + ["exec", "-T", node, "clickhouse-client", "-u", ADMIN_USER, + "--password", ADMIN_PASSWORD, "--multiquery"], + input=sql, capture_output=True, text=True) + if r.returncode: + raise RuntimeError(f"{node}: {r.stderr.strip()}") + return r.stdout + + +def user_sql(node, user, password, sql): + """As a service user, over HTTP. Returns (ok, text): text is the error on failure.""" + req = urllib.request.Request(f"http://127.0.0.1:{NODES[node]}/", data=sql.encode(), headers={ + "X-ClickHouse-User": user, "X-ClickHouse-Key": password}) + try: + with urllib.request.urlopen(req, timeout=30) as r: + return True, r.read().decode().strip() + except urllib.error.HTTPError as e: + return False, e.read().decode().strip() + except OSError as e: + return False, str(e) + + +def error_code(text): + m = re.search(r"Code: (\d+)", text) + return int(m.group(1)) if m else None + + +def state(node): + """What the role manages on a node: users, grants, quotas and SQL objects not in vars.""" + s = {"ids": {}, "users": {}, "quotas": {}} + for line in admin_sql(node, "SELECT name, id FROM system.users WHERE storage = 'local_directory' FORMAT TSV").splitlines(): + name, uid = line.split("\t") + s["ids"][name] = uid + for u in V["clickhouse_custom_users"]: + if u["name"] in s["ids"]: + grants = admin_sql(node, f"SHOW GRANTS FOR {u['name']}").splitlines() + # privileges ("GRANT ... ON ..."), which the role manages, apart from granted roles + s["users"][u["name"]] = (admin_sql(node, f"SHOW CREATE USER {u['name']}").strip(), + [g for g in grants if " ON " in g]) + s.setdefault("roles", {})[u["name"]] = [g for g in grants if " ON " not in g] + names = admin_sql(node, "SELECT name FROM system.quotas WHERE storage = 'local_directory' FORMAT TSV").split() + for q in names: + s["quotas"][q] = admin_sql(node, f"SHOW CREATE QUOTA {q}").strip() + return s + + +def quota_limits(show_create): + return re.sub(r" TO .*$", "", show_create) + + +def quota_to(show_create): + m = re.search(r" TO (.*)$", show_create) + return m.group(1) if m else None + + +# --- running the role ---------------------------------------------------------------- + +def run_role(label, limit=None, extra=None): + inventory = os.path.join(OUT, "inventory.ini") + open(inventory, "w").write("\n".join(NODES) + "\n") + cmd = ["ansible-playbook", "-i", inventory, os.path.join(TESTS, "play.yml"), + "-e", "@" + os.path.join(OUT, "vars.yml")] + if limit: + cmd += ["--limit", limit] + if extra: + cmd += ["-e", json.dumps(extra)] + env = dict(os.environ, ANSIBLE_ROLES_PATH=ROLES, TESTS=TESTS, ANSIBLE_NOCOLOR="1", + ANSIBLE_PYTHON_INTERPRETER=sys.executable, COMPOSE=" ".join(COMPOSE[:-2])) + r = subprocess.run(cmd, env=env, capture_output=True, text=True) + log = r.stdout + r.stderr + open(os.path.join(OUT, f"ansible-{label}.log"), "w").write(log) + if r.returncode and label != "error": # the "error" run is meant to fail + print(log[-3000:]) + return r.returncode, log + + +def leaks(log): + """Passwords or password hashes from the test vars that appear in ansible's output.""" + secrets = [ADMIN_PASSWORD, sha256(ADMIN_PASSWORD)] + for p in PASSWORDS.values(): + secrets += [p, sha256(p)] + return [s[:12] + "..." for s in secrets if s in log] + + +class Probe(threading.Thread): + """Inserts as fastpath and selects as oonimeasurements on a node until stopped.""" + def __init__(self, node): + super().__init__(daemon=True) + self.node, self.ok, self.errors, self.stop = node, 0, [], threading.Event() + + def run(self): + while not self.stop.is_set(): + for user, sql in (("fastpath", "INSERT INTO ooni.fastpath VALUES (1)"), + ("oonimeasurements", "SELECT count() FROM ooni.probe")): + ok, text = user_sql(self.node, user, PASSWORDS[user], sql) + if ok: + self.ok += 1 + else: + self.errors.append(f"{user}: {text[:120]}") + + +# --- per-user checks derived from vars --------------------------------------------------- + +def granted(user, privilege, database, table): + return any(user in g["to"] and privilege in g["privileges"] and database in g["databases"] + and (table in g["tables"] or "*" in g["tables"]) for g in V["clickhouse_custom_grants"]) + + +def grant_universe(): + """Every database.table the grants name, with "*" as the table `probe`, and extra.t, + which no one is granted, so that a too broad grant shows.""" + tables = {("extra", "t")} + for g in V["clickhouse_custom_grants"]: + for d in g["databases"]: + for t in g["tables"]: + tables.add((d, "probe" if t == "*" else t)) + return sorted(tables) + + +def check_user(node, user): + name, password = user["name"], PASSWORDS[user["name"]] + ok, text = user_sql(node, name, password, "SELECT currentUser()") + check(ok and text == name, f"{node}: {name} logs in with its password from vars") + for database, table in grant_universe(): + for privilege, sql in (("SELECT", f"SELECT count() FROM {database}.{table}"), + ("INSERT", f"INSERT INTO {database}.{table} VALUES (1)")): + ok, text = user_sql(node, name, password, sql) + want = granted(name, privilege, database, table) + check(ok == want and (ok or error_code(text) == 497), + f"{node}: {name} {'can' if want else 'cannot'} {privilege} {database}.{table}") + for setting in user.get("settings", []): + key, value, const = re.fullmatch(r"(\w+) = (\S+)( CONST)?", setting).groups() + ok, text = user_sql(node, name, password, f"SELECT getSetting('{key}')") + check(ok and float(text) == float(value), f"{node}: {name} has {key} = {value}") + if const: + ok, text = user_sql(node, name, password, f"SELECT 1 SETTINGS {key} = {int(float(value)) * 2}") + check(not ok and error_code(text) == 452, f"{node}: {name} cannot raise {key}") + ok, text = user_sql(node, name, password, "SELECT 1 SETTINGS log_comment = 'test'") + check(ok, f"{node}: {name} can set log_comment") + + +# --- the test ------------------------------------------------------------------------ + +def setup(): + tables = grant_universe() + ddl = "".join(f"CREATE DATABASE IF NOT EXISTS {d};\n" for d in sorted({d for d, _ in tables} | {"extra"})) + ddl += "".join(f"CREATE TABLE IF NOT EXISTS {d}.{t} (x UInt8) ENGINE = Memory;\n" for d, t in tables) + for node in NODES: + for _ in range(60): + try: + admin_sql(node, "SELECT 1") + break + except RuntimeError: + time.sleep(1) + admin_sql(node, ddl) + for node in SEEDED: + sql = open(os.path.join(TESTS, "seed", f"{node}.sql")).read() + sql = re.sub(r"\{password:(\w+)\}", lambda m: sha256(PASSWORDS[m.group(1)]), sql) + sql = re.sub(r"\{old_password:(\w+)\}", lambda m: sha256("old-" + m.group(1)), sql) + admin_sql(node, sql) + + +def main(): + global V, PASSWORDS, ADMIN_USER, ADMIN_PASSWORD + V, PASSWORDS, ADMIN_PASSWORD = write_test_vars() + ADMIN_USER = V["clickhouse_admin_user"] + if sys.argv[1:] == ["render"]: # run.sh renders users.xml before starting the containers + return + setup() + before = {n: state(n) for n in NODES} + check(user_sql("data3", "oonimeasurements", PASSWORDS["oonimeasurements"], "SELECT 1")[0] is False, + "data3: before the role, oonimeasurements has an old password") + + probe = Probe("data1") + probe.start() + rc1, log1 = run_role("run1") + rc2, log2 = run_role("run2") + probe.stop.set() + probe.join() + check(rc1 == 0, "first role run succeeds") + check(rc2 == 0, "second role run succeeds") + check(not probe.errors and probe.ok > 20, + f"data1: {probe.ok} queries during the runs, failures: {probe.errors[:3]}") + check(not leaks(log1 + log2), f"no password or hash in the output: {leaks(log1 + log2)}") + # What the probe can't catch reliably, a gap between two statements, can't happen if + # the role never revokes a privilege a user keeps: all users in vars have grants, so it + # sends no REVOKE at all. + for n in NODES: + admin_sql(n, "SYSTEM FLUSH LOGS") + revokes = admin_sql(n, "SELECT count() FROM system.query_log WHERE type = 'QueryStart' " + f"AND user = '{ADMIN_USER}' AND query ILIKE '%REVOKE%'").strip() + check(revokes == "0", f"{n}: the role sent no REVOKE ({revokes})") + + after = {n: state(n) for n in NODES} + for node in NODES: + users = after[node]["users"] + check(set(users) == {u["name"] for u in V["clickhouse_custom_users"]}, f"{node}: every user in vars exists") + if node != "fresh": + check(users == after["fresh"]["users"], f"{node}: users and grants are those of a fresh node") + kept = {n: i for n, i in before[node]["ids"].items() if n in after[node]["ids"]} + check(all(after[node]["ids"][n] == i for n, i in kept.items()), f"{node}: user ids are kept") + check(set(before[node]["ids"]) <= set(after[node]["ids"]), f"{node}: users not in vars are kept") + check(set(before[node]["quotas"]) <= set(after[node]["quotas"]), f"{node}: quotas not in vars are kept") + check(after[node].get("roles") == {u: before[node].get("roles", {}).get(u, []) for u in after[node]["roles"]}, + f"{node}: roles granted by hand are kept") + for q in V["clickhouse_custom_quotas"]: + show = after[node]["quotas"].get(q["name"], "") + check(quota_limits(show) == quota_limits(after["fresh"]["quotas"][q["name"]]), + f"{node}: quota {q['name']} has the limits from vars") + want_to = ", ".join(q["to"]) if "to" in q else quota_to(before[node]["quotas"].get(q["name"], "")) + check(quota_to(show) == want_to, f"{node}: quota {q['name']} is assigned to {want_to or 'nobody'}") + for user in V["clickhouse_custom_users"]: + check_user(node, user) + check(user_sql("data3", "oonimeasurements", "old-oonimeasurements", "SELECT 1")[0] is False, + "data3: the old password no longer works") + + rc3, _ = run_role("run3") + check(rc3 == 0 and {n: state(n) for n in NODES} == after, "a third run changes nothing") + + # An invalid setting fails the run, with ClickHouse's error and without the password hash + bad = [dict(u, settings=u.get("settings", []) + ["not_a_setting = 1"]) if u["name"] == "oonimeasurements" else u + for u in V["clickhouse_custom_users"]] + rc, log = run_role("error", limit="data1", extra={"clickhouse_custom_users": bad}) + check(rc != 0 and "UNKNOWN_SETTING" in log, "an invalid setting fails with ClickHouse's error in the output") + check(not leaks(log), f"no password or hash in the error output: {leaks(log)}") + + rc, _ = run_role("drop", limit="data1", extra={"clickhouse_sql_objects_drop_unlisted": True}) + s = state("data1") + check(rc == 0 and "manual_user" not in s["ids"] and "manual_quota" not in s["quotas"], + "data1: with clickhouse_sql_objects_drop_unlisted, users and quotas not in vars are dropped") + check(s["users"] == after["fresh"]["users"], "data1: users in vars are unaffected by the drop") + + print(f"\n{len(failures)} failed" if failures else "\nall passed") + sys.exit(1 if failures else 0) + + +if __name__ == "__main__": + main() diff --git a/ansible/roles/clickhouse/tests/upgrade/.dockerignore b/ansible/roles/clickhouse/tests/upgrade/.dockerignore new file mode 100644 index 00000000..d8a1d071 --- /dev/null +++ b/ansible/roles/clickhouse/tests/upgrade/.dockerignore @@ -0,0 +1,2 @@ +cache/ +out/ diff --git a/ansible/roles/clickhouse/tests/upgrade/.gitignore b/ansible/roles/clickhouse/tests/upgrade/.gitignore new file mode 100644 index 00000000..a97499d0 --- /dev/null +++ b/ansible/roles/clickhouse/tests/upgrade/.gitignore @@ -0,0 +1,2 @@ +out/ +cache/ diff --git a/ansible/roles/clickhouse/tests/upgrade/Dockerfile b/ansible/roles/clickhouse/tests/upgrade/Dockerfile new file mode 100644 index 00000000..05fdd2aa --- /dev/null +++ b/ansible/roles/clickhouse/tests/upgrade/Dockerfile @@ -0,0 +1,8 @@ +# A Debian 12 host with systemd, like the ClickHouse servers, for the role to install on +FROM docker.io/library/debian:12 +RUN apt-get update -qq \ + && apt-get install -y -qq --no-install-recommends systemd systemd-sysv python3 ca-certificates sudo >/dev/null \ + && rm -rf /var/lib/apt/lists/* \ + && rm -f /etc/apt/apt.conf.d/docker-clean # keep downloaded packages in the shared cache +STOPSIGNAL SIGRTMIN+3 +CMD ["/sbin/init"] diff --git a/ansible/roles/clickhouse/tests/upgrade/README.md b/ansible/roles/clickhouse/tests/upgrade/README.md new file mode 100644 index 00000000..36929b38 --- /dev/null +++ b/ansible/roles/clickhouse/tests/upgrade/README.md @@ -0,0 +1,36 @@ +# Upgrade test + +`run.sh` (`test_upgrade.py`) rolls a ClickHouse version bump out across the cluster with the +role, one host at a time, the way it's done in production: + +1. Installs `FROM_VERSION` (default: production's `clickhouse_version`) on three Debian 12 + hosts with systemd, named like the production replicas (`compose.yml`), with the real + group and host vars (secrets replaced by test values) and `deploy.yml`, which is what + `deploy-clickhouse.yml` does to a replica without the nftables rules. As for a new cluster, + the hosts are deployed with `--skip-tags clickhouse_schema` first, and the `ooni` database is + created once Keeper has a quorum. +2. Sets `clickhouse_version` to `TO_VERSION` (default: `next_version`, the next hop of the + upgrade plan in ooni/devops#477), then deploys each host with `--limit`, in turn, while a + writer inserts a row every half second, on each node in turn. The hosts start with the + apt source line the role's upstream version left, as production's do. + +After each host it checks that the role succeeded, which includes the role's own check that +the server runs `clickhouse_version`; that the host's `clickhouse-server` is active under +systemd and its packages are `TO_VERSION`, pinned; and that every node answers and reaches +Keeper. It prints each node's version and Keeper role (leader or follower) and how many inserts +failed meanwhile: errors while versions are mixed are expected for some hops (see #477). At +the end every node must run `TO_VERSION`, take inserts, have no read-only or failing +replicated tables, and have every row whose insert succeeded. + + FROM_VERSION=25.3.14.14 TO_VERSION=25.8.29.51 ./run.sh + +Needs docker compose (or rootless podman with +`DOCKER_HOST=unix:///run/user/$(id -u)/podman/podman.sock`), ansible-playbook (the +community.docker collection is installed into `cache/` if missing) and PyYAML, and network +access to packages.clickhouse.com. Downloaded packages are kept in `cache/debs` and copied +into each host's own apt cache before it's deployed, so each version is downloaded once. +`KEEP=1` leaves the hosts running. The ansible output of each deploy is in `out/`. + +In CI (`.github/workflows/test_clickhouse_upgrade.yml`) it runs when a PR changes +`clickhouse_version` (from the base's version to the PR's) or how the role installs or +restarts ClickHouse, and by hand with any two versions. diff --git a/ansible/roles/clickhouse/tests/upgrade/compose.yml b/ansible/roles/clickhouse/tests/upgrade/compose.yml new file mode 100644 index 00000000..10486509 --- /dev/null +++ b/ansible/roles/clickhouse/tests/upgrade/compose.yml @@ -0,0 +1,34 @@ +# Three Debian hosts with systemd, named like the production replicas (hostname and alias as +# in ../../../oonidata_clickhouse/tests/compose.yml), that ansible reaches with the +# community.docker connection and installs ClickHouse on with the role. The test copies the +# downloaded packages through cache/debs from host to host, so that each version is +# downloaded once (sharing apt's cache itself would make the hosts wait for its lock). +x-host: &host + build: . + image: localhost/clickhouse-role-upgrade-host + privileged: true # systemd + pids_limit: -1 # ClickHouse starts hundreds of threads + tmpfs: ["/run", "/run/lock"] + volumes: ["./cache/debs:/debs:z"] + mem_limit: 2g +services: + data1: + <<: *host + container_name: chupgrade-data1 + hostname: clickhouse1.prod.ooni.io + networks: {default: {aliases: [data1.htz-fsn.prod.ooni.nu, clickhouse1.prod.ooni.io]}} + data2: + <<: *host + container_name: chupgrade-data2 + hostname: clickhouse2.prod.ooni.io + networks: {default: {aliases: [data2.htz-fsn.prod.ooni.nu, clickhouse2.prod.ooni.io]}} + data3: + <<: *host + container_name: chupgrade-data3 + hostname: clickhouse3.prod.ooni.io + networks: {default: {aliases: [data3.htz-fsn.prod.ooni.nu, clickhouse3.prod.ooni.io]}} +# The role's config listens on "::" only, as in production. Docker turns IPv6 off +# in containers on an IPv4-only network, and ClickHouse then can't listen at all. +networks: + default: + enable_ipv6: true diff --git a/ansible/roles/clickhouse/tests/upgrade/deploy.yml b/ansible/roles/clickhouse/tests/upgrade/deploy.yml new file mode 100644 index 00000000..a386a0db --- /dev/null +++ b/ansible/roles/clickhouse/tests/upgrade/deploy.yml @@ -0,0 +1,13 @@ +# What deploy-clickhouse.yml does to a replica, one host at a time: the clickhouse role, then +# oonidata_clickhouse's schema step. Not oonidata_clickhouse's nftables rules, which only +# let production's addresses in. +- hosts: clickhouse + serial: 1 # as deploy-clickhouse.yml + gather_facts: true + vars: + oonidata_clickhouse_schema: "{{ lookup('env', 'SCHEMA') }}" + roles: + - clickhouse + tasks: + - ansible.builtin.include_role: {name: oonidata_clickhouse, tasks_from: schema.yml} + tags: [clickhouse_schema] diff --git a/ansible/roles/clickhouse/tests/upgrade/next_version b/ansible/roles/clickhouse/tests/upgrade/next_version new file mode 100644 index 00000000..1b46a4b2 --- /dev/null +++ b/ansible/roles/clickhouse/tests/upgrade/next_version @@ -0,0 +1,5 @@ +# The version the upgrade test upgrades to when a change doesn't bump clickhouse_version: +# the next hop after production's, from the upgrade plan in ooni/devops#477 (for #437): +# 24.8.6.70 -> 25.3.14.14 -> 25.8.29.51 -> 26.3.17.110 -> 26.8.9.10 +# Move it on when production's clickhouse_version reaches it. +25.3.14.14 diff --git a/ansible/roles/clickhouse/tests/upgrade/run.sh b/ansible/roles/clickhouse/tests/upgrade/run.sh new file mode 100755 index 00000000..53563f2e --- /dev/null +++ b/ansible/roles/clickhouse/tests/upgrade/run.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Rolls a ClickHouse version bump out with the role, one host at a time (test_upgrade.py): +# FROM_VERSION=24.8.6.70 TO_VERSION=25.3.14.14 ./run.sh +# Needs docker compose, ansible-playbook and PyYAML, and network access to +# packages.clickhouse.com; the packages are kept in cache/. With rootless podman, point +# DOCKER_HOST at its socket. KEEP=1 leaves the hosts running. +set -euo pipefail +cd "$(dirname "$0")" +export COMPOSE=${COMPOSE:-docker compose} +compose() { $COMPOSE -f compose.yml "$@"; } +journals() { for n in data1 data2 data3; do echo "--- $n"; compose exec -T $n journalctl -u clickhouse-server --no-pager -n 30 2>/dev/null; done; } +trap 'rc=$?; [ $rc = 0 ] || journals; [ -n "${KEEP:-}" ] || compose down -v --remove-orphans >/dev/null 2>&1' EXIT +compose down -v --remove-orphans >/dev/null 2>&1 # left from a run with KEEP=1 +rm -rf out && mkdir -p out cache +if ! ansible-galaxy collection list community.docker 2>/dev/null | grep -q community.docker; then + # the version the repo pins + ansible-galaxy collection install -p cache/collections -r ../../../../requirements/ansible-galaxy.yml >/dev/null + export ANSIBLE_COLLECTIONS_PATH="$PWD/cache/collections:${ANSIBLE_COLLECTIONS_PATH:-}" +fi +python3 test_upgrade.py diff --git a/ansible/roles/clickhouse/tests/upgrade/test_upgrade.py b/ansible/roles/clickhouse/tests/upgrade/test_upgrade.py new file mode 100755 index 00000000..ef6ed6e2 --- /dev/null +++ b/ansible/roles/clickhouse/tests/upgrade/test_upgrade.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python3 +""" +Rolls a ClickHouse version bump out across the cluster with the role, one host at a time, +as an operator would: deploy FROM_VERSION to three Debian hosts named like the production +replicas (compose.yml), with the real group and host vars (secrets replaced by test values), +then set clickhouse_version to TO_VERSION and deploy each host with --limit, in turn. + +After each host it checks that the role succeeded, that the host's ClickHouse runs +TO_VERSION under systemd with its packages pinned to it, and that every node still answers +and reaches Keeper; and it reports what the cluster looks like in between: each node's +version and Keeper role, and how many of the inserts made meanwhile failed (mixed-version +errors are expected for some hops; see ooni/devops#477). At the end every +node must run TO_VERSION, have no read-only or failing replicated tables, and have every +row whose insert succeeded. + +The hosts start with the apt source line the role's upstream version left (no signed-by), +as production's do. FROM_VERSION defaults to production's clickhouse_version, TO_VERSION to +next_version. Called by run.sh; needs ansible-playbook with community.docker, and PyYAML. +""" +import os, re, shutil, subprocess, sys, threading, time +import yaml + +TESTS = os.path.dirname(os.path.abspath(__file__)) +ANSIBLE = os.path.normpath(os.path.join(TESTS, "../../../..")) +REPO = os.path.dirname(ANSIBLE) +OUT = os.path.join(TESTS, "out") +INV = os.path.join(OUT, "inventory") +COMPOSE = os.environ.get("COMPOSE", "docker compose").split() + ["-f", os.path.join(TESTS, "compose.yml")] +NODES = {"data1": "data1.htz-fsn.prod.ooni.nu", "data2": "data2.htz-fsn.prod.ooni.nu", + "data3": "data3.htz-fsn.prod.ooni.nu"} +ADMIN_CONFIG = "/etc/clickhouse-client/ansible-admin.xml" # written by the role +failures = [] + + +def check(ok, what): + print(("ok " if ok else "FAIL ") + what, flush=True) + if not ok: + failures.append(what) + + +# --- inventory: production's vars, with the version the test sets ------------------------ + +def secrets_replaced(text): + return re.sub(r"lookup\('amazon\.aws\.aws_(?:ssm|secret)', '([^']+)'[^)]*\)", + lambda m: "'test-" + m.group(1).rsplit("/", 1)[-1] + "'", text) + + +def write_inventory(): + shutil.rmtree(INV, ignore_errors=True) + os.makedirs(os.path.join(INV, "group_vars")) + os.makedirs(os.path.join(INV, "host_vars")) + for src, dst in (("group_vars/all/vars.yml", "group_vars/all.yml"), + ("group_vars/clickhouse/vars.yml", "group_vars/clickhouse.yml")): + open(os.path.join(INV, dst), "w").write(secrets_replaced(open(os.path.join(ANSIBLE, src)).read())) + for node, host in NODES.items(): + src = os.path.join(ANSIBLE, "host_vars", host) + text = secrets_replaced(open(src).read()) if os.path.isfile(src) else "" + text += f"\nansible_connection: community.docker.docker\nansible_host: chupgrade-{node}\n" + open(os.path.join(INV, "host_vars", host + ".yml"), "w").write(text) + open(os.path.join(INV, "hosts.ini"), "w").write("[clickhouse]\n" + "\n".join(NODES.values()) + "\n") + + +def set_version(version): + """clickhouse_version in the group vars, as the version bump in a PR would change it.""" + p = os.path.join(INV, "group_vars/clickhouse.yml") + text, n = re.subn(r"^clickhouse_version: .*$", f"clickhouse_version: {version}", open(p).read(), flags=re.M) + assert n == 1 + open(p, "w").write(text) + + +def deploy(label, limit=None, args=()): + hosts = [n for n, h in NODES.items() if limit in (None, h)] + for n in hosts: # the packages downloaded by earlier hosts + sh(n, "cp -n /debs/*.deb /var/cache/apt/archives/ 2>/dev/null; true") + cmd = ["ansible-playbook", "-i", os.path.join(INV, "hosts.ini"), os.path.join(TESTS, "deploy.yml"), *args] + if limit: + cmd += ["--limit", limit] + env = dict(os.environ, ANSIBLE_ROLES_PATH=os.path.join(ANSIBLE, "roles"), ANSIBLE_NOCOLOR="1", + SCHEMA=os.path.join(REPO, "scripts/cluster-migration/schema.sql")) + started = time.time() + r = subprocess.run(cmd, env=env, capture_output=True, text=True) + open(os.path.join(OUT, f"ansible-{label}.log"), "w").write(r.stdout + r.stderr) + for n in hosts: + sh(n, "cp -n /var/cache/apt/archives/*.deb /debs/ 2>/dev/null; true") + if r.returncode: + print(r.stdout[-4000:] + r.stderr[-1500:]) + return r.returncode, time.time() - started + + +# --- the hosts ----------------------------------------------------------------------------- + +def podman_noise_removed(stderr): + """stderr without the notices of podman's docker emulation, if that's what runs compose""" + noise = ("Emulate Docker CLI", "Executing external compose provider", "Error: executing ") + return "\n".join(l for l in stderr.splitlines() if not any(n in l for n in noise) and l.strip("\x1b[0m ")).strip() + + +def sh(node, command, timeout=60): + r = subprocess.run(COMPOSE + ["exec", "-T", node, "bash", "-c", command], capture_output=True, + text=True, timeout=timeout) + return r.returncode, r.stdout.strip(), podman_noise_removed(r.stderr) + + +def sql(node, query, timeout=60): + r = subprocess.run(COMPOSE + ["exec", "-T", node, "clickhouse-client", "--config-file", ADMIN_CONFIG, + "--multiquery"], input=query, capture_output=True, text=True, timeout=timeout) + if r.returncode: + raise RuntimeError(podman_noise_removed(r.stderr)[:300]) + return r.stdout.strip() + + +def try_sql(node, query, timeout=30): + try: + return sql(node, query, timeout) + except (RuntimeError, subprocess.TimeoutExpired) as e: + return f"error: {str(e).splitlines()[0] if str(e) else 'timeout'}" + + +def keeper_role(node): + """This node's Keeper state (leader/follower), from the mntr four-letter command.""" + rc, out, _ = sh(node, "exec 3<>/dev/tcp/127.0.0.1/9181 && printf mntr >&3 && timeout 3 cat <&3", timeout=10) + m = re.search(r"zk_server_state\s+(\w+)", out) + return m.group(1) if m else "unreachable" + + +def snapshot(): + return {n: (try_sql(n, "SELECT version()"), keeper_role(n)) for n in NODES} + + +class Writer(threading.Thread): + """Inserts a row into citizenlab every half second, on each node in turn, and keeps the + urls whose insert succeeded, and the errors of those that didn't.""" + def __init__(self): + super().__init__(daemon=True) + self.ok, self.errors, self.stop, self.i = [], [], threading.Event(), 0 + + def run(self): + nodes = list(NODES) + while not self.stop.is_set(): + node = nodes[self.i % len(nodes)] + url = f"https://example.org/w{self.i}" + r = try_sql(node, f"INSERT INTO ooni.citizenlab VALUES ('example.org', '{url}', 'ZZ', 'TEST')") + (self.errors.append((node, r)) if r.startswith("error") else self.ok.append(url)) + self.i += 1 + time.sleep(0.5) + + def take(self): + ok, errors = len(self.ok), list(self.errors) + self.errors.clear() + return ok, errors + + +def report(title, writer=None, ok_before=0): + print(f"\n--- {title}") + for n, (version, keeper) in snapshot().items(): + print(f" {n}: ClickHouse {version}, Keeper {keeper}") + if writer: + ok, errors = writer.take() + print(f" inserts: {ok - ok_before} succeeded, {len(errors)} failed") + for node, e in errors[:5]: + print(f" {node}: {e[:160]}") + return ok + return ok_before + + +# --- the test ------------------------------------------------------------------------------ + +def main(): + os.makedirs(OUT, exist_ok=True) + os.makedirs(os.path.join(TESTS, "cache/debs"), exist_ok=True) + write_inventory() + prod = yaml.safe_load(open(os.path.join(INV, "group_vars/clickhouse.yml")))["clickhouse_version"] + from_version = os.environ.get("FROM_VERSION") or prod + to_version = os.environ.get("TO_VERSION") or next_version() + print(f"upgrading {from_version} -> {to_version}") + if from_version == to_version: + sys.exit(f"nothing to upgrade: FROM_VERSION and TO_VERSION are both {from_version}; " + "move next_version on") + + set_version(from_version) + r = compose("up", "-d", "--build", "--quiet-pull") + check(r.returncode == 0, "the three hosts start") + if r.returncode: + sys.exit(r.stderr[-2000:]) + for n in NODES: + # "degraded" is normal in a container (e.g. kernel mounts it can't have) + rc, state, _ = sh(n, "systemctl is-system-running --wait", timeout=120) + check(state in ("running", "degraded"), f"{n} boots with systemd ({state})") + # what the role's upstream version left on production's hosts: the repository + # without signed-by, and its key among apt's trusted keys (it used apt-key) + compose("cp", os.path.join(TESTS, "../../files/clickhouse-repo-keyring.asc"), f"{n}:/etc/apt/trusted.gpg.d/clickhouse.asc") + sh(n, "echo 'deb https://packages.clickhouse.com/deb lts main' > /etc/apt/sources.list.d/packages_clickhouse_com_deb.list") + # A new cluster: the hosts one at a time, then the ooni database once Keeper has a quorum + rc, took = deploy("from", args=["--skip-tags", "clickhouse_schema"]) + check(rc == 0, f"the role deploys {from_version} to the three hosts, one at a time ({took:.0f}s)") + rc2, _ = deploy("schema", args=["--tags", "clickhouse_schema"]) + check(rc2 == 0, "the schema step creates the ooni database on the three hosts") + if rc or rc2: + sys.exit(1) + for n in NODES: + check(try_sql(n, "SELECT version()") == from_version, f"{n} runs {from_version}") + sql("data1", "INSERT INTO ooni.citizenlab VALUES ('example.org', 'https://example.org/before', 'ZZ', 'TEST')") + report("before the upgrade") + + writer = Writer() + writer.start() + ok_so_far = 0 + set_version(to_version) + for i, (n, host) in enumerate(NODES.items(), 1): + rc, took = deploy(f"to-{n}", limit=host) + check(rc == 0, f"step {i}: the role upgrades {n} and finds it running {to_version} ({took:.0f}s)") + check(try_sql(n, "SELECT version()") == to_version, f"step {i}: {n} runs {to_version}") + rc, active, _ = sh(n, "systemctl is-active clickhouse-server") + check(active == "active", f"step {i}: clickhouse-server is active under systemd on {n}") + rc, pin, _ = sh(n, "grep '^Pin:' /etc/apt/preferences.d/clickhouse; dpkg-query -W -f '${Version}' clickhouse-server") + check(f"Pin: version {to_version}" in pin and pin.endswith(to_version), f"step {i}: {n}'s packages are {to_version}, pinned") + for other in NODES: + check(wait_keeper(other), f"step {i}: {other} answers and reaches Keeper") + ok_so_far = report(f"after step {i} ({n} upgraded)", writer, ok_so_far) + time.sleep(5) + writer.stop.set() + writer.join() + report("after the upgrade", writer, ok_so_far) + + check(len(writer.ok) > 20, f"inserts kept working during the rollout ({len(writer.ok)} succeeded)") + for n in NODES: # after the rollout, every node takes inserts + url = f"https://example.org/after-{n}" + check(not try_sql(n, f"INSERT INTO ooni.citizenlab VALUES ('example.org', '{url}', 'ZZ', 'TEST')").startswith("error"), + f"{n} takes inserts after the upgrade") + writer.ok.append(url) + for n in NODES: + check(try_sql(n, "SELECT version()") == to_version, f"{n} runs {to_version}") + sql(n, "SYSTEM SYNC REPLICA ooni.citizenlab", timeout=300) + bad = sql(n, "SELECT count() FROM system.replicas WHERE is_readonly OR is_session_expired") + check(bad == "0", f"{n} has no read-only replicated tables") + stuck = sql(n, "SELECT count() FROM system.replication_queue WHERE num_tries > 10") + check(stuck == "0", f"{n} has no replication tasks failing over and over") + urls = set(sql(n, "SELECT url FROM ooni.citizenlab FINAL").split()) + check("https://example.org/before" in urls, f"{n} has the row inserted before the upgrade") + missing = [u for u in writer.ok if u not in urls] + check(not missing, f"{n} has all {len(writer.ok)} rows whose insert succeeded (missing: {missing[:3]})") + + print(f"\n{len(failures)} failed" if failures else "\nall passed") + sys.exit(1 if failures else 0) + + +def next_version(): + lines = [l.strip() for l in open(os.path.join(TESTS, "next_version")) if l.strip() and not l.startswith("#")] + return lines[0] + + +def wait_keeper(node, seconds=60): + """Whether node answers and reaches Keeper within `seconds` (a Keeper election may be + under way after a restart).""" + for _ in range(seconds // 2): + if try_sql(node, "SELECT count() FROM system.zookeeper WHERE path = '/'").isdigit(): + return True + time.sleep(2) + return False + + +def compose(*args): + return subprocess.run(COMPOSE + list(args), capture_output=True, text=True) + + +if __name__ == "__main__": + main() diff --git a/ansible/roles/oonidata_clickhouse/defaults/main.yml b/ansible/roles/oonidata_clickhouse/defaults/main.yml index e69de29b..17e7032f 100644 --- a/ansible/roles/oonidata_clickhouse/defaults/main.yml +++ b/ansible/roles/oonidata_clickhouse/defaults/main.yml @@ -0,0 +1,8 @@ +--- +# The ooni database's schema (tasks/schema.yml), loaded on cluster members that don't +# have the database yet +oonidata_clickhouse_database: ooni +oonidata_clickhouse_schema: "{{ playbook_dir }}/../scripts/cluster-migration/schema.sql" +# Cluster members: the hosts named in a clickhouse_macros entry's `server` (each has its +# replica macro there) +oonidata_clickhouse_replica: "{{ inventory_hostname in (clickhouse_macros | default([]) | selectattr('server', 'defined') | map(attribute='server') | flatten) }}" diff --git a/ansible/roles/oonidata_clickhouse/tasks/main.yml b/ansible/roles/oonidata_clickhouse/tasks/main.yml index 42754f02..308c4a76 100644 --- a/ansible/roles/oonidata_clickhouse/tasks/main.yml +++ b/ansible/roles/oonidata_clickhouse/tasks/main.yml @@ -1,9 +1,16 @@ - ansible.builtin.include_role: - name: idealista.clickhouse_role + name: clickhouse tags: - oonidata - clickhouse +- name: Create the ooni database on a new replica + ansible.builtin.include_tasks: schema.yml + tags: + - oonidata + - clickhouse + - clickhouse_schema + - name: Create nftables rule for clickhouse native port set_fact: ch_nft_rule: '{{ ch_nft_rule | default([]) + [''add rule inet filter input ip saddr '' + item.ip + '' tcp dport 9000 counter accept comment "incoming clickhouse from '' + item.fqdn + ''"''] }}' diff --git a/ansible/roles/oonidata_clickhouse/tasks/schema.yml b/ansible/roles/oonidata_clickhouse/tasks/schema.yml new file mode 100644 index 00000000..04301058 --- /dev/null +++ b/ansible/roles/oonidata_clickhouse/tasks/schema.yml @@ -0,0 +1,42 @@ +--- + +# A cluster member without the ooni database gets it from the repo's schema.sql: a new +# replica, or one that was re-imaged. It is created on this host only (the file's +# ON CLUSTER is removed), and the replicated tables then fetch their data from the other +# replicas; the few tables that aren't replicated start empty. A host that has the +# database is left alone: changes to it go in scripts/cluster-migration/migrations/. +# A re-imaged replica must first be dropped from Keeper (SYSTEM DROP REPLICA), or +# creating its replicated tables fails because the replica already exists. +# If loading schema.sql fails, the deploy stops with ClickHouse's error and leaves the +# database as it is; once the cause is fixed, load schema.sql by hand on that host (see +# docs/Runbooks.md, "Adding and removing a replica"): its statements are IF NOT EXISTS. + +- name: Create the ooni database on a cluster member that doesn't have it + when: oonidata_clickhouse_replica + tags: [clickhouse_schema] + block: + - name: Check whether the ooni database exists + ansible.builtin.command: + cmd: >- + clickhouse-client --config-file {{ clickhouse_admin_client_config }} + --query "EXISTS DATABASE {{ oonidata_clickhouse_database }}" + register: oonidata_clickhouse_database_exists + changed_when: false + check_mode: false # read-only + + - name: Create the ooni database from schema.sql + when: oonidata_clickhouse_database_exists.stdout == "0" + block: + - name: Create the ooni database + ansible.builtin.command: + cmd: >- + clickhouse-client --config-file {{ clickhouse_admin_client_config }} + --query "CREATE DATABASE {{ oonidata_clickhouse_database }}" + + # schema.sql's statements say ON CLUSTER oonidata_cluster; run them on this host only + - name: Create the tables and views of schema.sql + ansible.builtin.command: + cmd: >- + clickhouse-client --config-file {{ clickhouse_admin_client_config }} + --database {{ oonidata_clickhouse_database }} --multiquery + stdin: "{{ lookup('file', oonidata_clickhouse_schema) | regex_replace(' ON CLUSTER \\w+', '') }}" diff --git a/ansible/roles/oonidata_clickhouse/tests/.gitignore b/ansible/roles/oonidata_clickhouse/tests/.gitignore new file mode 100644 index 00000000..89f9ac04 --- /dev/null +++ b/ansible/roles/oonidata_clickhouse/tests/.gitignore @@ -0,0 +1 @@ +out/ diff --git a/ansible/roles/oonidata_clickhouse/tests/README.md b/ansible/roles/oonidata_clickhouse/tests/README.md new file mode 100644 index 00000000..ca8bfb9f --- /dev/null +++ b/ansible/roles/oonidata_clickhouse/tests/README.md @@ -0,0 +1,22 @@ +# Tests + +`run.sh` (`test_replicas.py`) adds a replica to the cluster and removes it again the way the +runbook says (docs/Runbooks.md, "Adding and removing a replica"): by editing the cluster in +`ansible/group_vars/clickhouse/vars.yml` and rerunning the roles. It uses the real group and +host vars (secrets replaced by test values) and the configs the clickhouse role renders from +them, with one ClickHouse 24.8.6.70 container per production host, named and aliased like it +(`compose.yml`). It checks that: + +- data1-3 form the Keeper ensemble and `oonidata_cluster`, and `oonidata_clickhouse` creates + the `ooni` database from `scripts/cluster-migration/schema.sql` on each, but not on notebook1 + (in `[clickhouse]`, not a replica), and leaves existing databases alone; +- with data4 added to `clickhouse_remote_servers` and `clickhouse_macros`, and data1-3 + restarted one at a time, every node sees four replicas, data4 gets the database and catches + up, and inserts and `ON CLUSTER` DDL reach it; +- with data4 removed and dropped from Keeper, every node sees three replicas, Keeper has no + trace of it, `ON CLUSTER` DDL doesn't wait for it, and data1-3 still replicate; +- a re-imaged data4 (empty disk, same replica name) can be added again. + +Needs docker compose (or rootless podman with +`DOCKER_HOST=unix:///run/user/$(id -u)/podman/podman.sock`), ansible-playbook and PyYAML. +`KEEP=1` leaves the containers running. The ansible output of each run is in `out/`. diff --git a/ansible/roles/oonidata_clickhouse/tests/bin/clickhouse-client b/ansible/roles/oonidata_clickhouse/tests/bin/clickhouse-client new file mode 100755 index 00000000..66ad5f7b --- /dev/null +++ b/ansible/roles/oonidata_clickhouse/tests/bin/clickhouse-client @@ -0,0 +1,7 @@ +#!/bin/bash +# The roles run clickhouse-client on the ClickHouse host; run it in that host's container +# instead (CH_NODE), where tests/out/ is mounted at /out. +set -euo pipefail +args=() +for a in "$@"; do args+=("${a/#$TESTS\/out\//\/out\/}"); done +exec $COMPOSE -f "$TESTS/compose.yml" exec -T "$CH_NODE" clickhouse-client "${args[@]}" diff --git a/ansible/roles/oonidata_clickhouse/tests/compose.yml b/ansible/roles/oonidata_clickhouse/tests/compose.yml new file mode 100644 index 00000000..863bbc28 --- /dev/null +++ b/ansible/roles/oonidata_clickhouse/tests/compose.yml @@ -0,0 +1,38 @@ +# ClickHouse at production's version, one container per host, with the production host +# names as hostname and alias, so that the configs the clickhouse role renders from the +# real vars (out//) work unchanged: Keeper and replication use clickhouseN.prod.ooni.io, +# the role matches hosts by their inventory name dataN.htz-fsn.prod.ooni.nu. +# data4 is the replica that the test adds and removes. +x-node: &node + image: docker.io/clickhouse/clickhouse-server:24.8.6.70 + ulimits: {nofile: 262144} + mem_limit: 1500m +services: + data1: + <<: *node + hostname: clickhouse1.prod.ooni.io + networks: {default: {aliases: [data1.htz-fsn.prod.ooni.nu, clickhouse1.prod.ooni.io]}} + volumes: ["./out/data1.htz-fsn.prod.ooni.nu:/etc/clickhouse-server:ro,z", "./out:/out:ro,z"] + data2: + <<: *node + hostname: clickhouse2.prod.ooni.io + networks: {default: {aliases: [data2.htz-fsn.prod.ooni.nu, clickhouse2.prod.ooni.io]}} + volumes: ["./out/data2.htz-fsn.prod.ooni.nu:/etc/clickhouse-server:ro,z", "./out:/out:ro,z"] + data3: + <<: *node + hostname: clickhouse3.prod.ooni.io + networks: {default: {aliases: [data3.htz-fsn.prod.ooni.nu, clickhouse3.prod.ooni.io]}} + # data3's clickhouse_base_path is /data/clickhouse + volumes: ["./out/data3.htz-fsn.prod.ooni.nu:/etc/clickhouse-server:ro,z", "./out:/out:ro,z", "data3:/data"] + data4: + <<: *node + hostname: clickhouse4.prod.ooni.io + networks: {default: {aliases: [data4.htz-fsn.prod.ooni.nu, clickhouse4.prod.ooni.io]}} + volumes: ["./out/data4.htz-fsn.prod.ooni.nu:/etc/clickhouse-server:ro,z", "./out:/out:ro,z"] +volumes: + data3: +# The role's config listens on "::" only, as in production. Docker turns IPv6 off +# in containers on an IPv4-only network, and ClickHouse then can't listen at all. +networks: + default: + enable_ipv6: true diff --git a/ansible/roles/oonidata_clickhouse/tests/render.yml b/ansible/roles/oonidata_clickhouse/tests/render.yml new file mode 100644 index 00000000..6b3e29a8 --- /dev/null +++ b/ansible/roles/oonidata_clickhouse/tests/render.yml @@ -0,0 +1,15 @@ +# Renders each host's config.xml and users.xml with the clickhouse role's own tasks, into +# out//, which compose.yml mounts as the container's /etc/clickhouse-server. +# The test restarts the containers itself, one at a time, as serial: 1 would. notebook1 +# is rendered too, as it's in [clickhouse], but has no container. +- hosts: clickhouse + connection: local + gather_facts: false + vars: + clickhouse_config_directory: "{{ lookup('env', 'TESTS') }}/out/{{ inventory_hostname }}" + clickhouse_handler_on_config_change: Config changed + handlers: + - {name: Config changed, ansible.builtin.debug: {msg: "the test restarts the container"}} + tasks: + - ansible.builtin.file: {path: "{{ clickhouse_config_directory }}", state: directory, mode: "0755"} + - ansible.builtin.include_role: {name: clickhouse, tasks_from: config/clickhouse.yml} diff --git a/ansible/roles/oonidata_clickhouse/tests/run.sh b/ansible/roles/oonidata_clickhouse/tests/run.sh new file mode 100755 index 00000000..38d4c454 --- /dev/null +++ b/ansible/roles/oonidata_clickhouse/tests/run.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# Adds a replica to the cluster and removes it again through vars and the roles +# (test_replicas.py). Needs docker compose, ansible-playbook and PyYAML. With rootless +# podman, point DOCKER_HOST at its socket. KEEP=1 leaves the containers running. +set -euo pipefail +cd "$(dirname "$0")" +export COMPOSE=${COMPOSE:-docker compose} +compose() { $COMPOSE -f compose.yml "$@"; } +trap 'rc=$?; [ $rc = 0 ] || compose logs --no-color --tail=40; [ -n "${KEEP:-}" ] || compose down -v --remove-orphans >/dev/null 2>&1' EXIT +compose down -v --remove-orphans >/dev/null 2>&1 # left from a run with KEEP=1 +rm -rf out && mkdir out +python3 test_replicas.py diff --git a/ansible/roles/oonidata_clickhouse/tests/schema.yml b/ansible/roles/oonidata_clickhouse/tests/schema.yml new file mode 100644 index 00000000..e890dae1 --- /dev/null +++ b/ansible/roles/oonidata_clickhouse/tests/schema.yml @@ -0,0 +1,16 @@ +# oonidata_clickhouse's schema step (tasks/schema.yml) against the test containers +- hosts: all + connection: local + gather_facts: false + vars: + clickhouse_admin_client_config: "{{ lookup('env', 'TESTS') }}/out/admin-{{ inventory_hostname }}.xml" + oonidata_clickhouse_schema: "{{ lookup('env', 'SCHEMA') }}" + environment: + CH_NODE: "{{ inventory_hostname.split('.')[0] }}" + PATH: "{{ lookup('env', 'TESTS') }}/bin:{{ lookup('env', 'PATH') }}" + tasks: + # the clickhouse role's admin client config task, without owner root + - name: Admin client configuration + ansible.builtin.template: {src: ../../clickhouse/templates/admin-client.xml.j2, dest: "{{ clickhouse_admin_client_config }}", mode: "0644"} + no_log: true + - ansible.builtin.include_role: {name: oonidata_clickhouse, tasks_from: schema.yml} diff --git a/ansible/roles/oonidata_clickhouse/tests/test_replicas.py b/ansible/roles/oonidata_clickhouse/tests/test_replicas.py new file mode 100755 index 00000000..ca00b8da --- /dev/null +++ b/ansible/roles/oonidata_clickhouse/tests/test_replicas.py @@ -0,0 +1,280 @@ +#!/usr/bin/env python3 +""" +Adds a replica to the cluster and removes it again, the way an operator would: by editing +the cluster in vars and rerunning the roles. Uses the real ansible/group_vars and host_vars +(secrets replaced by test values) and the clickhouse role's rendered configs, with one +container per production host (compose.yml), and checks that: + +- data1-3 form the Keeper ensemble and oonidata_cluster, and oonidata_clickhouse's schema + step creates the ooni database from schema.sql on each, but not on a non-member host; +- after data4 is added to clickhouse_remote_servers and clickhouse_macros, every node + sees four replicas, the schema step creates the database on data4 alone, data4's + replicated tables catch up, and ON CLUSTER DDL and inserts reach it; +- after data4 is removed from vars and from Keeper (SYSTEM DROP REPLICA), every node sees + three replicas, Keeper has no trace of it, ON CLUSTER DDL doesn't wait for it, and + data1-3 still replicate; +- a re-imaged data4 (empty disk, same replica name) can be added again. + +Called by run.sh; needs ansible-playbook and PyYAML. +""" +import os, re, shutil, subprocess, sys, time +import yaml + +TESTS = os.path.dirname(os.path.abspath(__file__)) +ANSIBLE = os.path.normpath(os.path.join(TESTS, "../../..")) +REPO = os.path.dirname(ANSIBLE) +SCHEMA = os.path.join(REPO, "scripts/cluster-migration/schema.sql") +OUT = os.path.join(TESTS, "out") +INV = os.path.join(OUT, "inventory") +COMPOSE = os.environ.get("COMPOSE", "docker compose").split() + ["-f", os.path.join(TESTS, "compose.yml")] +HOSTS = {f"data{i}": f"data{i}.htz-fsn.prod.ooni.nu" for i in (1, 2, 3, 4)} +NON_MEMBER = "notebook1.htz-fsn.prod.ooni.nu" # in [clickhouse], as in production, but not a replica +failures = [] + + +def check(ok, what): + print(("ok " if ok else "FAIL ") + what) + if not ok: + failures.append(what) + + +def secrets_replaced(text): + return re.sub(r"lookup\('amazon\.aws\.aws_(?:ssm|secret)', '([^']+)'[^)]*\)", + lambda m: "'test-" + m.group(1).rsplit("/", 1)[-1] + "'", text) + + +# --- the inventory: production's vars, and a cluster that the test edits ----------------- + +def write_inventory(): + shutil.rmtree(INV, ignore_errors=True) + os.makedirs(os.path.join(INV, "group_vars")) + os.makedirs(os.path.join(INV, "host_vars")) + for src, dst in (("group_vars/all/vars.yml", "group_vars/all.yml"), + ("group_vars/clickhouse/vars.yml", "group_vars/clickhouse.yml")): + open(os.path.join(INV, dst), "w").write(secrets_replaced(open(os.path.join(ANSIBLE, src)).read())) + for host in list(HOSTS.values()) + [NON_MEMBER]: + src = os.path.join(ANSIBLE, "host_vars", host) + if os.path.isfile(src): + open(os.path.join(INV, "host_vars", host + ".yml"), "w").write(secrets_replaced(open(src).read())) + return yaml.safe_load(open(os.path.join(INV, "group_vars/clickhouse.yml"))) + + +def set_cluster(nodes): + """The cluster in vars as an operator edits it: a replica in oonidata_cluster and a + replica macro for each node (data1-3 keep production's entries).""" + v = dict(VARS) + replicas = [{"replica": None, "host": f"clickhouse{n[-1]}.prod.ooni.io", "port": 9000} for n in nodes] + server = dict(v["clickhouse_remote_servers"][0]) + server["shards"] = [dict(server["shards"][0], replicas=replicas)] + v["clickhouse_remote_servers"] = [server] + macros = [m for m in VARS["clickhouse_macros"] if "server" not in m or m["server"][0] in [HOSTS[n] for n in nodes]] + if "data4" in nodes: + macros.insert(-1, {"macro": "01\n04\n", "server": [HOSTS["data4"]]}) + v["clickhouse_macros"] = macros + yaml.safe_dump(v, open(os.path.join(INV, "group_vars/clickhouse.yml"), "w"), sort_keys=False) + open(os.path.join(INV, "hosts.ini"), "w").write( + "[clickhouse]\n" + "\n".join([HOSTS[n] for n in nodes] + [NON_MEMBER]) + "\n") + + +def ansible(playbook, limit=None): + cmd = ["ansible-playbook", "-i", os.path.join(INV, "hosts.ini"), os.path.join(TESTS, playbook)] + if limit: + cmd += ["--limit", limit] + env = dict(os.environ, ANSIBLE_ROLES_PATH=os.path.join(ANSIBLE, "roles"), TESTS=TESTS, SCHEMA=SCHEMA, + ANSIBLE_NOCOLOR="1", ANSIBLE_PYTHON_INTERPRETER=sys.executable, + COMPOSE=" ".join(COMPOSE[:-2])) + r = subprocess.run(cmd, env=env, capture_output=True, text=True) + open(os.path.join(OUT, f"ansible-{playbook}-{time.strftime('%H%M%S')}.log"), "w").write(r.stdout + r.stderr) + if r.returncode: + print(r.stdout[-3000:] + r.stderr[-2000:]) + return r.returncode, r.stdout + + +def status(stdout, task, host): + """What `task` reported for `host` in ansible's output: ok, changed, skipping or None.""" + m = re.search(r"TASK \[[^\]]*" + re.escape(task) + r"\][^\n]*\n(.*?)(?=\nTASK |\nPLAY RECAP)", stdout, re.S) + s = m and re.search(r"^(ok|changed|skipping|fatal): \[" + re.escape(host) + r"\]", m.group(1), re.M) + return s.group(1) if s else None + + +def changed(stdout, task, host): + return status(stdout, task, host) == "changed" + + +# --- the containers ---------------------------------------------------------------------- + +def compose(*args): + return subprocess.run(COMPOSE + list(args), capture_output=True, text=True) + + +def sql(node, query, timeout=60): + r = subprocess.run(COMPOSE + ["exec", "-T", node, "clickhouse-client", "-u", ADMIN_USER, "--password", + ADMIN_PASSWORD, "--multiquery"], input=query, capture_output=True, text=True, + timeout=timeout) + if r.returncode: + error = "\n".join(l for l in r.stderr.splitlines() if not l.startswith("Emulate Docker CLI")) + raise RuntimeError(f"{node}: {error.strip()[:500]}") + return r.stdout.strip() + + +def wait_ready(node): + for _ in range(120): + try: + if sql(node, "SELECT count() FROM system.zookeeper WHERE path = '/'") != "": + return True + except (RuntimeError, subprocess.TimeoutExpired): + pass + time.sleep(1) + return False + + +def keeper_role(node): + """This node's Keeper state (leader/follower), from the mntr four-letter command.""" + r = subprocess.run(COMPOSE + ["exec", "-T", node, "bash", "-c", + "exec 3<>/dev/tcp/127.0.0.1/9181 && printf mntr >&3 && timeout 3 cat <&3"], + capture_output=True, text=True, timeout=15) + m = re.search(r"zk_server_state\s+(\w+)", r.stdout) + return m.group(1) if m else "unreachable" + + +def restart_one_at_a_time(nodes): + """What a deploy with serial: 1 does after a config change: restart each node and wait + until it's back, its own Keeper too, before the next; then check that inserts work.""" + for n in nodes: + compose("restart", n) + check(wait_ready(n), f"{n} is back after its restart") + keeper = None + for _ in range(60): + keeper = keeper_role(n) + if keeper in ("leader", "follower"): + break + time.sleep(1) + check(keeper in ("leader", "follower"), f"{n}'s Keeper is back in the ensemble ({keeper})") + insert([o for o in nodes if o != n][0], f"after-{n}-restart") + + +def replicas_in_cluster(node): + return sql(node, "SELECT host_name FROM system.clusters WHERE cluster = 'oonidata_cluster' ORDER BY replica_num FORMAT TSV").split() + + +def insert(node, tag): + sql(node, f"INSERT INTO ooni.citizenlab VALUES ('example.org', 'https://example.org/{tag}', 'ZZ', 'TEST')") + + +def citizenlab(node): + sql(node, "SYSTEM SYNC REPLICA ooni.citizenlab", timeout=120) + return sql(node, "SELECT groupArray(url) FROM (SELECT url FROM ooni.citizenlab FINAL ORDER BY url)") + + +def replicated_tables(node): + return sql(node, "SELECT table FROM system.replicas WHERE database = 'ooni' ORDER BY table FORMAT TSV").split() + + +def keeper_replicas(node): + """Replica names under each replicated table's Keeper path, as seen from node.""" + out = {} + for line in sql(node, "SELECT table, zookeeper_path FROM system.replicas WHERE database = 'ooni' FORMAT TSV").splitlines(): + table, path = line.split("\t") + out[table] = sql(node, f"SELECT groupArray(name) FROM (SELECT name FROM system.zookeeper WHERE path = '{path}/replicas' ORDER BY name)") + return out + + +def schema_objects(): + return len(re.findall(r"^CREATE (?:TABLE|MATERIALIZED VIEW) IF NOT EXISTS", open(SCHEMA).read(), re.M)) + + +def on_cluster_ddl(node, name): + sql(node, f"SET distributed_ddl_task_timeout = 60; " + f"CREATE TABLE ooni.{name} ON CLUSTER oonidata_cluster (x UInt8) ENGINE = Memory") + + +# --- the test ---------------------------------------------------------------------------- + +def main(): + global VARS, ADMIN_USER, ADMIN_PASSWORD + os.makedirs(OUT, exist_ok=True) + VARS = write_inventory() + ADMIN_USER = VARS["clickhouse_admin_user"] + ADMIN_PASSWORD = re.fullmatch(r"\{\{ '([^']+)' \}\}", VARS["clickhouse_admin_password"]).group(1) + three = ["data1", "data2", "data3"] + + # A three-node cluster from production's vars + set_cluster(three) + check(ansible("render.yml")[0] == 0, "the clickhouse role renders config.xml and users.xml for data1-3") + compose("up", "-d", "--quiet-pull", *three) + check(all(wait_ready(n) for n in three), "data1-3 start and reach Keeper") + rc, out = ansible("schema.yml") + check(rc == 0, "the schema step runs on data1-3") + check(all(changed(out, "Create the tables and views of schema.sql", HOSTS[n]) for n in three), + "the schema step creates the ooni database on data1-3") + check(status(out, "Check whether the ooni database exists", NON_MEMBER) == "skipping", + "the schema step skips a host that isn't in the cluster") + n_objects = schema_objects() + for n in three: + count = int(sql(n, "SELECT count() FROM system.tables WHERE database = 'ooni'")) + check(count == n_objects, f"{n} has the {n_objects} tables and views of schema.sql ({count})") + rc, out = ansible("schema.yml") + check(rc == 0 and all(status(out, "Create the ooni database", HOSTS[n]) == "skipping" for n in three), + "a second run leaves the existing databases alone") + insert("data1", "before") + check(len({citizenlab(n) for n in three}) == 1, "data1-3 replicate citizenlab") + + # Add data4: edit the cluster in vars, render, restart one at a time, start data4 + set_cluster(three + ["data4"]) + check(ansible("render.yml")[0] == 0, "the clickhouse role renders the configs with data4") + restart_one_at_a_time(three) + compose("up", "-d", "--quiet-pull", "data4") + check(wait_ready("data4"), "data4 starts and reaches Keeper") + for n in three + ["data4"]: + check(replicas_in_cluster(n) == [f"clickhouse{i}.prod.ooni.io" for i in (1, 2, 3, 4)], + f"{n} sees four replicas in oonidata_cluster") + rc, out = ansible("schema.yml") + check(rc == 0 and changed(out, "Create the tables and views of schema.sql", HOSTS["data4"]) + and all(status(out, "Create the ooni database", HOSTS[n]) == "skipping" for n in three), + "the schema step creates the ooni database on data4 alone") + check(replicated_tables("data4") == replicated_tables("data1"), "data4 has every replicated table") + check(all("'04'" in names for names in keeper_replicas("data1").values()), + "Keeper lists replica 04 for every replicated table") + check(citizenlab("data4") == citizenlab("data1"), "data4 catches up on citizenlab") + insert("data4", "from-data4") + check(len({citizenlab(n) for n in three + ["data4"]}) == 1, "an insert on data4 reaches data1-3") + on_cluster_ddl("data1", "ddl_with_four") + check(all(sql(n, "EXISTS TABLE ooni.ddl_with_four") == "1" for n in three + ["data4"]), + "ON CLUSTER DDL reaches data4") + + # Remove data4: edit vars back, render, restart one at a time, stop data4, drop its replica + set_cluster(three) + check(ansible("render.yml")[0] == 0, "the clickhouse role renders the configs without data4") + restart_one_at_a_time(three) + compose("stop", "data4") + sql("data1", "SYSTEM DROP REPLICA '04'") + for n in three: + check(replicas_in_cluster(n) == [f"clickhouse{i}.prod.ooni.io" for i in (1, 2, 3)], + f"{n} sees three replicas in oonidata_cluster") + check(all("'04'" not in names for names in keeper_replicas("data1").values()), + "Keeper has no replica 04 left") + start = time.time() + on_cluster_ddl("data1", "ddl_with_three") + check(time.time() - start < 30 and all(sql(n, "EXISTS TABLE ooni.ddl_with_three") == "1" for n in three), + "ON CLUSTER DDL completes without data4") + insert("data2", "after-removal") + check(len({citizenlab(n) for n in three}) == 1, "data1-3 still replicate") + + # Re-image data4 (empty disk, same replica name) and add it again + compose("rm", "-f", "-s", "-v", "data4") + set_cluster(three + ["data4"]) + check(ansible("render.yml")[0] == 0, "the configs with data4 are rendered again") + restart_one_at_a_time(three) + compose("up", "-d", "data4") + check(wait_ready("data4"), "the re-imaged data4 starts") + rc, out = ansible("schema.yml", limit=HOSTS["data4"]) + check(rc == 0 and changed(out, "Create the tables and views of schema.sql", HOSTS["data4"]), + "the schema step creates the ooni database on the re-imaged data4") + check(citizenlab("data4") == citizenlab("data1"), "the re-imaged data4 catches up") + + print(f"\n{len(failures)} failed" if failures else "\nall passed") + sys.exit(1 if failures else 0) + + +if __name__ == "__main__": + main() diff --git a/docs/Runbooks.md b/docs/Runbooks.md index c1b14ffa..6085f498 100644 --- a/docs/Runbooks.md +++ b/docs/Runbooks.md @@ -1074,7 +1074,7 @@ See the following links for more information: You should first create the replicated database cluster following the instructions at the [clickhouse docs](https://clickhouse.com/docs/en/architecture/replication). -The ooni-devops repo has a role called `oonidata_clickhouse` that does that by using the [idealista.clickhouse_role](https://github.com/idealista/clickhouse_role). +The ooni-devops repo has a role called `oonidata_clickhouse` that does that by using the `clickhouse` role (`ansible/roles/clickhouse`, vendored from [idealista/clickhouse_role](https://github.com/idealista/clickhouse_role)). Once the cluster is created you can proceed with creating a DATABASE on the cluster by running: @@ -1185,6 +1185,71 @@ PARTITION BY concat(substring(bucket_date, 1, 4), substring(bucket_date, 6, 2)) PRIMARY KEY (measurement_uid, observation_idx) ORDER BY (measurement_uid, observation_idx, measurement_start_time, hostname) SETTINGS index_granularity = 8192 ``` +### Upgrading ClickHouse + +ClickHouse is upgraded one hop at a time, along the plan of the upgrade test in +ooni/devops#477 (for #437), whose next hop `ansible/roles/clickhouse/tests/upgrade/next_version` +names: + +1. Change `clickhouse_version` in `ansible/group_vars/clickhouse/vars.yml` in a PR. CI + (`test_clickhouse_upgrade.yml`) rolls that bump out on a test cluster, one host at a time, + and shows what each step looks like. +2. Deploy one host at a time, and look at the cluster before going on to the next: + ``` + ./play deploy-clickhouse.yml -l data1.htz-fsn.prod.ooni.nu --diff -i inventory + ``` + The role installs the new packages, pins them to the new version in + `/etc/apt/preferences.d/clickhouse` (so `apt upgrade` can't move them), restarts + ClickHouse, fails unless the server then runs `clickhouse_version`, and waits until none + of its replicated tables is read-only, i.e. until it's back in Keeper. + +Until every replica has been upgraded, some hops log errors from the replicas that run the +other version; finish the rollout rather than leaving the cluster half upgraded. + +### Adding and removing a replica + +The cluster's replicas are defined in `ansible/group_vars/clickhouse/vars.yml`: +`clickhouse_remote_servers` (the replicas of `oonidata_cluster`) and `clickhouse_macros` (each +host's `shard` and `replica`). `ansible/roles/oonidata_clickhouse/tests/run.sh` does all of this +on containers. + +To add a replica, e.g. `data4.htz-fsn.prod.ooni.nu` as `clickhouse4.prod.ooni.io`: + +1. Add the host to the inventory's `[clickhouse]` group and to `deploy-clickhouse.yml`'s hosts, + a replica to `clickhouse_remote_servers`, and a macro with an unused replica name + (`04`) to `clickhouse_macros`. It doesn't join Keeper unless it's also + added to `clickhouse_keeper` and `clickhouse_zookeeper`. +2. Deploy the existing replicas one at a time, then the new one: + ``` + ./play deploy-clickhouse.yml -l data1.htz-fsn.prod.ooni.nu --diff -i inventory + ``` + On the new host, `oonidata_clickhouse` creates the `ooni` database from + `scripts/cluster-migration/schema.sql`, and its replicated tables fetch their data from the + other replicas. Tables that aren't replicated start empty. Only migrations change the + database of a replica that has it. If loading `schema.sql` fails, the deploy stops with + ClickHouse's error and leaves the database as it is: once the cause is fixed, load it by + hand on that host (its statements are `IF NOT EXISTS`, so it only creates what's missing): + ``` + sed 's/ ON CLUSTER [a-z_]*//' scripts/cluster-migration/schema.sql | ssh data4.htz-fsn.prod.ooni.nu \ + sudo clickhouse-client --config-file /etc/clickhouse-client/ansible-admin.xml --database ooni --multiquery + ``` + +A new cluster has no Keeper quorum until two of its three hosts run, so deploy its hosts +with `--skip-tags clickhouse_schema` first, then once more with `--tags clickhouse_schema` to +create the `ooni` database on each. + +To remove a replica: + +1. Remove it from `clickhouse_remote_servers` and `clickhouse_macros`, and deploy the remaining + replicas one at a time. +2. Stop ClickHouse on the removed host, then remove its replica from Keeper, on any other replica: + ```sql + SYSTEM DROP REPLICA '04' + ``` + +A re-imaged replica that keeps its name must also be dropped from Keeper before it's deployed +again, or creating its replicated tables fails because the replica already exists. + ### Adding users to notebook server In order to add new users to the jupyterlab notebook server hosted