From 0713308980681d0cf53fcb53b363404b6e728ef9 Mon Sep 17 00:00:00 2001 From: DmitriiAn Date: Wed, 12 Aug 2026 11:31:05 +0200 Subject: [PATCH 1/3] Read TABLE_JOBS/INDEX_JOBS from ~/.env --- pgcopydb-helpers/AGENTS.md | 24 ++++++++++++++---------- pgcopydb-helpers/README.md | 17 ++++++++++------- pgcopydb-helpers/resume-cdc.sh | 2 +- pgcopydb-helpers/resume-migration.sh | 4 ++-- pgcopydb-helpers/run-migration.sh | 4 ++-- 5 files changed, 29 insertions(+), 22 deletions(-) diff --git a/pgcopydb-helpers/AGENTS.md b/pgcopydb-helpers/AGENTS.md index d88d4e4..f0086fa 100644 --- a/pgcopydb-helpers/AGENTS.md +++ b/pgcopydb-helpers/AGENTS.md @@ -12,6 +12,10 @@ All scripts read connection strings from `~/.env`: export PGCOPYDB_SOURCE_PGURI='postgresql://user:pass@source-host:5432/dbname' export PGCOPYDB_TARGET_PGURI='postgresql://user:pass@target-host:5432/dbname' export SLACK_WEBHOOK_URL='https://hooks.slack.com/services/...' # optional, for Slack alerts + +# parallelism tuning, shared by all migration scripts +export TABLE_JOBS=8 # default 8 +export INDEX_JOBS=6 # default 6 ``` ## Script Reference @@ -188,9 +192,9 @@ Starts a full `pgcopydb clone --follow` migration. Creates a new timestamped dir ~/run-migration.sh ``` -**Default configuration (edit the script to adjust):** -- `TABLE_JOBS=16` — parallel COPY workers -- `INDEX_JOBS=12` — parallel index creation workers +**Default configuration:** +- `TABLE_JOBS=8` — parallel COPY workers (set in `~/.env`) +- `INDEX_JOBS=6` — parallel index creation workers (set in `~/.env`) - `--split-tables-larger-than 50GB` — splits large tables into parts - `--split-max-parts` matches TABLE_JOBS - `--plugin wal2json` — logical decoding plugin for CDC @@ -549,14 +553,14 @@ IF SOMETHING GOES WRONG: ## Configuration -All scripts use variables at the top that can be adjusted per migration. See [Cluster configuration parameters](https://planetscale.com/docs/postgres/cluster-configuration/parameters) for understanding target-side capacity when tuning these values: +`TABLE_JOBS` and `INDEX_JOBS` are set once in `~/.env` and picked up by every script that uses them; the others are variables at the top of each script. See [Cluster configuration parameters](https://planetscale.com/docs/postgres/cluster-configuration/parameters) for understanding target-side capacity when tuning these values: -| Variable | Default | Used in | -|----------|---------|---------| -| `TABLE_JOBS` | 16 | run-migration.sh, resume-migration.sh | -| `INDEX_JOBS` | 12 | run-migration.sh, resume-migration.sh | -| `FILTER_FILE` | ~/filters.ini | run-migration.sh, resume-migration.sh, resume-cdc.sh | -| `--split-tables-larger-than` | 50GB | run-migration.sh, resume-migration.sh | +| Variable | Default | Set in | Used in | +|----------|---------|--------|---------| +| `TABLE_JOBS` | 8 | `~/.env` | run-migration.sh, resume-migration.sh, resume-cdc.sh | +| `INDEX_JOBS` | 6 | `~/.env` | run-migration.sh, resume-migration.sh | +| `FILTER_FILE` | ~/filters.ini | script | run-migration.sh, resume-migration.sh, resume-cdc.sh | +| `--split-tables-larger-than` | 50GB | script | run-migration.sh, resume-migration.sh | ## Critical Warnings diff --git a/pgcopydb-helpers/README.md b/pgcopydb-helpers/README.md index 1fb11d3..a7faf84 100644 --- a/pgcopydb-helpers/README.md +++ b/pgcopydb-helpers/README.md @@ -119,6 +119,10 @@ SHOW wal_level; -- should return 'logical' export PGCOPYDB_SOURCE_PGURI='postgresql://user:pass@source-host:5432/dbname' export PGCOPYDB_TARGET_PGURI='postgresql://user:pass@target-host:5432/dbname' export SLACK_WEBHOOK_URL='https://hooks.slack.com/services/...' # optional, for Slack alerts + + # parallelism tuning + export TABLE_JOBS=8 # default 8 + export INDEX_JOBS=6 # default 6 ``` 3. **Customize `~/filters.ini`** to exclude schemas, tables, and extensions that should not be migrated. See [Filter Configuration](#filter-configuration) below. @@ -352,14 +356,13 @@ google_ml_integration ## Script Configuration -The migration scripts have tunable parameters at the top of each file: -| Parameter | Default | Description | -|-----------|---------|-------------| -| `TABLE_JOBS` | 16 | Parallel COPY workers | -| `INDEX_JOBS` | 12 | Parallel index creation workers | -| `--split-tables-larger-than` | 50GB | Threshold for splitting large tables into parts | -| `--split-max-parts` | Same as TABLE_JOBS | Maximum number of parts per split table | +| Parameter | Default | Source | Description | +|-----------|---------|--------|-------------| +| `TABLE_JOBS` | 8 | `~/.env` | Parallel COPY workers | +| `INDEX_JOBS` | 6 | `~/.env` | Parallel index creation workers | +| `--split-tables-larger-than` | 50GB | script | Threshold for splitting large tables into parts | +| `--split-max-parts` | Same as TABLE_JOBS | script | Maximum number of parts per split table | Adjust these based on your instance size and database characteristics. More jobs require more CPU cores and memory. A good baseline for `TABLE_JOBS` is fewer than the vCPU count of whichever is smaller — the SOURCE or TARGET. `INDEX_JOBS` should be fewer than the vCPUs on the TARGET. Exceeding these numbers can overwhelm the SOURCE during the COPY phase or the TARGET during index rebuilding. See [Cluster configuration parameters](https://planetscale.com/docs/postgres/cluster-configuration/parameters) for understanding target-side capacity. diff --git a/pgcopydb-helpers/resume-cdc.sh b/pgcopydb-helpers/resume-cdc.sh index efb2023..0bbec1c 100755 --- a/pgcopydb-helpers/resume-cdc.sh +++ b/pgcopydb-helpers/resume-cdc.sh @@ -40,7 +40,7 @@ echo "Resuming CDC in: $MIGRATION_DIR" LOGFILE=$MIGRATION_DIR/migration.log FILTER_FILE=~/filters.ini -TABLE_JOBS=16 +TABLE_JOBS="${TABLE_JOBS:-8}" cd "$MIGRATION_DIR" # Core dumps help debug rare native crashes; not required for a successful migrate. diff --git a/pgcopydb-helpers/resume-migration.sh b/pgcopydb-helpers/resume-migration.sh index 47129c9..32f6f9e 100755 --- a/pgcopydb-helpers/resume-migration.sh +++ b/pgcopydb-helpers/resume-migration.sh @@ -54,8 +54,8 @@ echo "Resuming migration in: $MIGRATION_DIR" LOGFILE=$MIGRATION_DIR/migration.log FILTER_FILE=~/filters.ini -TABLE_JOBS=16 -INDEX_JOBS=12 +TABLE_JOBS="${TABLE_JOBS:-8}" +INDEX_JOBS="${INDEX_JOBS:-6}" cd "$MIGRATION_DIR" # Core dumps help debug rare native crashes; not required for a successful migrate. diff --git a/pgcopydb-helpers/run-migration.sh b/pgcopydb-helpers/run-migration.sh index 23a92c7..33e7ba8 100644 --- a/pgcopydb-helpers/run-migration.sh +++ b/pgcopydb-helpers/run-migration.sh @@ -39,8 +39,8 @@ PGCOPYDB_BIN=$(find_pgcopydb) || { echo "ERROR: pgcopydb not found on PATH or un MIGRATION_DIR=~/migration_$(date +%Y%m%d-%H%M%S) LOGFILE=$MIGRATION_DIR/migration.log FILTER_FILE=~/filters.ini -TABLE_JOBS=16 -INDEX_JOBS=12 +TABLE_JOBS="${TABLE_JOBS:-8}" +INDEX_JOBS="${INDEX_JOBS:-6}" mkdir -p "$MIGRATION_DIR" cd "$MIGRATION_DIR" From 5aa591e630127129efd62f8c45aa7a1ae97b2d40 Mon Sep 17 00:00:00 2001 From: DmitriiAn Date: Wed, 12 Aug 2026 13:37:07 +0200 Subject: [PATCH 2/3] feat pgcopydb-templates: add TABLE_JOBS and INDEX_JOBS to the .env file --- .../aws-cloudformation/pgcopydb-migration-instance.yaml | 4 ++++ pgcopydb-templates/aws-terraform/user-data.sh | 4 ++++ pgcopydb-templates/gcp-terraform/startup-script.sh | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/pgcopydb-templates/aws-cloudformation/pgcopydb-migration-instance.yaml b/pgcopydb-templates/aws-cloudformation/pgcopydb-migration-instance.yaml index 8f389c6..5ef0b3a 100644 --- a/pgcopydb-templates/aws-cloudformation/pgcopydb-migration-instance.yaml +++ b/pgcopydb-templates/aws-cloudformation/pgcopydb-migration-instance.yaml @@ -356,6 +356,10 @@ Resources: # Target Database (PlanetScale) PGCOPYDB_TARGET_PGURI="postgresql://user:password@target-host.connect.psdb.cloud:5432/dbname?sslmode=require" + + # Parallelism tuning (used by run-migration.sh, resume-migration.sh, resume-cdc.sh) + TABLE_JOBS=8 + INDEX_JOBS=6 mode: '000600' owner: ubuntu group: ubuntu diff --git a/pgcopydb-templates/aws-terraform/user-data.sh b/pgcopydb-templates/aws-terraform/user-data.sh index 2ece30c..b7085fe 100644 --- a/pgcopydb-templates/aws-terraform/user-data.sh +++ b/pgcopydb-templates/aws-terraform/user-data.sh @@ -122,6 +122,10 @@ PGCOPYDB_SOURCE_PGURI="postgresql://user:password@source-host:5432/dbname?sslmod # Target Database (PlanetScale) PGCOPYDB_TARGET_PGURI="postgresql://user:password@target-host.connect.psdb.cloud:5432/dbname?sslmode=require" + +# Parallelism tuning (used by run-migration.sh, resume-migration.sh, resume-cdc.sh) +TABLE_JOBS=8 +INDEX_JOBS=6 ENV_EOF chmod 600 /home/ubuntu/.env chown ubuntu:ubuntu /home/ubuntu/.env diff --git a/pgcopydb-templates/gcp-terraform/startup-script.sh b/pgcopydb-templates/gcp-terraform/startup-script.sh index f22e9ed..476adef 100644 --- a/pgcopydb-templates/gcp-terraform/startup-script.sh +++ b/pgcopydb-templates/gcp-terraform/startup-script.sh @@ -106,6 +106,10 @@ PGCOPYDB_SOURCE_PGURI="postgresql://user:password@source-host:5432/dbname?sslmod # Target Database (PlanetScale) PGCOPYDB_TARGET_PGURI="postgresql://user:password@target-host.connect.psdb.cloud:5432/dbname?sslmode=require" + +# Parallelism tuning (used by run-migration.sh, resume-migration.sh, resume-cdc.sh) +TABLE_JOBS=8 +INDEX_JOBS=6 ENV_EOF chmod 600 /home/ubuntu/.env chown ubuntu:ubuntu /home/ubuntu/.env From ad05285401f249bd87f4e0c308208f4c12ba8810 Mon Sep 17 00:00:00 2001 From: Chris Munns Date: Thu, 27 Aug 2026 15:46:03 -0400 Subject: [PATCH 3/3] Move .env ownership to this repo and default CDC to pgoutput Add pgcopydb-helpers/env-template as the reference ~/.env. The user copies it to ~/.env on the migration instance. The provisioning templates no longer create that file. The file cannot be named .env here: .gitignore blocks the name to keep credentials out of git, and the templates deploy with "cp -r pgcopydb-helpers/* /home/ubuntu/", where the shell glob does not match dotfiles. Move the remaining hardcoded tunables into ~/.env: - OUTPUT_PLUGIN, new, default pgoutput. planetscale/pgcopydb#58 adds the pgoutput plugin and makes it the pgcopydb default. It is part of PostgreSQL core, so the source server needs no extension. Set OUTPUT_PLUGIN=wal2json to keep the previous plugin. - SPLIT_TABLES_LARGER_THAN, default 50GB, was a literal in three scripts. - FILTER_FILE, default ~/filters.ini, was a literal in five scripts. run-migration.sh, resume-migration.sh and resume-cdc.sh now log the effective plugin, jobs, split size and filter path to migration.log. drop-replication-slots.sh also drops the source publication. pgcopydb creates one named after the replication slot for pgoutput, so without this it is left behind on the source after every migration. The drop needs ownership of the publication, so the script warns and continues on failure. Every script that sources ~/.env now checks the file exists first and prints the cp command if it does not. Without a guard the scripts failed with a raw "No such file or directory" from the shell, which is reachable now that no template creates the file. Checked with bash -n and shellcheck on the helper scripts and the terraform templates, and cfn-lint on the CloudFormation template. --- pgcopydb-helpers/AGENTS.md | 70 ++++++++++++------ pgcopydb-helpers/README.md | 73 +++++++++++++++---- pgcopydb-helpers/check-cdc-status.sh | 5 ++ pgcopydb-helpers/check-copy-stall.sh | 5 ++ pgcopydb-helpers/check-migration-status.sh | 5 ++ pgcopydb-helpers/compare-pg-params.sh | 5 ++ pgcopydb-helpers/drop-replication-slots.sh | 40 +++++++++- pgcopydb-helpers/env-template | 49 +++++++++++++ pgcopydb-helpers/fix-replica-identity.sh | 5 ++ pgcopydb-helpers/preflight-check.sh | 19 +++-- pgcopydb-helpers/resume-cdc.sh | 18 ++++- pgcopydb-helpers/resume-migration.sh | 18 ++++- pgcopydb-helpers/run-migration.sh | 17 ++++- pgcopydb-helpers/slack-migration-alerts.sh | 5 ++ pgcopydb-helpers/stop-cdc.sh | 5 ++ pgcopydb-helpers/target-clean.sh | 5 ++ pgcopydb-helpers/verify-migration.sh | 11 ++- pgcopydb-templates/README.md | 15 +++- .../pgcopydb-migration-instance.yaml | 17 ----- pgcopydb-templates/aws-terraform/user-data.sh | 18 ----- .../gcp-terraform/startup-script.sh | 18 ----- 21 files changed, 310 insertions(+), 113 deletions(-) create mode 100644 pgcopydb-helpers/env-template diff --git a/pgcopydb-helpers/AGENTS.md b/pgcopydb-helpers/AGENTS.md index f0086fa..ec06411 100644 --- a/pgcopydb-helpers/AGENTS.md +++ b/pgcopydb-helpers/AGENTS.md @@ -6,18 +6,34 @@ This file provides guidance to AI coding assistants (Claude Code, Cursor, Copilo These scripts run on a **migration instance** (EC2 or GCP Compute) that sits between the source PostgreSQL database and the [PlanetScale for Postgres](https://planetscale.com/docs/postgres/) target. The instance has pgcopydb installed and network access to both databases. -All scripts read connection strings from `~/.env`: +All scripts read their configuration from `~/.env`. This repo owns the reference copy: `env-template`. The user copies it to `~/.env` and edits the values: + +```bash +cp ~/env-template ~/.env +chmod 600 ~/.env +``` ```bash export PGCOPYDB_SOURCE_PGURI='postgresql://user:pass@source-host:5432/dbname' export PGCOPYDB_TARGET_PGURI='postgresql://user:pass@target-host:5432/dbname' -export SLACK_WEBHOOK_URL='https://hooks.slack.com/services/...' # optional, for Slack alerts -# parallelism tuning, shared by all migration scripts -export TABLE_JOBS=8 # default 8 -export INDEX_JOBS=6 # default 6 +export TABLE_JOBS=8 # parallel COPY workers +export INDEX_JOBS=6 # parallel index build workers +export SPLIT_TABLES_LARGER_THAN=50GB # copy larger tables in parts +export OUTPUT_PLUGIN=pgoutput # logical decoding plugin for CDC +export FILTER_FILE=~/filters.ini # pgcopydb filter file + +#export SLACK_WEBHOOK_URL='https://hooks.slack.com/services/...' # optional ``` +Each script applies the same default if a variable is unset, with the form `TABLE_JOBS="${TABLE_JOBS:-8}"`. To add a new tunable, edit `env-template`, the scripts that use it, the Configuration table below, and the Script Configuration table in `README.md`. + +The `pgcopydb-templates/` templates no longer create `~/.env`. They copy the whole `pgcopydb-helpers/` directory to `/home/ubuntu/`, so `env-template` arrives with the scripts and the user makes `~/.env` from it. Do not add `.env` generation back to a template. + +`env-template` cannot be named `.env` in this repo. `.gitignore` blocks that name to keep credentials out of git, and the templates deploy with `cp -r pgcopydb-helpers/* /home/ubuntu/`, where the shell glob `*` does not match dotfiles. + +Every script that sources `~/.env` first checks that the file exists and, if not, prints the `cp ~/env-template ~/.env` command and exits 1. Keep that guard in any new script that reads `~/.env`. + ## Script Reference Scripts are organized by migration phase: preparation, execution, monitoring, recovery, and cutover. @@ -192,17 +208,16 @@ Starts a full `pgcopydb clone --follow` migration. Creates a new timestamped dir ~/run-migration.sh ``` -**Default configuration:** -- `TABLE_JOBS=8` — parallel COPY workers (set in `~/.env`) -- `INDEX_JOBS=6` — parallel index creation workers (set in `~/.env`) -- `--split-tables-larger-than 50GB` — splits large tables into parts -- `--split-max-parts` matches TABLE_JOBS -- `--plugin wal2json` — logical decoding plugin for CDC -- `--filter ~/filters.ini` +**Default configuration (all values come from `~/.env`):** +- `TABLE_JOBS=8` — parallel COPY workers, also used for `--split-max-parts` +- `INDEX_JOBS=6` — parallel index creation workers +- `SPLIT_TABLES_LARGER_THAN=50GB` — splits large tables into parts +- `OUTPUT_PLUGIN=pgoutput` — logical decoding plugin for CDC +- `FILTER_FILE=~/filters.ini` **When to use:** Starting a fresh migration. For a COPY-only test (no CDC), remove the `--follow` and `--plugin` flags. -**Requires:** `PGCOPYDB_SOURCE_PGURI`, `PGCOPYDB_TARGET_PGURI`, `~/filters.ini` +**Requires:** `PGCOPYDB_SOURCE_PGURI`, `PGCOPYDB_TARGET_PGURI`, the file named by `FILTER_FILE` --- @@ -354,7 +369,7 @@ Resumes a previously interrupted `pgcopydb clone --follow` migration. Backs up t MIGRATION_DIR=~/migration_YYYYMMDD-HHMMSS ~/resume-migration.sh # specify explicitly ``` -**Important:** The script passes `--split-tables-larger-than` to match `run-migration.sh`. pgcopydb requires catalog consistency — if the original run used split tables, the resume must pass the same value. +**Important:** The script reads `SPLIT_TABLES_LARGER_THAN` and `OUTPUT_PLUGIN` from the same `~/.env` as `run-migration.sh`. pgcopydb requires catalog consistency — do not change either value between the original run and the resume. **When to use:** After pgcopydb crashes, the instance reboots, or the migration is interrupted. To start completely over instead, run `~/target-clean.sh` + `~/drop-replication-slots.sh` first, then `~/start-migration-screen.sh`. @@ -408,6 +423,7 @@ Cleans up pgcopydb replication artifacts on both source and target databases. **What it cleans:** - **Source:** Drops the logical replication slot (terminates active consumer if needed) +- **Source:** Drops the publication of the same name. pgcopydb creates it only for the `pgoutput` plugin. The drop needs ownership of the publication; the script warns and continues if it fails - **Target:** Drops the replication origin and the `pgcopydb` sentinel schema **When to use:** After a migration completes or is abandoned. Replication slots that are not consumed will cause WAL to accumulate on the source until the disk fills up. Always clean up slots when done. @@ -553,14 +569,26 @@ IF SOMETHING GOES WRONG: ## Configuration -`TABLE_JOBS` and `INDEX_JOBS` are set once in `~/.env` and picked up by every script that uses them; the others are variables at the top of each script. See [Cluster configuration parameters](https://planetscale.com/docs/postgres/cluster-configuration/parameters) for understanding target-side capacity when tuning these values: +Every tunable is set once in `~/.env` and picked up by every script that uses it. No script hardcodes these values. See [Cluster configuration parameters](https://planetscale.com/docs/postgres/cluster-configuration/parameters) for understanding target-side capacity when tuning them: + +| Variable | Default | pgcopydb option | Used in | +|----------|---------|-----------------|---------| +| `PGCOPYDB_SOURCE_PGURI` | none (required) | `--source` | all scripts | +| `PGCOPYDB_TARGET_PGURI` | none (required) | `--target` | all scripts | +| `TABLE_JOBS` | 8 | `--table-jobs`, `--split-max-parts` | run-migration.sh, resume-migration.sh, resume-cdc.sh | +| `INDEX_JOBS` | 6 | `--index-jobs` | run-migration.sh, resume-migration.sh | +| `SPLIT_TABLES_LARGER_THAN` | 50GB | `--split-tables-larger-than` | run-migration.sh, resume-migration.sh, resume-cdc.sh | +| `OUTPUT_PLUGIN` | pgoutput | `--plugin` | run-migration.sh, resume-migration.sh, resume-cdc.sh | +| `FILTER_FILE` | `~/filters.ini` | `--filter` | run-migration.sh, resume-migration.sh, resume-cdc.sh, preflight-check.sh, verify-migration.sh | +| `SLACK_WEBHOOK_URL` | unset | — | slack-migration-alerts.sh | + +### `OUTPUT_PLUGIN` + +`pgoutput` is the default. It is part of PostgreSQL core, so the source server needs no extension. pgcopydb builds a publication from the table list in `FILTER_FILE` and drops it during `pgcopydb stream cleanup`. Compared to `wal2json` it sends about 4.5x less network volume and uses about 4x less CPU on the source. + +`wal2json` and `test_decoding` remain supported. Set `OUTPUT_PLUGIN=wal2json` to use the previous plugin; the source server must have the `wal2json` extension installed. -| Variable | Default | Set in | Used in | -|----------|---------|--------|---------| -| `TABLE_JOBS` | 8 | `~/.env` | run-migration.sh, resume-migration.sh, resume-cdc.sh | -| `INDEX_JOBS` | 6 | `~/.env` | run-migration.sh, resume-migration.sh | -| `FILTER_FILE` | ~/filters.ini | script | run-migration.sh, resume-migration.sh, resume-cdc.sh | -| `--split-tables-larger-than` | 50GB | script | run-migration.sh, resume-migration.sh | +`OUTPUT_PLUGIN` and `SPLIT_TABLES_LARGER_THAN` must not change between a run and its resume. pgcopydb requires catalog consistency. ## Critical Warnings diff --git a/pgcopydb-helpers/README.md b/pgcopydb-helpers/README.md index a7faf84..1005c46 100644 --- a/pgcopydb-helpers/README.md +++ b/pgcopydb-helpers/README.md @@ -113,16 +113,37 @@ SHOW wal_level; -- should return 'logical' 1. **Deploy these scripts** to the migration instance home directory (`~/`). -2. **Create `~/.env`** with your connection strings: +2. **Create `~/.env` from `~/env-template`** and edit the values: + + ```bash + cp ~/env-template ~/.env + chmod 600 ~/.env + ``` + + `env-template` holds every setting the scripts read. Only the two connection + strings are required. The remaining settings have working defaults: ```bash export PGCOPYDB_SOURCE_PGURI='postgresql://user:pass@source-host:5432/dbname' export PGCOPYDB_TARGET_PGURI='postgresql://user:pass@target-host:5432/dbname' - export SLACK_WEBHOOK_URL='https://hooks.slack.com/services/...' # optional, for Slack alerts - # parallelism tuning - export TABLE_JOBS=8 # default 8 - export INDEX_JOBS=6 # default 6 + export TABLE_JOBS=8 # parallel COPY workers + export INDEX_JOBS=6 # parallel index build workers + export SPLIT_TABLES_LARGER_THAN=50GB # copy larger tables in parts + export OUTPUT_PLUGIN=pgoutput # logical decoding plugin for CDC + export FILTER_FILE=~/filters.ini # pgcopydb filter file + + #export SLACK_WEBHOOK_URL='https://hooks.slack.com/services/...' # optional + ``` + + See [Script Configuration](#script-configuration) for how to tune each value. + + The provisioning templates do not create `~/.env` for you. Every script stops + with this message until you create it: + + ``` + ERROR: ~/.env not found. Create it from the template: + cp ~/env-template ~/.env && chmod 600 ~/.env ``` 3. **Customize `~/filters.ini`** to exclude schemas, tables, and extensions that should not be migrated. See [Filter Configuration](#filter-configuration) below. @@ -237,7 +258,7 @@ After the migration is complete (or abandoned), clean up replication artifacts: ~/drop-replication-slots.sh my_slot # custom slot name ``` -This drops the replication slot on the source, the replication origin on the target, and the pgcopydb sentinel schema. **Always do this** — unconsumed replication slots cause WAL to accumulate on the source until the disk fills up. +This drops the replication slot on the source, the replication origin on the target, and the pgcopydb sentinel schema. It also drops the publication that pgcopydb creates on the source for the `pgoutput` plugin, which carries the same name as the slot. **Always do this** — unconsumed replication slots cause WAL to accumulate on the source until the disk fills up. ## Recovery @@ -257,7 +278,7 @@ If pgcopydb crashes, the instance reboots, or the migration is interrupted: MIGRATION_DIR=~/migration_YYYYMMDD-HHMMSS ~/resume-migration.sh # or specify explicitly ``` -This backs up the SQLite catalog before resuming and uses `--not-consistent` to allow resuming from a mid-transaction state. The script passes `--split-tables-larger-than` to match `run-migration.sh` — pgcopydb requires catalog consistency, so the resume must use the same split value as the original run. +This backs up the SQLite catalog before resuming and uses `--not-consistent` to allow resuming from a mid-transaction state. The script reads `SPLIT_TABLES_LARGER_THAN` and `OUTPUT_PLUGIN` from the same `~/.env` as `run-migration.sh` — pgcopydb requires catalog consistency, so do not change these values between the original run and the resume. If the initial COPY completed successfully but CDC was interrupted, you can resume only the CDC phase without re-attempting the clone: @@ -278,7 +299,7 @@ To start completely over, wipe the target and clean up replication: ## Filter Configuration -Every migration needs a `~/filters.ini` file to exclude objects that should not be copied. Use the filter to exclude source-specific schemas, tables, and extensions that are not needed on the target — particularly extensions not [supported by PlanetScale](https://planetscale.com/docs/postgres/extensions). The file uses pgcopydb's [filter syntax](https://github.com/planetscale/pgcopydb/blob/main/docs/ref/pgcopydb_filter.rst): +Every migration needs a filter file to exclude objects that should not be copied. The scripts read the path from `FILTER_FILE` in `~/.env`, which defaults to `~/filters.ini`. Use the filter to exclude source-specific schemas, tables, and extensions that are not needed on the target — particularly extensions not [supported by PlanetScale](https://planetscale.com/docs/postgres/extensions). The file uses pgcopydb's [filter syntax](https://github.com/planetscale/pgcopydb/blob/main/docs/ref/pgcopydb_filter.rst): ```ini [exclude-schema] @@ -356,15 +377,34 @@ google_ml_integration ## Script Configuration +Every tunable setting lives in `~/.env`. Change a value once and all scripts use it. `env-template` in this directory is the reference copy. + +| Variable | Default | pgcopydb option | Description | +|----------|---------|-----------------|-------------| +| `PGCOPYDB_SOURCE_PGURI` | none (required) | `--source` | Source connection string | +| `PGCOPYDB_TARGET_PGURI` | none (required) | `--target` | Target connection string | +| `TABLE_JOBS` | 8 | `--table-jobs`, `--split-max-parts` | Parallel COPY workers | +| `INDEX_JOBS` | 6 | `--index-jobs` | Parallel index build workers | +| `SPLIT_TABLES_LARGER_THAN` | 50GB | `--split-tables-larger-than` | Size above which a table is copied in parts | +| `OUTPUT_PLUGIN` | pgoutput | `--plugin` | Logical decoding plugin for CDC | +| `FILTER_FILE` | `~/filters.ini` | `--filter` | pgcopydb filter file | +| `SLACK_WEBHOOK_URL` | unset | — | Webhook for `slack-migration-alerts.sh` | + +Adjust `TABLE_JOBS` and `INDEX_JOBS` based on your instance size and database characteristics. More jobs require more CPU cores and memory. A good baseline for `TABLE_JOBS` is fewer than the vCPU count of whichever is smaller — the SOURCE or TARGET. `INDEX_JOBS` should be fewer than the vCPUs on the TARGET. Exceeding these numbers can overwhelm the SOURCE during the COPY phase or the TARGET during index rebuilding. See [Cluster configuration parameters](https://planetscale.com/docs/postgres/cluster-configuration/parameters) for understanding target-side capacity. + +`SPLIT_TABLES_LARGER_THAN` must not change between a run and its resume. pgcopydb requires catalog consistency, so `resume-migration.sh` and `resume-cdc.sh` must use the same value as the original `run-migration.sh`. Set 0 to disable splitting. + +### Output plugin + +`OUTPUT_PLUGIN` selects the logical decoding plugin that pgcopydb uses for CDC. The default is `pgoutput`: + +- `pgoutput` is part of PostgreSQL core, so the source server needs no extension. +- On a mixed INSERT/UPDATE/DELETE workload it sends about 4.5x less network volume and uses about 4x less CPU on the source than `wal2json`. +- pgcopydb builds a publication from the table list in `FILTER_FILE`, so the source server does the filtering. `pgcopydb stream cleanup` drops that publication. -| Parameter | Default | Source | Description | -|-----------|---------|--------|-------------| -| `TABLE_JOBS` | 8 | `~/.env` | Parallel COPY workers | -| `INDEX_JOBS` | 6 | `~/.env` | Parallel index creation workers | -| `--split-tables-larger-than` | 50GB | script | Threshold for splitting large tables into parts | -| `--split-max-parts` | Same as TABLE_JOBS | script | Maximum number of parts per split table | +Set `OUTPUT_PLUGIN=wal2json` to use the previous plugin. `wal2json` and `test_decoding` remain supported, but `wal2json` must be installed on the source server. -Adjust these based on your instance size and database characteristics. More jobs require more CPU cores and memory. A good baseline for `TABLE_JOBS` is fewer than the vCPU count of whichever is smaller — the SOURCE or TARGET. `INDEX_JOBS` should be fewer than the vCPUs on the TARGET. Exceeding these numbers can overwhelm the SOURCE during the COPY phase or the TARGET during index rebuilding. See [Cluster configuration parameters](https://planetscale.com/docs/postgres/cluster-configuration/parameters) for understanding target-side capacity. +Do not change `OUTPUT_PLUGIN` in the middle of a migration. A resume must use the same plugin as the original run. ## Troubleshooting @@ -427,6 +467,7 @@ sqlite3 ~/migration_*/schema/filter.db "SELECT COUNT(*) FROM s_depend;" | Script | Phase | Description | |--------|-------|-------------| +| `env-template` | Prepare | Reference `~/.env`. Copy to `~/.env` and edit before you start | | `compare-pg-params.sh` | Prepare | Compare PostgreSQL parameters between source and target | | `preflight-check.sh` | Prepare | Validate migration prerequisites (connectivity, WAL level, permissions, slots, extension compatibility) | | `fix-replica-identity.sh` | Prepare | Set REPLICA IDENTITY FULL on tables without primary keys | @@ -442,7 +483,7 @@ sqlite3 ~/migration_*/schema/filter.db "SELECT COUNT(*) FROM s_depend;" | `resume-migration.sh` | Recovery | Resume an interrupted migration (full clone + CDC) | | `resume-cdc.sh` | Recovery | Resume only the CDC phase (skips clone) | | `target-clean.sh` | Recovery | Wipe target database for re-migration (prompts for confirmation) | -| `drop-replication-slots.sh` | Cleanup | Remove replication slots and origins | +| `drop-replication-slots.sh` | Cleanup | Remove replication slots, the pgoutput publication, and origins | | `stop-cdc.sh` | Cutover | Set CDC endpoint via SQLite to initiate cutover | | `verify-migration.sh` | Cutover | Verify schema and data consistency between source and target | diff --git a/pgcopydb-helpers/check-cdc-status.sh b/pgcopydb-helpers/check-cdc-status.sh index 863c649..2d68a45 100755 --- a/pgcopydb-helpers/check-cdc-status.sh +++ b/pgcopydb-helpers/check-cdc-status.sh @@ -9,6 +9,11 @@ set -euo pipefail # --- Load environment --- +if [ ! -f ~/.env ]; then + echo "ERROR: ~/.env not found. Create it from the template:" >&2 + echo " cp ~/env-template ~/.env && chmod 600 ~/.env" >&2 + exit 1 +fi set +u set -a source ~/.env diff --git a/pgcopydb-helpers/check-copy-stall.sh b/pgcopydb-helpers/check-copy-stall.sh index 7f2bc1f..5cba74a 100755 --- a/pgcopydb-helpers/check-copy-stall.sh +++ b/pgcopydb-helpers/check-copy-stall.sh @@ -42,6 +42,11 @@ while [ $# -gt 0 ]; do done # --- Load environment --- +if [ ! -f ~/.env ]; then + echo "ERROR: ~/.env not found. Create it from the template:" >&2 + echo " cp ~/env-template ~/.env && chmod 600 ~/.env" >&2 + exit 1 +fi set +u set -a # shellcheck disable=SC1090 diff --git a/pgcopydb-helpers/check-migration-status.sh b/pgcopydb-helpers/check-migration-status.sh index 8641c68..6382053 100644 --- a/pgcopydb-helpers/check-migration-status.sh +++ b/pgcopydb-helpers/check-migration-status.sh @@ -21,6 +21,11 @@ echo -e "${BLUE}╚════════════════════ echo "" # --- Load environment --- +if [ ! -f ~/.env ]; then + echo "ERROR: ~/.env not found. Create it from the template:" >&2 + echo " cp ~/env-template ~/.env && chmod 600 ~/.env" >&2 + exit 1 +fi set +u set -a source ~/.env diff --git a/pgcopydb-helpers/compare-pg-params.sh b/pgcopydb-helpers/compare-pg-params.sh index e1a4188..83f605c 100644 --- a/pgcopydb-helpers/compare-pg-params.sh +++ b/pgcopydb-helpers/compare-pg-params.sh @@ -11,6 +11,11 @@ set -euo pipefail # --- Load environment --- +if [ ! -f ~/.env ]; then + echo "ERROR: ~/.env not found. Create it from the template:" >&2 + echo " cp ~/env-template ~/.env && chmod 600 ~/.env" >&2 + exit 1 +fi set +u set -a source ~/.env diff --git a/pgcopydb-helpers/drop-replication-slots.sh b/pgcopydb-helpers/drop-replication-slots.sh index adf5e3f..03a7e3f 100755 --- a/pgcopydb-helpers/drop-replication-slots.sh +++ b/pgcopydb-helpers/drop-replication-slots.sh @@ -3,13 +3,22 @@ # Usage: ~/drop-replication-slots.sh [slot_name] # Example: ~/drop-replication-slots.sh pgcopydb # -# Cleans up pgcopydb replication artifacts: drops the replication slot on -# the source, the replication origin on the target, and the pgcopydb -# sentinel schema on the target. Defaults to slot/origin name "pgcopydb". +# Cleans up pgcopydb replication artifacts: drops the replication slot and +# the pgoutput publication on the source, the replication origin on the +# target, and the pgcopydb sentinel schema on the target. Defaults to the +# slot/origin/publication name "pgcopydb". +# +# pgcopydb creates the publication only when OUTPUT_PLUGIN is pgoutput. It +# names the publication after the replication slot. # set -e # --- Load environment --- +if [ ! -f ~/.env ]; then + echo "ERROR: ~/.env not found. Create it from the template:" >&2 + echo " cp ~/env-template ~/.env && chmod 600 ~/.env" >&2 + exit 1 +fi set +u set -a source ~/.env @@ -24,6 +33,7 @@ fi SLOT_NAME="${1:-pgcopydb}" ORIGIN_NAME="${1:-pgcopydb}" +PUBLICATION_NAME="${1:-pgcopydb}" echo "=== Cleaning up replication artifacts for slot/origin: $SLOT_NAME ===" echo "" @@ -54,6 +64,30 @@ fi echo "" +# --- SOURCE: drop the pgoutput publication --- +# pgcopydb creates this publication only for the pgoutput plugin, and names it +# after the replication slot. Nothing exists to drop for wal2json. +echo "--- Source: checking publication ---" +PUB_EXISTS=$(psql "$PGCOPYDB_SOURCE_PGURI" -t -A -c \ + "SELECT COUNT(*) FROM pg_publication WHERE pubname = '$PUBLICATION_NAME';") + +if [ "$PUB_EXISTS" -gt 0 ]; then + echo " Dropping publication '$PUBLICATION_NAME'..." + DROP_PUB_SQL=$(psql "$PGCOPYDB_SOURCE_PGURI" -t -A -c \ + "SELECT 'DROP PUBLICATION ' || quote_ident(pubname) || ';' + FROM pg_publication WHERE pubname = '$PUBLICATION_NAME';") + if psql "$PGCOPYDB_SOURCE_PGURI" -v ON_ERROR_STOP=1 -c "$DROP_PUB_SQL" > /dev/null; then + echo " Done." + else + echo " WARN: could not drop publication '$PUBLICATION_NAME'." + echo " The source user must own it. Drop it manually as the owner." + fi +else + echo " No publication '$PUBLICATION_NAME' found (already clean)." +fi + +echo "" + # --- TARGET: drop replication origin --- echo "--- Target: checking replication origin ---" ORIGIN_EXISTS=$(psql "$PGCOPYDB_TARGET_PGURI" -t -A -c \ diff --git a/pgcopydb-helpers/env-template b/pgcopydb-helpers/env-template new file mode 100644 index 0000000..dc37a6d --- /dev/null +++ b/pgcopydb-helpers/env-template @@ -0,0 +1,49 @@ +# pgcopydb migration environment +# +# Copy this file to ~/.env on the migration instance, then edit the values: +# +# cp ~/env-template ~/.env +# chmod 600 ~/.env +# +# Every helper script in this directory reads ~/.env. Change a value here once +# and all scripts use it. Do not commit a filled-in copy of this file. + +# ── Connection strings (required) ───────────────────────────────────────────── + +# Source database. +export PGCOPYDB_SOURCE_PGURI='postgresql://user:password@source-host:5432/dbname?sslmode=require' + +# Target database (PlanetScale). +export PGCOPYDB_TARGET_PGURI='postgresql://user:password@target-host.connect.psdb.cloud:5432/dbname?sslmode=require' + +# ── Parallelism ─────────────────────────────────────────────────────────────── + +# Parallel COPY workers. Keep this below the vCPU count of the smaller of the +# SOURCE and the TARGET. Also used as --split-max-parts. +export TABLE_JOBS=8 + +# Parallel index build workers. Keep this below the TARGET vCPU count. +export INDEX_JOBS=6 + +# ── Table splitting ─────────────────────────────────────────────────────────── + +# pgcopydb copies tables larger than this size in parts. Set 0 to disable. +# A resume must use the same value as the original run, because pgcopydb +# requires catalog consistency. +export SPLIT_TABLES_LARGER_THAN=50GB + +# ── Logical decoding ────────────────────────────────────────────────────────── + +# Output plugin for CDC. pgoutput is part of PostgreSQL core, so the source +# server needs no extension. Use wal2json only if the migration needs it. +export OUTPUT_PLUGIN=pgoutput + +# ── Filters ─────────────────────────────────────────────────────────────────── + +# Path to the pgcopydb filter file. See filters.ini for the syntax. +export FILTER_FILE=~/filters.ini + +# ── Slack alerts (optional) ─────────────────────────────────────────────────── + +# Webhook used by slack-migration-alerts.sh. Leave unset to disable alerts. +#export SLACK_WEBHOOK_URL='https://hooks.slack.com/services/...' diff --git a/pgcopydb-helpers/fix-replica-identity.sh b/pgcopydb-helpers/fix-replica-identity.sh index 40b2d21..571489a 100755 --- a/pgcopydb-helpers/fix-replica-identity.sh +++ b/pgcopydb-helpers/fix-replica-identity.sh @@ -9,6 +9,11 @@ # # --- Load environment --- +if [ ! -f ~/.env ]; then + echo "ERROR: ~/.env not found. Create it from the template:" >&2 + echo " cp ~/env-template ~/.env && chmod 600 ~/.env" >&2 + exit 1 +fi set +u set -a source ~/.env diff --git a/pgcopydb-helpers/preflight-check.sh b/pgcopydb-helpers/preflight-check.sh index c1a0566..bf361e3 100755 --- a/pgcopydb-helpers/preflight-check.sh +++ b/pgcopydb-helpers/preflight-check.sh @@ -21,6 +21,11 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "$SCRIPT_DIR/filters-lib.sh" # --- Load environment --- +if [ ! -f ~/.env ]; then + echo "ERROR: ~/.env not found. Create it from the template:" >&2 + echo " cp ~/env-template ~/.env && chmod 600 ~/.env" >&2 + exit 1 +fi set +u set -a source ~/.env @@ -81,8 +86,10 @@ tgt_query() { psql "$PGCOPYDB_TARGET_PGURI" -t -A -c "$1" 2>/dev/null || echo "" } -# ── Pre-parse filters.ini (scope needed for source permission checks) ─ -[ -f ~/filters.ini ] && parse_filters_ini ~/filters.ini +# ── Pre-parse the filter file (scope needed for source permission checks) ─ +# FILTER_FILE comes from ~/.env. See env-template. +FILTER_FILE="${FILTER_FILE:-$HOME/filters.ini}" +[ -f "$FILTER_FILE" ] && parse_filters_ini "$FILTER_FILE" # ══════════════════════════════════════════════════════════════════ NOW=$(date -u '+%Y-%m-%d %H:%M:%S UTC') @@ -319,13 +326,13 @@ echo "" echo " MIGRATION INSTANCE" echo " ────────────────────────────────────────────────────────────────" -# 14. filters.ini -if [ -f ~/filters.ini ]; then - pass "filters.ini" "~/filters.ini found" +# 14. filter file +if [ -f "$FILTER_FILE" ]; then + pass "filters.ini" "$FILTER_FILE found" _conflicts=$(filter_conflicts) [ -n "$_conflicts" ] && warn "filters.ini combination" "$_conflicts — pgcopydb disallows these together" else - fail "filters.ini" "~/filters.ini not found" + fail "filters.ini" "$FILTER_FILE not found" fi # 15. pgcopydb binary diff --git a/pgcopydb-helpers/resume-cdc.sh b/pgcopydb-helpers/resume-cdc.sh index 0bbec1c..487ee55 100755 --- a/pgcopydb-helpers/resume-cdc.sh +++ b/pgcopydb-helpers/resume-cdc.sh @@ -15,6 +15,11 @@ set -eo pipefail # --- Load environment --- +if [ ! -f ~/.env ]; then + echo "ERROR: ~/.env not found. Create it from the template:" >&2 + echo " cp ~/env-template ~/.env && chmod 600 ~/.env" >&2 + exit 1 +fi set +u set -a source ~/.env @@ -39,8 +44,13 @@ fi echo "Resuming CDC in: $MIGRATION_DIR" LOGFILE=$MIGRATION_DIR/migration.log -FILTER_FILE=~/filters.ini + +# Tunables come from ~/.env. See env-template for the full list. +# Keep these in sync with run-migration.sh — a resume must use the same values. +FILTER_FILE="${FILTER_FILE:-$HOME/filters.ini}" TABLE_JOBS="${TABLE_JOBS:-8}" +SPLIT_TABLES_LARGER_THAN="${SPLIT_TABLES_LARGER_THAN:-50GB}" +OUTPUT_PLUGIN="${OUTPUT_PLUGIN:-pgoutput}" cd "$MIGRATION_DIR" # Core dumps help debug rare native crashes; not required for a successful migrate. @@ -74,17 +84,19 @@ cp "$MIGRATION_DIR/schema/source.db" "$MIGRATION_DIR/schema/source.db.bak.$(date echo "==========================================" echo "Resuming CDC (follow only) at $(date)" echo "Migration dir: $MIGRATION_DIR" + echo "Plugin: $OUTPUT_PLUGIN | table-jobs: $TABLE_JOBS" + echo "Split tables larger than: $SPLIT_TABLES_LARGER_THAN | filter: $FILTER_FILE" echo "==========================================" "$PGCOPYDB_BIN" follow \ - --plugin wal2json \ + --plugin "$OUTPUT_PLUGIN" \ --resume \ --not-consistent \ --verbose \ --source "$PGCOPYDB_SOURCE_PGURI" \ --target "$PGCOPYDB_TARGET_PGURI" \ --filter "$FILTER_FILE" \ - --split-tables-larger-than 50GB \ + --split-tables-larger-than "$SPLIT_TABLES_LARGER_THAN" \ --split-max-parts "$TABLE_JOBS" \ --dir "$MIGRATION_DIR" diff --git a/pgcopydb-helpers/resume-migration.sh b/pgcopydb-helpers/resume-migration.sh index 32f6f9e..31277ca 100755 --- a/pgcopydb-helpers/resume-migration.sh +++ b/pgcopydb-helpers/resume-migration.sh @@ -14,6 +14,11 @@ set -eo pipefail # --- Load environment --- +if [ ! -f ~/.env ]; then + echo "ERROR: ~/.env not found. Create it from the template:" >&2 + echo " cp ~/env-template ~/.env && chmod 600 ~/.env" >&2 + exit 1 +fi set +u set -a source ~/.env @@ -53,9 +58,14 @@ fi echo "Resuming migration in: $MIGRATION_DIR" LOGFILE=$MIGRATION_DIR/migration.log -FILTER_FILE=~/filters.ini + +# Tunables come from ~/.env. See env-template for the full list. +# Keep these in sync with run-migration.sh — a resume must use the same values. +FILTER_FILE="${FILTER_FILE:-$HOME/filters.ini}" TABLE_JOBS="${TABLE_JOBS:-8}" INDEX_JOBS="${INDEX_JOBS:-6}" +SPLIT_TABLES_LARGER_THAN="${SPLIT_TABLES_LARGER_THAN:-50GB}" +OUTPUT_PLUGIN="${OUTPUT_PLUGIN:-pgoutput}" cd "$MIGRATION_DIR" # Core dumps help debug rare native crashes; not required for a successful migrate. @@ -75,11 +85,13 @@ cp "$MIGRATION_DIR/schema/source.db" "$MIGRATION_DIR/schema/source.db.bak.$(date echo "==========================================" echo "Resuming clone --follow at $(date)" echo "Migration dir: $MIGRATION_DIR" + echo "Plugin: $OUTPUT_PLUGIN | table-jobs: $TABLE_JOBS | index-jobs: $INDEX_JOBS" + echo "Split tables larger than: $SPLIT_TABLES_LARGER_THAN | filter: $FILTER_FILE" echo "==========================================" "$PGCOPYDB_BIN" clone \ --follow \ - --plugin wal2json \ + --plugin "$OUTPUT_PLUGIN" \ --resume \ --not-consistent \ --verbose \ @@ -91,7 +103,7 @@ cp "$MIGRATION_DIR/schema/source.db" "$MIGRATION_DIR/schema/source.db.bak.$(date --skip-db-properties \ --table-jobs "$TABLE_JOBS" \ --index-jobs "$INDEX_JOBS" \ - --split-tables-larger-than 50GB \ + --split-tables-larger-than "$SPLIT_TABLES_LARGER_THAN" \ --split-max-parts "$TABLE_JOBS" \ --dir "$MIGRATION_DIR" diff --git a/pgcopydb-helpers/run-migration.sh b/pgcopydb-helpers/run-migration.sh index 33e7ba8..fbd0c93 100644 --- a/pgcopydb-helpers/run-migration.sh +++ b/pgcopydb-helpers/run-migration.sh @@ -9,6 +9,11 @@ set -eo pipefail # --- Load environment --- +if [ ! -f ~/.env ]; then + echo "ERROR: ~/.env not found. Create it from the template:" >&2 + echo " cp ~/env-template ~/.env && chmod 600 ~/.env" >&2 + exit 1 +fi set +u set -a source ~/.env @@ -38,9 +43,13 @@ PGCOPYDB_BIN=$(find_pgcopydb) || { echo "ERROR: pgcopydb not found on PATH or un MIGRATION_DIR=~/migration_$(date +%Y%m%d-%H%M%S) LOGFILE=$MIGRATION_DIR/migration.log -FILTER_FILE=~/filters.ini + +# Tunables come from ~/.env. See env-template for the full list. +FILTER_FILE="${FILTER_FILE:-$HOME/filters.ini}" TABLE_JOBS="${TABLE_JOBS:-8}" INDEX_JOBS="${INDEX_JOBS:-6}" +SPLIT_TABLES_LARGER_THAN="${SPLIT_TABLES_LARGER_THAN:-50GB}" +OUTPUT_PLUGIN="${OUTPUT_PLUGIN:-pgoutput}" mkdir -p "$MIGRATION_DIR" cd "$MIGRATION_DIR" @@ -57,11 +66,13 @@ fi echo "" echo "==========================================" echo "Starting clone --follow at $(date)" + echo "Plugin: $OUTPUT_PLUGIN | table-jobs: $TABLE_JOBS | index-jobs: $INDEX_JOBS" + echo "Split tables larger than: $SPLIT_TABLES_LARGER_THAN | filter: $FILTER_FILE" echo "==========================================" "$PGCOPYDB_BIN" clone \ --follow \ - --plugin wal2json \ + --plugin "$OUTPUT_PLUGIN" \ --verbose \ --source "$PGCOPYDB_SOURCE_PGURI" \ --target "$PGCOPYDB_TARGET_PGURI" \ @@ -71,7 +82,7 @@ fi --skip-db-properties \ --table-jobs "$TABLE_JOBS" \ --index-jobs "$INDEX_JOBS" \ - --split-tables-larger-than 50GB \ + --split-tables-larger-than "$SPLIT_TABLES_LARGER_THAN" \ --split-max-parts "$TABLE_JOBS" \ --dir "$MIGRATION_DIR" diff --git a/pgcopydb-helpers/slack-migration-alerts.sh b/pgcopydb-helpers/slack-migration-alerts.sh index a0de528..5b8b65b 100755 --- a/pgcopydb-helpers/slack-migration-alerts.sh +++ b/pgcopydb-helpers/slack-migration-alerts.sh @@ -55,6 +55,11 @@ while [ $# -gt 0 ]; do done # ── Load environment ─────────────────────────────────────────────── +if [ ! -f ~/.env ]; then + echo "ERROR: ~/.env not found. Create it from the template:" >&2 + echo " cp ~/env-template ~/.env && chmod 600 ~/.env" >&2 + exit 1 +fi set +u set -a source ~/.env diff --git a/pgcopydb-helpers/stop-cdc.sh b/pgcopydb-helpers/stop-cdc.sh index ce1390b..e18d938 100644 --- a/pgcopydb-helpers/stop-cdc.sh +++ b/pgcopydb-helpers/stop-cdc.sh @@ -11,6 +11,11 @@ # --- Load environment --- +if [ ! -f ~/.env ]; then + echo "ERROR: ~/.env not found. Create it from the template:" >&2 + echo " cp ~/env-template ~/.env && chmod 600 ~/.env" >&2 + exit 1 +fi set +u set -a source ~/.env diff --git a/pgcopydb-helpers/target-clean.sh b/pgcopydb-helpers/target-clean.sh index 9f1ac9f..51ad92f 100644 --- a/pgcopydb-helpers/target-clean.sh +++ b/pgcopydb-helpers/target-clean.sh @@ -10,6 +10,11 @@ set -e # --- Load environment --- +if [ ! -f ~/.env ]; then + echo "ERROR: ~/.env not found. Create it from the template:" >&2 + echo " cp ~/env-template ~/.env && chmod 600 ~/.env" >&2 + exit 1 +fi set +u set -a source ~/.env diff --git a/pgcopydb-helpers/verify-migration.sh b/pgcopydb-helpers/verify-migration.sh index e5dae24..0667ae0 100755 --- a/pgcopydb-helpers/verify-migration.sh +++ b/pgcopydb-helpers/verify-migration.sh @@ -20,7 +20,8 @@ # --exact-count-tables Random tables to exact-count (default: 10, 0=skip) # --exact-count-max-gb Max table size in GB for exact count (default: 10) # --exact-count-timeout Per-table COUNT(*) timeout in seconds (default: 120) -# --filters Path to filters.ini (default: ~/filters.ini). Required — +# --filters Path to filters.ini (default: FILTER_FILE from ~/.env, +# or ~/filters.ini). Required — # verify scopes to the same object set the migration used. # # Exit codes: @@ -55,7 +56,9 @@ trap 'rm -f "$ERRFILE"' EXIT # ── Load connection strings from ~/.env ─────────────────────────────────────── if [[ ! -f ~/.env ]]; then - echo "ERROR: ~/.env not found. Create it with PGCOPYDB_SOURCE_PGURI and PGCOPYDB_TARGET_PGURI." >&2 + echo "ERROR: ~/.env not found. Create it from the template:" >&2 + echo " cp ~/env-template ~/.env && chmod 600 ~/.env" >&2 + echo "Then set PGCOPYDB_SOURCE_PGURI and PGCOPYDB_TARGET_PGURI in ~/.env." >&2 exit 1 fi set +u @@ -94,7 +97,9 @@ NO_SPOT_CHECK=false EXACT_COUNT_N=10 # number of random tables to exact-count EXACT_COUNT_MAX_GB=10 # tables larger than this are skipped EXACT_COUNT_TIMEOUT=120 -FILTERS_FILE=~/filters.ini # scope checks to objects this filter migrates +# Scope checks to objects this filter migrates. FILTER_FILE comes from ~/.env +# (see env-template); --filters overrides it. +FILTERS_FILE="${FILTER_FILE:-$HOME/filters.ini}" # ── Helpers ─────────────────────────────────────────────────────────────────── usage() { diff --git a/pgcopydb-templates/README.md b/pgcopydb-templates/README.md index 9a01616..59446d3 100644 --- a/pgcopydb-templates/README.md +++ b/pgcopydb-templates/README.md @@ -17,8 +17,19 @@ All three templates produce an equivalent migration instance — choose based on - A compute instance with pgcopydb and PostgreSQL 18 client tools - An attached data volume for migration working data - Network and access configuration (security group/firewall rule, IAM/SSH via SSM or IAP) -- Migration helper scripts from this repo deployed to `/home/ubuntu/` +- Migration helper scripts from this repo deployed to `/home/ubuntu/`, including `env-template` + +The templates do not create `~/.env`. The helper scripts own that file, and `env-template` is its reference copy. ## After Provisioning -Once the instance is running, connect to it and follow the [migration workflow](../pgcopydb-helpers/README.md#migration-workflow) starting with setting up `~/.env` and `~/filters.ini`. +Connect to the instance, then create the configuration file that every helper script reads: + +```bash +cp ~/env-template ~/.env +chmod 600 ~/.env +``` + +Edit `~/.env` and set `PGCOPYDB_SOURCE_PGURI` and `PGCOPYDB_TARGET_PGURI`. The other settings have working defaults — see [Script Configuration](../pgcopydb-helpers/README.md#script-configuration). + +Then customize `~/filters.ini` and follow the [migration workflow](../pgcopydb-helpers/README.md#migration-workflow). diff --git a/pgcopydb-templates/aws-cloudformation/pgcopydb-migration-instance.yaml b/pgcopydb-templates/aws-cloudformation/pgcopydb-migration-instance.yaml index 5ef0b3a..22d2960 100644 --- a/pgcopydb-templates/aws-cloudformation/pgcopydb-migration-instance.yaml +++ b/pgcopydb-templates/aws-cloudformation/pgcopydb-migration-instance.yaml @@ -346,23 +346,6 @@ Resources: mode: '000644' owner: root group: root - /home/ubuntu/.env: - content: | - # PlanetScale Migration Environment Variables - # Edit these values before running the migration - - # Source Database - PGCOPYDB_SOURCE_PGURI="postgresql://user:password@source-host:5432/dbname?sslmode=require" - - # Target Database (PlanetScale) - PGCOPYDB_TARGET_PGURI="postgresql://user:password@target-host.connect.psdb.cloud:5432/dbname?sslmode=require" - - # Parallelism tuning (used by run-migration.sh, resume-migration.sh, resume-cdc.sh) - TABLE_JOBS=8 - INDEX_JOBS=6 - mode: '000600' - owner: ubuntu - group: ubuntu commands: 02_clone_migration_scripts: command: | diff --git a/pgcopydb-templates/aws-terraform/user-data.sh b/pgcopydb-templates/aws-terraform/user-data.sh index b7085fe..b987424 100644 --- a/pgcopydb-templates/aws-terraform/user-data.sh +++ b/pgcopydb-templates/aws-terraform/user-data.sh @@ -112,24 +112,6 @@ alias psql-version='psql --version' alias check-planetscale='nc -zv app.connect.psdb.cloud 443 2>&1 | grep succeeded' PROFILE_EOF -# .env file -cat > /home/ubuntu/.env << 'ENV_EOF' -# PlanetScale Migration Environment Variables -# Edit these values before running the migration - -# Source Database -PGCOPYDB_SOURCE_PGURI="postgresql://user:password@source-host:5432/dbname?sslmode=require" - -# Target Database (PlanetScale) -PGCOPYDB_TARGET_PGURI="postgresql://user:password@target-host.connect.psdb.cloud:5432/dbname?sslmode=require" - -# Parallelism tuning (used by run-migration.sh, resume-migration.sh, resume-cdc.sh) -TABLE_JOBS=8 -INDEX_JOBS=6 -ENV_EOF -chmod 600 /home/ubuntu/.env -chown ubuntu:ubuntu /home/ubuntu/.env - # ============================================================================= # Pull PlanetScale Migration Helper Scripts # ============================================================================= diff --git a/pgcopydb-templates/gcp-terraform/startup-script.sh b/pgcopydb-templates/gcp-terraform/startup-script.sh index 476adef..a42585e 100644 --- a/pgcopydb-templates/gcp-terraform/startup-script.sh +++ b/pgcopydb-templates/gcp-terraform/startup-script.sh @@ -96,24 +96,6 @@ alias psql-version='psql --version' alias check-planetscale='nc -zv app.connect.psdb.cloud 443 2>&1 | grep succeeded' PROFILE_EOF -# .env file -cat > /home/ubuntu/.env << 'ENV_EOF' -# PlanetScale Migration Environment Variables -# Edit these values before running the migration - -# Source Database -PGCOPYDB_SOURCE_PGURI="postgresql://user:password@source-host:5432/dbname?sslmode=require" - -# Target Database (PlanetScale) -PGCOPYDB_TARGET_PGURI="postgresql://user:password@target-host.connect.psdb.cloud:5432/dbname?sslmode=require" - -# Parallelism tuning (used by run-migration.sh, resume-migration.sh, resume-cdc.sh) -TABLE_JOBS=8 -INDEX_JOBS=6 -ENV_EOF -chmod 600 /home/ubuntu/.env -chown ubuntu:ubuntu /home/ubuntu/.env - # Pull PlanetScale migration helper scripts echo "Cloning PlanetScale migration helper scripts..." git clone --depth 1 https://github.com/planetscale/migration-scripts.git /tmp/migration-scripts