diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eb5cc12..5fd4dd5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,12 @@ on: jobs: lint: + # A release (release-builder's PR, and its chore(release) commit on master) + # only bumps the version and CHANGELOG: nothing to test. + if: >- + !(github.event_name == 'pull_request' && github.head_ref == 'release' + && github.event.pull_request.user.login == 'github-actions[bot]') + && !(github.event_name == 'push' && startsWith(github.event.head_commit.message, 'chore(release):')) runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -24,6 +30,12 @@ jobs: run: script/lint test: + # A release (release-builder's PR, and its chore(release) commit on master) + # only bumps the version and CHANGELOG: nothing to test. + if: >- + !(github.event_name == 'pull_request' && github.head_ref == 'release' + && github.event.pull_request.user.login == 'github-actions[bot]') + && !(github.event_name == 'push' && startsWith(github.event.head_commit.message, 'chore(release):')) runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -42,9 +54,22 @@ jobs: # Real containers: a local registry, stub services, and an sshd, driven by # flow-deploy as a subprocess. Kept out of the unit job's coverage gate. + # Sharded across runners (pytest-split, balanced by the recorded + # tests/integration/.test_durations); locally, run it with -n auto instead. integration: + # A release (release-builder's PR, and its chore(release) commit on master) + # only bumps the version and CHANGELOG: nothing to test. + if: >- + !(github.event_name == 'pull_request' && github.head_ref == 'release' + && github.event.pull_request.user.login == 'github-actions[bot]') + && !(github.event_name == 'push' && startsWith(github.event.head_commit.message, 'chore(release):')) + name: integration (${{ matrix.group }}/4) runs-on: ubuntu-latest timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + group: [1, 2, 3, 4] steps: - uses: actions/checkout@v4 @@ -58,4 +83,8 @@ jobs: poetry install --with test - name: Integration tests - run: poetry run pytest -m integration --no-cov -p no:randomly + run: >- + poetry run pytest -m integration --no-cov -p no:randomly + --splits 4 --group ${{ matrix.group }} + --splitting-algorithm least_duration + --durations-path tests/integration/.test_durations diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bb94dba..69579e0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -65,7 +65,13 @@ jobs: dist/flow-deploy-linux-glibc build_and_deploy: - if: inputs.force_deploy || needs.release.outputs.created-tag + # build_release is skipped on a force_deploy run that cut no release; a + # plain `if:` implies success(), which would skip this job with it. + if: >- + !cancelled() + && needs.release.result == 'success' + && (needs.build_release.result == 'success' || needs.build_release.result == 'skipped') + && (inputs.force_deploy || needs.release.outputs.created-tag) name: Build & Deploy runs-on: ubuntu-latest needs: [release, build_release] @@ -83,12 +89,30 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + # Deploy by the release's version tag (the git tag build_release made, + # v), or by the commit SHA on a force_deploy run with no release. + - id: deploy_tag + name: Choose the deploy tag + env: + CREATED_TAG: ${{ needs.release.outputs.created-tag }} + VERSION_TAG: v${{ needs.release.outputs.current-version }} + run: | + if [ -n "$CREATED_TAG" ]; then + echo "tag=$VERSION_TAG" >> "$GITHUB_OUTPUT" + else + echo "tag=$GITHUB_SHA" >> "$GITHUB_OUTPUT" + fi + + # The image is pushed under every name a deploy or a revert may use: + # the version tag (releases only), and the short and full SHA. - id: meta uses: docker/metadata-action@v5 with: images: ${{ env.IMAGE }} tags: | + type=raw,value=v${{ needs.release.outputs.current-version }},enable=${{ needs.release.outputs.created-tag != '' }} type=sha,prefix= + type=sha,format=long,prefix= - uses: docker/build-push-action@v6 with: @@ -98,7 +122,7 @@ jobs: - uses: flowcanon/deploy-action@master with: - tag: ${{ steps.meta.outputs.version }} + tag: ${{ steps.deploy_tag.outputs.tag }} ssh-key: ${{ secrets.DEPLOY_SSH_KEY }} host: ${{ vars.HOST_NAME }} user: ${{ vars.HOST_USER }} diff --git a/poetry.lock b/poetry.lock index 5a7cdbd..276e5d0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -244,6 +244,21 @@ typing-extensions = {version = ">=4.6.0", markers = "python_version < \"3.13\""} [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "execnet" +version = "2.1.2" +description = "execnet: rapid multi-Python deployment" +optional = false +python-versions = ">=3.8" +groups = ["test"] +files = [ + {file = "execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec"}, + {file = "execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd"}, +] + +[package.extras] +testing = ["hatch", "pre-commit", "pytest", "tox"] + [[package]] name = "flake8" version = "7.3.0" @@ -502,6 +517,42 @@ files = [ [package.dependencies] pytest = "*" +[[package]] +name = "pytest-split" +version = "0.10.0" +description = "Pytest plugin which splits the test suite to equally sized sub suites based on test execution time." +optional = false +python-versions = "<4.0,>=3.8.1" +groups = ["test"] +files = [ + {file = "pytest_split-0.10.0-py3-none-any.whl", hash = "sha256:466096b086a7147bcd423c6e6c2e57fc62af1c5ea2e256b4ed50fc030fc3dddc"}, + {file = "pytest_split-0.10.0.tar.gz", hash = "sha256:adf80ba9fef7be89500d571e705b4f963dfa05038edf35e4925817e6b34ea66f"}, +] + +[package.dependencies] +pytest = ">=5,<9" + +[[package]] +name = "pytest-xdist" +version = "3.8.0" +description = "pytest xdist plugin for distributed testing, most importantly across multiple CPUs" +optional = false +python-versions = ">=3.9" +groups = ["test"] +files = [ + {file = "pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88"}, + {file = "pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1"}, +] + +[package.dependencies] +execnet = ">=2.1" +pytest = ">=7.0.0" + +[package.extras] +psutil = ["psutil (>=3.0)"] +setproctitle = ["setproctitle"] +testing = ["filelock"] + [[package]] name = "pyyaml" version = "6.0.3" @@ -659,4 +710,4 @@ files = [ [metadata] lock-version = "2.1" python-versions = "^3.10" -content-hash = "3d199a4a1005de1cdc263fb70366ccb77e09d983097a69c512341c4e29cbcead" +content-hash = "ba22d138365576bb50d1540735796f41744f2e1248242bbf1af01641837f924b" diff --git a/pyproject.toml b/pyproject.toml index 13eabb9..6263482 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,8 @@ autoflake = "^2.3" pytest = "^8.0" pytest-cov = "^6.0" pytest-randomly = "^3.16" +pytest-xdist = "^3.6" +pytest-split = "^0.10" [tool.poetry.scripts] flow-deploy = "flow_deploy.cli:main" @@ -41,7 +43,7 @@ line_length = 100 [tool.pytest.ini_options] testpaths = ["tests"] -# Integration tests (real Docker) run separately: pytest -m integration --no-cov +# Integration tests (real Docker) run separately, in parallel: pytest -m integration --no-cov -n auto addopts = "-v --tb=short --cov=flow_deploy --cov-report=term-missing -m 'not integration'" markers = ["integration: needs a real Docker daemon (tests/integration)"] diff --git a/tests/integration/.test_durations b/tests/integration/.test_durations new file mode 100644 index 0000000..1839840 --- /dev/null +++ b/tests/integration/.test_durations @@ -0,0 +1,35 @@ +{ + "tests/integration/test_ci_cancel.py::test_with_a_forced_pty_the_remote_deploy_gets_sighup": 7.867385687306523, + "tests/integration/test_ci_cancel.py::test_without_a_pty_the_remote_deploy_gets_no_signal_and_runs_on[SIGKILL]": 8.759982107207179, + "tests/integration/test_ci_cancel.py::test_without_a_pty_the_remote_deploy_gets_no_signal_and_runs_on[SIGTERM]": 7.4407060500234365, + "tests/integration/test_deploys.py::test_a_clean_deploy_warns_about_nothing": 8.96708051301539, + "tests/integration/test_deploys.py::test_a_revert_waits_for_restored_services_and_exits_3_if_unhealthy": 11.525916080921888, + "tests/integration/test_deploys.py::test_annotated_version_tag_deploy": 9.19166273996234, + "tests/integration/test_deploys.py::test_another_project_with_other_services_in_the_directory_is_fine": 10.586324241012335, + "tests/integration/test_deploys.py::test_bad_tag_fails_at_pull_before_anything_stops": 3.823063874617219, + "tests/integration/test_deploys.py::test_containers_of_another_project_name_here_are_refused": 5.549123642966151, + "tests/integration/test_deploys.py::test_crash_looping_drain_first_worker_is_drained": 11.324078911915421, + "tests/integration/test_deploys.py::test_digest_pinned_worker_is_restored_by_the_previous_commit": 11.196486052125692, + "tests/integration/test_deploys.py::test_discovery_works_without_the_apps_env": 0.28856269642710686, + "tests/integration/test_deploys.py::test_drain_first_service_fails_its_healthcheck_in_phase_3": 16.68825369887054, + "tests/integration/test_deploys.py::test_drain_that_hits_its_timeout_is_killed_and_the_deploy_continues": 12.823152102530003, + "tests/integration/test_deploys.py::test_every_deploy_leaves_a_host_side_log": 23.279017873108387, + "tests/integration/test_deploys.py::test_failed_deploy_restores_the_version_that_was_running[annotated]": 23.906655395403504, + "tests/integration/test_deploys.py::test_failed_deploy_restores_the_version_that_was_running[lightweight]": 24.169602466747165, + "tests/integration/test_deploys.py::test_first_deploy_from_latest_restores_by_the_previous_commit": 17.861694214865565, + "tests/integration/test_deploys.py::test_first_deploy_of_a_new_drain_first_service": 8.649966936558485, + "tests/integration/test_deploys.py::test_flow_deploy_never_leaves_the_wrapper_to_find_deploy_tag": 10.473964996635914, + "tests/integration/test_deploys.py::test_happy_path": 13.902889674529433, + "tests/integration/test_deploys.py::test_lock_held_by_a_live_process_exits_2_and_changes_nothing": 3.1837950069457293, + "tests/integration/test_deploys.py::test_manual_compose_keeps_the_deployed_version": 10.220468347892165, + "tests/integration/test_deploys.py::test_manual_compose_refuses_when_nothing_tells_the_tag": 0.3125146795064211, + "tests/integration/test_deploys.py::test_moved_tag_deploys_its_new_commit_and_image": 15.277251683175564, + "tests/integration/test_deploys.py::test_rolling_cutover_that_has_to_kill_warns": 7.621987188234925, + "tests/integration/test_deploys.py::test_rolling_healthcheck_that_passes_late_still_deploys": 13.103993121534586, + "tests/integration/test_deploys.py::test_rolling_only_project_keeps_its_old_container_on_a_failed_healthcheck": 6.816878102719784, + "tests/integration/test_deploys.py::test_rolling_service_fails_its_healthcheck": 11.677399668842554, + "tests/integration/test_deploys.py::test_sha_deploy": 9.240587916225195, + "tests/integration/test_deploys.py::test_signal_mid_drain_reverts[SIGINT-to-process-group]": 16.096034040674567, + "tests/integration/test_deploys.py::test_signal_mid_drain_reverts[SIGINT]": 16.133966751396656, + "tests/integration/test_deploys.py::test_signal_mid_drain_reverts[SIGTERM]": 16.09249211475253 +} \ No newline at end of file diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 4c9d389..d49b0ec 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -5,10 +5,13 @@ compose project of stub services pulled from a local registry. flow-deploy runs as a real subprocess, so tests can signal it, against real Docker. -Run with: poetry run pytest -m integration --no-cov +Run with: poetry run pytest -m integration --no-cov -n auto +(-n auto runs the tests in parallel, one per core; drop it to run them in order.) Skipped automatically when Docker isn't available. """ +import contextlib +import fcntl import json import os import shutil @@ -55,9 +58,11 @@ def docker(): pytest.skip("Docker Compose is too old: `docker compose up --wait` is required") -@pytest.fixture(scope="session") -def registry(docker): - """A local registry:2 with the stub image pushed under each STUB_TAGS tag.""" +def _start_registry() -> tuple[str, str]: + """Start a local registry:2 and push the stub image under each STUB_TAGS tag. + + Returns (host, container name). + """ name = f"flowdeploy-it-registry-{uuid.uuid4().hex[:8]}" run(["docker", "run", "-d", "--name", name, "-p", "127.0.0.1::5000", "registry:2"]) try: @@ -75,9 +80,57 @@ def registry(docker): # :latest starts out as v1, as a host first brought up by hand would be. run(["docker", "tag", f"{host}/stub:v1", f"{host}/stub:latest"]) run(["docker", "push", "-q", f"{host}/stub:latest"]) - yield host - finally: + except BaseException: run(["docker", "rm", "-f", name], check=False) + raise + return host, name + + +@contextlib.contextmanager +def _locked(path: Path): + with open(path, "a") as f: + fcntl.flock(f, fcntl.LOCK_EX) + yield + + +@pytest.fixture(scope="session") +def registry(docker, request, tmp_path_factory): + """A local registry with the stub images, shared by every test. + + Under pytest-xdist (`-n auto`) every worker shares one: the first to need + it starts it, and the last to finish removes it. Tests only push tags + unique to them (per-stack commit SHAs, pid-stamped names), so sharing is + safe. + """ + if not hasattr(request.config, "workerinput"): # not under xdist + host, name = _start_registry() + try: + yield host + finally: + run(["docker", "rm", "-f", name], check=False) + return + + shared = tmp_path_factory.getbasetemp().parent # the same for every worker + state = shared / "registry.json" + with _locked(shared / "registry.lock"): + if state.exists(): + data = json.loads(state.read_text()) + else: + host, name = _start_registry() + data = {"host": host, "name": name, "users": 0} + data["users"] += 1 + state.write_text(json.dumps(data)) + try: + yield data["host"] + finally: + with _locked(shared / "registry.lock"): + data = json.loads(state.read_text()) + data["users"] -= 1 + if data["users"] == 0: + run(["docker", "rm", "-f", data["name"]], check=False) + state.unlink() + else: + state.write_text(json.dumps(data)) def service( @@ -137,7 +190,9 @@ def __init__(self, root: Path, registry: str): self.repo = root / "repo" run(["git", "init", "-q", "--bare", str(self.origin)]) run(["git", "init", "-q", "-b", "main", str(self.repo)]) - self.git("config", "user.email", "it@example.com") + # A per-stack author makes every commit SHA unique, so stacks pushing + # images under their SHAs can't collide in the shared registry. + self.git("config", "user.email", f"it-{self.project}@example.com") self.git("config", "user.name", "it") self.git("config", "commit.gpgsign", "false") self.git("remote", "add", "origin", str(self.origin)) @@ -165,7 +220,19 @@ def commit(self, tag, compose, annotated=False, force=False): """Commit ``compose`` as docker-compose.yml, tag it ``tag`` (unless None), push. ``force`` moves an existing tag, here and on origin, as a re-release does. + + Every service gets ``network_mode: none``: the stubs never talk to each + other (health checks run inside each container), and without it each + stack's default network uses up one of Docker's address pools, which + run out at a few dozen, well short of running in parallel. """ + compose = { + **compose, + "services": { + name: {"network_mode": "none", **svc} + for name, svc in compose.get("services", {}).items() + }, + } (self.repo / "docker-compose.yml").write_text(yaml.safe_dump(compose, sort_keys=False)) self.git("add", "docker-compose.yml") self.git("commit", "-q", "--allow-empty", "-m", tag or "untagged") diff --git a/tests/integration/test_ci_cancel.py b/tests/integration/test_ci_cancel.py index c36a313..9c737d6 100644 --- a/tests/integration/test_ci_cancel.py +++ b/tests/integration/test_ci_cancel.py @@ -21,7 +21,7 @@ from tests.integration.conftest import IMAGES, SRC, run -PROBE_SECONDS = 6 +PROBE_SECONDS = 4 # killed ~1s in; long enough to tell running on from dead @pytest.fixture(scope="module") diff --git a/tests/integration/test_deploys.py b/tests/integration/test_deploys.py index be41034..32e12f0 100644 --- a/tests/integration/test_deploys.py +++ b/tests/integration/test_deploys.py @@ -137,6 +137,9 @@ def test_drain_that_hits_its_timeout_is_killed_and_the_deploy_continues(stack): assert "didn't exit on SIGTERM within 3s and was killed" in result.stderr +SIGTERM_DELAY = 4 + + @pytest.mark.parametrize( ("signame", "to_group"), [("SIGINT", False), ("SIGTERM", False), ("SIGINT", True)], @@ -146,7 +149,9 @@ def test_signal_mid_drain_reverts(stack, signame, to_group): """to_group sends the signal to flow-deploy's whole process group, as a terminal's Ctrl-C does, so the docker stop child would get it too if it weren't in its own session.""" - slow = service(300, "drain-first", drain=30, env={"STUB_SIGTERM_DELAY": "10"}) + # The worker takes SIGTERM_DELAY to stop, so the signal (sent ~1s into the + # drain) lands while the drain's docker stop is still waiting. + slow = service(300, "drain-first", drain=30, env={"STUB_SIGTERM_DELAY": str(SIGTERM_DELAY)}) stack.commit("v1", standard(worker=slow)) stack.commit("v2", standard(worker=slow)) stack.up("v1") @@ -163,7 +168,7 @@ def test_signal_mid_drain_reverts(stack, signame, to_group): assert "finishing the drain" in out # Outlast the old worker's SIGTERM delay: a restore that only "saw" a # still-stopping container would leave nothing running once it exits. - time.sleep(12) + time.sleep(SIGTERM_DELAY + 2) assert_reverted(stack, ALL_V1, code, out) @@ -220,7 +225,7 @@ def test_rolling_healthcheck_that_passes_late_still_deploys(stack): """A slow starter needs a start_period: without one Docker marks it unhealthy after `retries` failed checks, and flow-deploy aborts on that.""" stack.commit("v1", standard()) - slow_web = service(100, env={"STUB_HEALTH": "after:4"}, start_period="15s") + slow_web = service(100, env={"STUB_HEALTH": "after:3"}, start_period="8s") stack.commit("v2", standard(web=slow_web)) stack.up("v1") @@ -266,12 +271,14 @@ def test_sha_deploy(stack): assert stack.flow("tag").stdout == f"{sha}\n" -def test_failed_deploy_restores_the_version_that_was_running(stack): +@pytest.mark.parametrize("annotated", [False, True], ids=["lightweight", "annotated"]) +def test_failed_deploy_restores_the_version_that_was_running(stack, annotated): """v1 -> v2 succeeds; v3 then fails in phase 3 and everything returns to - v2 (not v1, not :latest), shipped web included.""" - stack.commit("v1", standard()) - stack.commit("v2", standard()) - stack.commit("v3", standard(worker=BAD_WORKER)) + v2 (not v1, not :latest), shipped web included. Release workflows make + annotated tags, so both kinds restore by the version tag.""" + stack.commit("v1", standard(), annotated=annotated) + stack.commit("v2", standard(), annotated=annotated) + stack.commit("v3", standard(worker=BAD_WORKER), annotated=annotated) stack.up("v1") assert stack.deploy("v2").returncode == 0