From 0357b769060442298427b8d064136f5b06aab9f3 Mon Sep 17 00:00:00 2001 From: jammyfu <1559847+jammyfu@users.noreply.github.com> Date: Sun, 1 Nov 2026 17:08:48 +0800 Subject: [PATCH] fix(ci): daily-update validates data instead of running unit tests on live data; API/push retries --- .github/workflows/daily-update.yml | 16 ++- CURRENT_PLAN.md | 27 ++-- TODO_BACKLOG.md | 2 +- docs/DAILY_UPDATE.md | 13 +- docs/project-governance/CHANGELOG.md | 5 + docs/project-governance/DECISIONS.md | 8 ++ docs/project-governance/WORKLOG.md | 6 + tests/test_core.py | 7 +- tests/test_daily_robustness.py | 198 +++++++++++++++++++++++++++ tests/test_repo.py | 12 +- tools/data_checks.py | 95 +++++++++++++ tools/local_daily.sh | 2 +- tools/risers_core.py | 19 ++- tools/update_stars.py | 37 ++++- tools/verify.py | 17 ++- 15 files changed, 421 insertions(+), 43 deletions(-) create mode 100644 tests/test_daily_robustness.py create mode 100644 tools/data_checks.py diff --git a/.github/workflows/daily-update.yml b/.github/workflows/daily-update.yml index 39f0604..45ec225 100644 --- a/.github/workflows/daily-update.yml +++ b/.github/workflows/daily-update.yml @@ -81,7 +81,9 @@ jobs: run: python3 tools/update_stars.py --slot "$SLOT" - name: Verify if: steps.gate.outputs.run == 'true' - run: python3 tools/verify.py + # Data checks only (tools/data_checks.py): hard errors fail, drift against the GitHub API warns. + # The unit tests run in ci.yml on every pull request, not against live data here. + run: python3 tools/verify.py --daily - name: Commit and push if anything changed if: steps.gate.outputs.run == 'true' run: | @@ -93,4 +95,14 @@ jobs: git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git add data rankings latest README*.md docs/HOW_IT_WORKS*.md docs/DETAILS*.md git commit -m "chore(data): daily star update $(date -u +%F)" - git push origin HEAD:main + # main can move while the job runs (a merged PR): rebase onto it and retry instead of failing the day. + for attempt in 1 2 3 4 5; do + if git push origin HEAD:main; then + exit 0 + fi + echo "push rejected (attempt $attempt); rebasing onto origin/main" + sleep $((attempt * 5)) + git pull --rebase origin main || { git rebase --abort || true; echo "rebase conflict" >&2; exit 1; } + done + echo "could not push after 5 attempts" >&2 + exit 1 diff --git a/CURRENT_PLAN.md b/CURRENT_PLAN.md index 4b8abb7..8981d97 100644 --- a/CURRENT_PLAN.md +++ b/CURRENT_PLAN.md @@ -2,40 +2,39 @@ ## Plan ID -PLAN-003 +PLAN-004 ## Goal -Polish the generated README first screen and stop baking an update date into the badge URL; the date must come from `data/stats.json` and refresh on every daily-update run. +Stop the recurring daily-update failures at the root: the scheduled run must validate the data it generated (hard errors fail, GitHub API timing drift warns) instead of running the unit-test suite against live data, and must survive rate limits and a moving `main`. ## In Scope -- `tools/render_readme.py` and `tools/i18n.py` (README / HOW_IT_WORKS / DETAILS copy) -- Regenerated `README*.md` (and the other generated pages the renderer writes) -- Tests that the update badge is dynamic and the stamp text is filled from `stats.json` -- Concise first-screen copy: what the list is, the Actions schedule, how numbers are computed, how to use or contribute, links +- `tools/data_checks.py`, `tools/verify.py --daily`, `.github/workflows/daily-update.yml`, `tools/local_daily.sh` +- `tools/update_stars.py` API retry/backoff, `tools/risers_core.py` hard limit for history drift +- Tests in `tests/test_daily_robustness.py`; docs in `docs/DAILY_UPDATE.md` and governance files ## Out of Scope -- Ranking logic, categories, the ⚠ suspected-crack rules, `data/stats.json` numbers -- Secrets, the daily-update cron times, GEO files (`docs/FAQ.md`, `docs/FACTS.md`, `CITATION.cff`, social preview) -- Merging to `main` +- Ranking logic, README copy, cron times, secrets, merging to `main` ## Tasks -- [x] Replace the static `updated-YYYY--MM--DD` shields badge with a dynamic badge that reads `today_utc` from `data/stats.json` -- [x] Keep the stamp line filled from `stats.generated_at` on every `render_readme` / daily-update run -- [x] Add two first-screen FAQ items (how numbers are computed; how to use or contribute) linking existing `docs/FAQ.md` anchors -- [x] Add a test that templates contain no hard-coded update date and that the rendered stamp follows `generated_at` +- [x] Group the failed runs by cause (41122642510, 41235403615) +- [x] Add `tools/data_checks.py` with error and warning severities and `verify.py --daily` +- [x] Wire the workflow and `local_daily.sh` to `--daily`; unit tests stay in `ci.yml` +- [x] Retry 429 / rate-limited 403 / 5xx / network errors; retry the push after rebasing onto `main` +- [x] Regression tests for all of the above - [ ] Owner reviews the PR; merge only with approval ## Verification - `python3 tools/verify.py` +- `python3 tools/verify.py --daily` - `python3 tools/next_plan.py --check` ## Next Candidates -- After merge, confirm the next `daily-update` run commits data +- After merge, confirm the next scheduled `daily-update` run commits data - Thin Developer Tools / Design / Data new-repo boards: see `TODO_BACKLOG.md` - Acceleration column (this week vs the week before) diff --git a/TODO_BACKLOG.md b/TODO_BACKLOG.md index 87a45aa..7a4f908 100644 --- a/TODO_BACKLOG.md +++ b/TODO_BACKLOG.md @@ -2,7 +2,7 @@ ## Now -- [ ] After the history-slack PR is merged, confirm the next `daily-update` run commits data (the 2026-11-01 pattern of Verify blocking a 3-star `history_total` gap must not recur) +- [ ] After the daily-update reliability PR (PLAN-004) is merged, confirm the next scheduled `daily-update` run commits data (Verify must only fail on real data errors) - [ ] Confirm the first unattended cron runs after the merge: `cn-0000` (Beijing) and `pt-0000` (Los Angeles) folders appear under `rankings//` (`gh run list --workflow daily-update.yml`) - [ ] Developer Tools new-repo board: 15 of 20 entries were flagged ⚠ in the last run because star-farmed repos use `cli` / `developer-tools`; decide whether to add a quality signal (owner call) - [ ] Design and Data new-repo boards are thin (0-2 entries): widen with specific topics only (no generic ones; `tests/test_spam.py` guards that) diff --git a/docs/DAILY_UPDATE.md b/docs/DAILY_UPDATE.md index 3f235f1..91e8b36 100644 --- a/docs/DAILY_UPDATE.md +++ b/docs/DAILY_UPDATE.md @@ -85,8 +85,17 @@ would skip. The Beijing slot and the next day's runs still refresh the data, so ## Option A: GitHub Actions (preferred when it runs) -`.github/workflows/daily-update.yml` regenerates the data, runs `tools/verify.py`, and commits to `main` if anything changed. +`.github/workflows/daily-update.yml` regenerates the data, runs `tools/verify.py --daily`, and commits to `main` if anything changed. It uses only first-party `actions/*` and the built-in `GITHUB_TOKEN`. + +Why `--daily` (root cause of the 2026-10-30 and 2026-10-31 failures): the run used to execute the whole unit-test suite +against the data it had just fetched, so anything that moves with GitHub (a new repo without a hook yet, a 3-star gap between +`/stargazers/history` and `/repos`) failed the run and the day's update was lost. Now the daily run only validates the data +(`tools/data_checks.py`): **errors** fail (project list mismatch, negative or inconsistent growth, no snapshot for today, +history vs stars off by more than `max(50, 1 %)`, more than 25 % of repos fetched as missing) and **warnings** (history vs stars +off by more than `HISTORY_TOTAL_STAR_SLACK`, a few missing repos) show as `::warning::` annotations. The unit tests still run in +`ci.yml` on every pull request. API calls retry 5xx, network errors, `429` and rate-limited `403` with backoff (honouring +`Retry-After` / `X-RateLimit-Reset`, capped at 120 s); the push step rebases onto `main` and retries up to 5 times. `.github/workflows/ci.yml` runs `tools/verify.py` on every pull request and push to `main` (no network). Scheduled workflows run from the default branch only. @@ -115,7 +124,7 @@ gh api repos/jammyfu/github-risers/check-runs//annotations --jq '.[].mes ```bash GITHUB_TOKEN=$(gh auth token) python3 tools/update_stars.py # fetch + render only -tools/local_daily.sh # update, verify, commit, push to main (any time) +tools/local_daily.sh # update, verify (--daily), commit, push to main (any time) tools/local_daily.sh --no-push # update, verify, commit locally tools/local_daily.sh --slot beijing # only proceeds if it is 00:xx in Asia/Shanghai tools/local_daily.sh --slot la # only proceeds if it is 00:xx in America/Los_Angeles (DST-aware) diff --git a/docs/project-governance/CHANGELOG.md b/docs/project-governance/CHANGELOG.md index c5de926..0fc037f 100644 --- a/docs/project-governance/CHANGELOG.md +++ b/docs/project-governance/CHANGELOG.md @@ -1,5 +1,10 @@ # CHANGELOG.md +## 2026-11-01 (daily-update reliability) + +- The daily run no longer runs the unit tests against live data. `python3 tools/verify.py --daily` validates the generated data (`tools/data_checks.py`): real errors fail, GitHub API timing drift only warns. CI still runs every unit test on pull requests. +- API retries for rate limits (`429`, rate-limited `403`), 5xx and network errors with capped backoff; the push step rebases onto `main` and retries. + ## 2026-11-01 (README date and first screen) - The README `updated` badge is a dynamic shields.io JSON badge that reads `today_utc` from `data/stats.json`. The stamp line is filled from `generated_at` on every daily-update render. No update date is typed into `tools/i18n.py`. diff --git a/docs/project-governance/DECISIONS.md b/docs/project-governance/DECISIONS.md index 028437b..3118337 100644 --- a/docs/project-governance/DECISIONS.md +++ b/docs/project-governance/DECISIONS.md @@ -68,6 +68,14 @@ - The daily-update Verify step (`tests/test_repo.py`) still requires a full history sum to stay close to `stargazers_count`. The allowed absolute delta is now 5 (`tools/risers_core.py::HISTORY_TOTAL_STAR_SLACK`). - A 3-star gap is accepted; a gap larger than 5 still fails. The check is not removed. +## 2026-11-01 (daily-update root cause) + +### The daily run validates data; it does not run the unit tests against live data + +- Failures: run 41122642510 (2026-10-30 20:22Z, hook test: new repos without a hook) and run 41235403615 (2026-10-31 19:41Z, `history_total` vs `stars` off by 3). Each fix raised one limit; the next live value would have failed again, and a failed run discards the whole day's update. +- Decision: `tools/verify.py --daily` = compile + `tools/data_checks.py` (hard errors fail, API-timing drift warns) + README freshness. Unit tests stay in CI (`ci.yml`) on every PR. Real data errors still fail. +- Plus: `api_get` retries 429 / rate-limited 403 / 5xx / network errors with capped backoff; the push step rebases and retries. + ## 2026-11-01 (README update date) ### Do not bake the update date into the badge URL diff --git a/docs/project-governance/WORKLOG.md b/docs/project-governance/WORKLOG.md index 0298b8b..7c6eea0 100644 --- a/docs/project-governance/WORKLOG.md +++ b/docs/project-governance/WORKLOG.md @@ -59,6 +59,12 @@ - Gaps found: stale `CURRENT_PLAN.md`/`MASTER_PLAN.md`/`TODO_BACKLOG.md`/governance files; dirty-worktree and record-order rules only in chat; no check of the plan shape; no regression test that `verify.py` and `local_daily.sh` avoid PATH/`shell=True`. - Added `tools/next_plan.py --check`, `tests/test_harness.py`, and wrote the rules into `AGENTS.md` and `docs/`. +## 2026-11-01 — daily-update root cause (PLAN-004) + +- Failed runs: 41122642510 (hook completeness test on live data) and 41235403615 (`history_total` vs `stars` gap of 3). Common cause: Verify ran the full unit-test suite on freshly fetched data. +- Added `tools/data_checks.py`, `verify.py --daily`, rate-limit/5xx retry in `api_get`, push retry with rebase; tests in `tests/test_daily_robustness.py`. +- Verification: `python3 tools/verify.py` green; `python3 tools/verify.py --daily` green. + ## 2026-11-01 — README date and first-screen polish (PLAN-003) - Replaced the static `updated-YYYY--MM--DD` shields badge with a dynamic JSON badge that reads `$.today_utc` from `data/stats.json` on `main`. diff --git a/tests/test_core.py b/tests/test_core.py index bcc4302..671d9ea 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -56,9 +56,10 @@ def test_history_total_slack_accepts_a_three_star_gap(self): self.assertTrue(core.history_total_within_slack(100, 95)) self.assertTrue(core.history_total_within_slack(100, 100)) self.assertFalse(core.history_total_within_slack(100, 106)) - src = Path(__file__).resolve().parents[1].joinpath("tests/test_repo.py").read_text("utf-8") - self.assertIn("history_total_within_slack", src) - self.assertNotRegex(src, r"assertLessEqual\(abs\(p\[.history_total.\] - p\[.stars.\]\), 2") + src = Path(__file__).resolve().parents[1].joinpath("tools/data_checks.py").read_text("utf-8") + self.assertIn("history_total_within_slack", src) # the daily data check warns on this drift + repo_tests = Path(__file__).resolve().parents[1].joinpath("tests/test_repo.py").read_text("utf-8") + self.assertNotIn("history_total_within_slack", repo_tests) # drift must not fail a unit test class RankTests(unittest.TestCase): diff --git a/tests/test_daily_robustness.py b/tests/test_daily_robustness.py new file mode 100644 index 0000000..063f52b --- /dev/null +++ b/tests/test_daily_robustness.py @@ -0,0 +1,198 @@ +"""Regression tests for the recurring daily-update failures (runs 41122642510 and 41235403615). + +Root cause: the scheduled run verified by running the whole unit-test suite against live data, so any value that +moves with GitHub (hook gaps for new repos, history_total vs stars drift) failed the run and discarded the day's +update. The daily run now validates data only (tools/data_checks.py): hard errors fail, API timing drift warns. +It also retries rate limits / 5xx with backoff and retries the push when main moved. +""" +import copy +import io +import json +import sys +import unittest +import urllib.error +from pathlib import Path +from unittest import mock + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "tools")) +import data_checks # noqa: E402 +import risers_core as core # noqa: E402 +import update_stars # noqa: E402 +import verify # noqa: E402 + +WF = (ROOT / ".github/workflows/daily-update.yml").read_text("utf-8") + + +def growth(gained=10, days_=(7, 30, 365)): + return {k: {"window_days": d, "gained": gained, "effective_days": d, "per_day": gained / d, "young": False} + for k, d in zip(("week", "month", "year"), days_)} + + +def project(name, stars=1000, total=None, **kw): + p = {"name": name, "ok": True, "stars": stars, "growth": growth(), "history_complete": True, + "history_total": stars if total is None else total} + p.update(kw) + return p + + +def world(rows): + stats = {"today_utc": "2026-11-01", "projects": rows} + projects = [{"name": r["name"]} for r in rows] + snaps = {r["name"]: {"2026-11-01": r.get("stars")} for r in rows if r.get("ok")} + return stats, projects, snaps + + +class DataCheckTests(unittest.TestCase): + def check(self, rows): + return data_checks.check(*world(rows)) + + def test_clean_data_has_no_errors_or_warnings(self): + self.assertEqual(self.check([project("a/a"), project("b/b")]), ([], [])) + + def test_small_history_drift_warns_but_never_fails(self): + for drift in (3, 5, 9, 40): # run 41235403615: affaan-m/ECC was off by 3 + errors, warnings = self.check([project("a/a", total=1000 - drift) if drift > 5 + else project("a/a", total=1000 + drift)]) + self.assertEqual(errors, [], drift) + self.assertEqual(bool(warnings), drift > core.HISTORY_TOTAL_STAR_SLACK, drift) + + def test_drift_inside_slack_is_silent(self): + self.assertEqual(self.check([project("a/a", total=1003)]), ([], [])) + + def test_large_history_gap_is_a_real_error(self): + errors, _ = self.check([project("a/a", stars=1000, total=800)]) + self.assertTrue(any("history_total 800" in e for e in errors)) + big = project("b/b", stars=400000, total=400000 - 3000) # 1 % of 400k is 4000: still a warning + self.assertEqual(self.check([big])[0], []) + big = project("b/b", stars=400000, total=400000 - 5000) + self.assertTrue(self.check([big])[0]) + + def test_hard_limit_floor_and_share(self): + self.assertEqual(core.history_total_hard_limit(1000), 50) + self.assertEqual(core.history_total_hard_limit(400000), 4000) + + def test_real_data_errors_still_fail(self): + bad = project("a/a") + bad["growth"]["week"]["gained"] = -1 + self.assertTrue(self.check([bad])[0]) + bad = project("a/a") + bad["growth"]["month"]["per_day"] = 99 + self.assertTrue(self.check([bad])[0]) + bad = project("a/a") + del bad["growth"]["year"] + self.assertTrue(self.check([bad])[0]) + stats, projects, snaps = world([project("a/a")]) + snaps["a/a"] = {} + self.assertTrue(data_checks.check(stats, projects, snaps)[0]) # no snapshot for today + stats, projects, snaps = world([project("a/a")]) + self.assertTrue(data_checks.check(stats, projects + [{"name": "x/x"}], snaps)[0]) # project list mismatch + + def test_one_missing_repo_warns_an_outage_fails(self): + gone = {"name": "c/c", "ok": False, "error": "HTTP 404: Not Found"} + errors, warnings = self.check([project("a/a"), project("b/b"), project("d/d"), project("e/e"), gone]) + self.assertEqual(errors, []) + self.assertTrue(any("c/c" in w for w in warnings)) + rows = [project("a/a")] + [dict(gone, name=f"m/{i}") for i in range(3)] # 75 % missing: rate limit / outage + self.assertTrue(any("outage" in e for e in self.check(rows)[0])) + self.assertTrue(self.check([{"name": "c/c", "ok": False}, project("a/a"), project("b/b"), project("d/d"), + project("e/e")])[0]) # missing without an error message + + def test_committed_data_passes(self): + self.assertEqual(data_checks.main(ROOT), 0) + + def test_main_prints_actions_annotations(self): + out, err = io.StringIO(), io.StringIO() + with mock.patch("sys.stdout", out), mock.patch("sys.stderr", err): + self.assertEqual(data_checks.main(ROOT), 0) + self.assertIn("data check passed", out.getvalue()) + + +class VerifyModeTests(unittest.TestCase): + def joined(self, daily): + cmds = verify.commands(ROOT, daily=daily) + for c in cmds: + self.assertEqual(c[0], sys.executable) + return [" ".join(c[1:]) for c in cmds] + + def test_default_runs_the_unit_tests(self): + j = self.joined(False) + self.assertTrue(any("unittest discover" in x for x in j)) + self.assertFalse(any("data_checks" in x for x in j)) + + def test_daily_runs_data_checks_not_the_unit_tests(self): + j = self.joined(True) + self.assertFalse(any("unittest" in x for x in j)) + self.assertTrue(any("tools/data_checks.py" in x for x in j)) + self.assertTrue(any("render_readme.py --check" in x for x in j)) + + def test_daily_flag_reaches_commands(self): + with mock.patch.object(verify, "commands", return_value=[]) as c: + self.assertEqual(verify.main(["--daily"]), 0) + self.assertTrue(c.call_args.kwargs["daily"]) + verify.main([]) + self.assertFalse(c.call_args.kwargs["daily"]) + + +class WorkflowRobustnessTests(unittest.TestCase): + def test_scheduled_paths_verify_data_not_live_unit_tests(self): + self.assertIn("tools/verify.py --daily", WF) + self.assertNotIn("run: python3 tools/verify.py\n", WF) + sh = (ROOT / "tools/local_daily.sh").read_text("utf-8") + self.assertIn('"$PY" tools/verify.py --daily', sh) + ci = (ROOT / ".github/workflows/ci.yml").read_text("utf-8") + self.assertIn("python3 tools/verify.py\n", ci) # CI still runs every unit test + self.assertNotIn("--daily", ci) + + def test_runs_never_overlap_and_are_not_cancelled(self): + self.assertIn("group: daily-update", WF) + self.assertIn("cancel-in-progress: false", WF) + + def test_push_retries_after_rebasing_onto_main(self): + self.assertIn("for attempt in 1 2 3 4 5", WF) + self.assertIn("git pull --rebase origin main", WF) + self.assertLess(WF.index("git commit"), WF.index("git pull --rebase")) + + +class RetryTests(unittest.TestCase): + def http_error(self, code, headers=None): + return urllib.error.HTTPError("https://api.github.com/x", code, "err", headers or {}, io.BytesIO(b"{}")) + + def test_retry_wait_policy(self): + self.assertEqual(update_stars.retry_wait(503, {}, 0), 2) + self.assertEqual(update_stars.retry_wait(502, {}, 2), 8) + self.assertEqual(update_stars.retry_wait(429, {}, 0), 2) + self.assertEqual(update_stars.retry_wait(403, {"Retry-After": "10"}, 0), 11) + self.assertEqual(update_stars.retry_wait(403, {"Retry-After": "9999"}, 0), update_stars.MAX_WAIT) + with mock.patch("time.time", return_value=1000): + h = {"X-RateLimit-Remaining": "0", "X-RateLimit-Reset": "1030"} + self.assertEqual(update_stars.retry_wait(403, h, 0), 31) + self.assertIsNone(update_stars.retry_wait(403, {}, 0)) # plain forbidden is final + self.assertIsNone(update_stars.retry_wait(404, {}, 0)) + self.assertIsNone(update_stars.retry_wait(422, {}, 0)) + + def test_api_get_waits_out_a_rate_limit_then_succeeds(self): + ok = mock.MagicMock() + ok.__enter__.return_value.read.return_value = b'{"stargazers_count": 7}' + calls = [self.http_error(403, {"Retry-After": "3"}), self.http_error(503), ok] + with mock.patch("urllib.request.urlopen", side_effect=calls), mock.patch("time.sleep") as sleep: + self.assertEqual(update_stars.api_get("/repos/a/b", "t"), {"stargazers_count": 7}) + self.assertEqual([c.args[0] for c in sleep.call_args_list], [4, 4]) + + def test_api_get_gives_up_on_final_errors_without_sleeping(self): + with mock.patch("urllib.request.urlopen", side_effect=self.http_error(404)), mock.patch("time.sleep") as sleep: + with self.assertRaises(update_stars.GitHubError) as cm: + update_stars.api_get("/repos/a/gone", "t") + self.assertEqual(cm.exception.status, 404) + sleep.assert_not_called() + + def test_api_get_retries_network_errors_then_raises(self): + err = urllib.error.URLError("boom") + with mock.patch("urllib.request.urlopen", side_effect=err) as u, mock.patch("time.sleep"): + with self.assertRaises(update_stars.GitHubError): + update_stars.api_get("/x", "t", retries=3) + self.assertEqual(u.call_count, 3) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_repo.py b/tests/test_repo.py index 74477e6..3967c08 100644 --- a/tests/test_repo.py +++ b/tests/test_repo.py @@ -9,6 +9,7 @@ sys.path.insert(0, str(ROOT / "tools")) import risers_core as core # noqa: E402 import archive # noqa: E402 +import data_checks # noqa: E402 import render_readme # noqa: E402 from i18n import LANGS # noqa: E402 @@ -49,14 +50,9 @@ def test_stats_cover_every_project_and_are_consistent(self): self.assertGreaterEqual(g["gained"], 0) self.assertLessEqual(g["effective_days"], days) self.assertAlmostEqual(g["per_day"], g["gained"] / g["effective_days"], places=2) - if p["history_complete"]: - # full history from the API should add up to the current star count; - # GitHub allows a small edge discrepancy (removed stars / bucket boundaries) - self.assertTrue( - core.history_total_within_slack(p["history_total"], p["stars"]), - f"{p['name']}: |history_total {p['history_total']} - stars {p['stars']}| " - f"> {core.HISTORY_TOTAL_STAR_SLACK}", - ) + # live-data drift (history vs live stars) is a warning in tools/data_checks.py, never a unit-test failure + errors, _warnings = data_checks.check(stats, projects, json.loads((ROOT / "data/stars.json").read_text("utf-8"))) + self.assertEqual(errors, []) def test_snapshots_have_today(self): snaps = json.loads((ROOT / "data/stars.json").read_text("utf-8")) diff --git a/tools/data_checks.py b/tools/data_checks.py new file mode 100644 index 0000000..10744e1 --- /dev/null +++ b/tools/data_checks.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Validate the freshly generated data (data/stats.json, data/stars.json) after a daily update. + +The scheduled run used to "verify" by running the whole unit-test suite against live data. Any value that +drifts with GitHub (a few stars between /stargazers/history and /repos, a new repo without a hook yet) +then failed the run and threw away the whole day's update. That happened twice in three days +(runs 41122642510 and 41235403615). Code tests belong to CI; the daily run only validates the data, with +two severities: + +* errors - the data is wrong or incomplete: the run fails and nothing is committed. +* warnings - the data is fine but not perfect (small history/star drift, a few repos fetched as missing): + printed as GitHub Actions ``::warning::`` lines, the run stays green. + + python3 tools/data_checks.py # exit 1 only on errors +""" +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import risers_core as core # noqa: E402 + +ROOT = Path(__file__).resolve().parents[1] + +# Above this share of tracked projects fetched as missing the run is treated as an API outage / rate-limit +# problem: publishing it would replace real numbers with "missing", so it fails instead of committing. +MAX_MISSING_SHARE = 0.25 + + +def check(stats: dict[str, Any], projects: list[dict], snaps: dict[str, dict[str, int]]) -> tuple[list[str], list[str]]: + """Return (errors, warnings).""" + errors: list[str] = [] + warnings: list[str] = [] + rows = stats.get("projects") or [] + if [p.get("name") for p in rows] != [p["name"] for p in projects]: + errors.append("stats.json projects do not match data/projects.json (names or order)") + missing = [p for p in rows if not p.get("ok")] + if rows and len(missing) / len(rows) > MAX_MISSING_SHARE: + errors.append(f"{len(missing)}/{len(rows)} projects fetched as missing (> {MAX_MISSING_SHARE:.0%}): " + "looks like an API outage or rate limit; not publishing") + elif missing: + warnings.append(f"{len(missing)} project(s) fetched as missing: " + ", ".join(p["name"] for p in missing)) + for p in rows: + name = p.get("name") + if not p.get("ok"): + if not p.get("error"): + errors.append(f"{name}: missing without an error message") + continue + for key, days in core.WINDOWS.items(): + g = (p.get("growth") or {}).get(key) + if not g: + errors.append(f"{name}: no {key} growth") + continue + if g["window_days"] != days: + errors.append(f"{name}: {key} window_days {g['window_days']} != {days}") + if g["gained"] < 0: + errors.append(f"{name}: negative {key} gain {g['gained']}") + if g["effective_days"] > days: + errors.append(f"{name}: {key} effective_days {g['effective_days']} > {days}") + elif g["effective_days"] > 0 and abs(g["per_day"] - g["gained"] / g["effective_days"]) > 0.01: + errors.append(f"{name}: {key} per_day does not match gained/effective_days") + if p.get("history_complete") and p.get("history_total") is not None: + total, stars = p["history_total"], p["stars"] + if not core.history_total_within_hard_limit(total, stars): + errors.append(f"{name}: history_total {total} vs stars {stars} differ by more than " + f"{core.history_total_hard_limit(stars)}") + elif not core.history_total_within_slack(total, stars): + warnings.append(f"{name}: history_total {total} vs stars {stars} drift by {abs(total - stars)} " + f"(> {core.HISTORY_TOTAL_STAR_SLACK}; GitHub history lags the live count)") + if snaps.get(name, {}).get(stats.get("today_utc")) != p["stars"]: + errors.append(f"{name}: data/stars.json has no snapshot for {stats.get('today_utc')}") + return errors, warnings + + +def main(root: Path = ROOT) -> int: + stats = json.loads((root / "data/stats.json").read_text("utf-8")) + projects = json.loads((root / "data/projects.json").read_text("utf-8")) + snaps = json.loads((root / "data/stars.json").read_text("utf-8")) + errors, warnings = check(stats, projects, snaps) + for w in warnings: + print(f"::warning title=data check::{w}") + for e in errors: + print(f"::error title=data check::{e}", file=sys.stderr) + if errors: + print(f"data check failed: {len(errors)} error(s), {len(warnings)} warning(s)", file=sys.stderr) + return 1 + print(f"data check passed: {len(stats.get('projects') or [])} projects, {len(warnings)} warning(s)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/local_daily.sh b/tools/local_daily.sh index 0daba9c..ec8b082 100755 --- a/tools/local_daily.sh +++ b/tools/local_daily.sh @@ -66,7 +66,7 @@ git pull --ff-only export RISERS_DAILY_RUN=1 export I18N_POLISH="${I18N_POLISH:-1}" I18N_TRANSLATE_BACKEND="${I18N_TRANSLATE_BACKEND:-agy}" I18N_POLISH_BACKEND=agy "$PY" tools/update_stars.py --slot "$UPDATE_SLOT" -"$PY" tools/verify.py +"$PY" tools/verify.py --daily if [ -z "$(git status --porcelain)" ]; then echo "No changes." diff --git a/tools/risers_core.py b/tools/risers_core.py index 81ccddc..4312ced 100644 --- a/tools/risers_core.py +++ b/tools/risers_core.py @@ -14,12 +14,27 @@ # Allowed |history_total - stargazers_count| when the full history was fetched. # /stargazers/history buckets by week and can lag the live count by a few stars -# (un-stars, same-day stars, UTC vs bucket boundaries). Slack was 2 until -# 2026-11-01, when affaan-m/ECC was off by 3 and blocked the daily push +# (un-stars, same-day stars, UTC vs bucket boundaries). Over this the daily data check only WARNS +# (tools/data_checks.py); it fails above history_total_hard_limit(). Slack was 2 until 2026-11-01, when +# affaan-m/ECC was off by 3 and blocked the daily push # (https://github.com/jammyfu/github-risers/actions/runs/41235403615). HISTORY_TOTAL_STAR_SLACK = 5 +# Beyond this the history and the star count disagree for a real reason (truncated pagination, wrong repo), +# not API timing: the daily data check fails instead of warning. Max of an absolute floor and 1 % of the stars. +HISTORY_TOTAL_HARD_FLOOR = 50 +HISTORY_TOTAL_HARD_SHARE = 0.01 + + +def history_total_hard_limit(stars: int) -> int: + return max(HISTORY_TOTAL_HARD_FLOOR, int(int(stars) * HISTORY_TOTAL_HARD_SHARE)) + + +def history_total_within_hard_limit(history_total: int, stars: int) -> bool: + return abs(int(history_total) - int(stars)) <= history_total_hard_limit(stars) + + def history_total_within_slack(history_total: int, stars: int) -> bool: """True when the summed history is close enough to the live star count.""" return abs(int(history_total) - int(stars)) <= HISTORY_TOTAL_STAR_SLACK diff --git a/tools/update_stars.py b/tools/update_stars.py index eaf8aa3..3d785ea 100755 --- a/tools/update_stars.py +++ b/tools/update_stars.py @@ -41,7 +41,31 @@ def __init__(self, status: int, message: str): self.status = status -def api_get(path: str, token: str, retries: int = 3) -> Any: +RETRY_STATUS = (500, 502, 503, 504) +MAX_WAIT = 120 # never sleep longer than this for one retry (the job has a 45 minute budget) + + +def retry_wait(code: int, headers: Any, attempt: int) -> float | None: + """Seconds to wait before retrying a failed request, or None when the failure is final. + + 5xx: exponential backoff. 429 and 403 with a rate-limit signal (Retry-After, or X-RateLimit-Remaining 0 + with a reset time): wait as the server says, capped. A plain 403/404/422 is final (access, missing repo). + """ + backoff = min(2 ** (attempt + 1), 30) + if code in RETRY_STATUS: + return backoff + if code in (403, 429): + retry_after = headers.get("Retry-After") if headers else None + if retry_after and str(retry_after).isdigit(): + return min(int(retry_after) + 1, MAX_WAIT) + if headers and headers.get("X-RateLimit-Remaining") == "0" and str(headers.get("X-RateLimit-Reset", "")).isdigit(): + return min(max(int(headers["X-RateLimit-Reset"]) - time.time(), 0) + 1, MAX_WAIT) + if code == 429: + return backoff + return None + + +def api_get(path: str, token: str, retries: int = 5) -> Any: req = urllib.request.Request( API + path, headers={ @@ -57,15 +81,16 @@ def api_get(path: str, token: str, retries: int = 3) -> Any: return json.loads(resp.read().decode("utf-8")) except urllib.error.HTTPError as e: body = e.read().decode("utf-8", "replace")[:200] - if e.code in (500, 502, 503, 504) and attempt < retries - 1: - time.sleep(2 * (attempt + 1)) + wait = retry_wait(e.code, e.headers, attempt) + if wait is not None and attempt < retries - 1: + time.sleep(wait) continue raise GitHubError(e.code, body) from None - except urllib.error.URLError as e: + except (urllib.error.URLError, TimeoutError, ConnectionError) as e: if attempt < retries - 1: - time.sleep(2 * (attempt + 1)) + time.sleep(min(2 ** (attempt + 1), 30)) continue - raise GitHubError(0, str(e.reason)) from None + raise GitHubError(0, str(getattr(e, "reason", e))) from None raise GitHubError(0, "unreachable") diff --git a/tools/verify.py b/tools/verify.py index fb2af5f..391a276 100755 --- a/tools/verify.py +++ b/tools/verify.py @@ -3,6 +3,11 @@ no shell and no PATH lookups, so it also works under cron/launchd. Steps: compile all Python, run unit tests, check generated READMEs are current. + +``--daily`` is for the scheduled data run (Actions / tools/local_daily.sh): the unit tests are code tests and run in +CI on every pull request; the daily run only validates the data it just generated (tools/data_checks.py: hard +errors fail, small drift against the GitHub API only warns), so a value that moves with GitHub can never throw away a +day's update. See docs/DAILY_UPDATE.md. """ from __future__ import annotations @@ -11,19 +16,23 @@ from pathlib import Path -def commands(root: Path) -> list[list[str]]: +def commands(root: Path, daily: bool = False) -> list[list[str]]: py = sys.executable + tests = [] if daily else [[py, "-m", "unittest", "discover", "-s", "tests", "-v"]] + data = [[py, "tools/data_checks.py"]] if daily else [] return [ [py, "-m", "compileall", "-q", "tools", "tests"], - [py, "-m", "unittest", "discover", "-s", "tests", "-v"], + *tests, + *data, [py, "tools/render_readme.py", "--check"], [py, "tools/newcomer_i18n.py"], # lists untranslated new-repo descriptions; informational, never fails ] -def main() -> int: +def main(argv: list[str] | None = None) -> int: + daily = "--daily" in (sys.argv[1:] if argv is None else argv) root = Path(__file__).resolve().parents[1] - for cmd in commands(root): + for cmd in commands(root, daily=daily): print(">>>", " ".join(cmd), flush=True) rc = subprocess.run(cmd, cwd=root).returncode if rc != 0: