From 34a43a8dca41e44d31a92a34d3a2e4f89d033f79 Mon Sep 17 00:00:00 2001 From: Hugh Runyan Date: Mon, 26 Oct 2026 13:23:00 -0700 Subject: [PATCH 1/3] A Climate TRACE site can grow with the city near it A site row may carry city_population: the nearby city's UN World Urbanization Prospects 2025 population by year. SWEET grows the site's waste with that series instead of the country's. Before the city's first year and after its last, the city's nearest number moves with the country's population; between two reported years growth is constant. A row without the column grows with the country, as before. Co-Authored-By: Claude Opus 5.5 --- SWEET_python/MODEL_DECISIONS.md | 15 ++-- SWEET_python/city_params.py | 15 +++- SWEET_python/population.py | 73 +++++++++++++++++ changelog/2026-10.md | 10 +++ tests/test_site_city_population.py | 125 +++++++++++++++++++++++++++++ 5 files changed, 230 insertions(+), 8 deletions(-) create mode 100644 tests/test_site_city_population.py diff --git a/SWEET_python/MODEL_DECISIONS.md b/SWEET_python/MODEL_DECISIONS.md index bd01245..728b79d 100644 --- a/SWEET_python/MODEL_DECISIONS.md +++ b/SWEET_python/MODEL_DECISIONS.md @@ -216,25 +216,26 @@ Every tonne a city generates is diverted or disposed of. None is left uncollecte ### `population-growth` -Waste grows with the country's UN population each year. A big UN-listed city keeps its own two UN rates. With no usable series, fixed rates apply. +Waste grows with the country's UN population each year. A Climate TRACE site near a UN-listed city grows with that city. A big UN-listed city keeps its own two UN rates. With no usable series, fixed rates apply. - **Kind:** fallback - **Why:** Population is the main driver of how much waste a place generates over time, and a big city can outgrow its country. - **Options, best first:** - 1. `un-city-rates`: a UN-listed city of 300,000 or more, with plausible rates, compounds its own historic and future UN rates. - 2. `un-population`: the country's UN World Population Prospects 2024 population, by year, from SWEET's table. - 3. `constant-rate`: fixed historic and future rates, when there is no usable yearly series. + 1. `un-city-series`: the nearby city's UN population by year, which the caller supplies. Outside the city's years it moves with the country's population. + 2. `un-city-rates`: a UN-listed city of 300,000 or more, with plausible rates, compounds its own historic and future UN rates. + 3. `un-population`: the country's UN World Population Prospects 2024 population, by year, from SWEET's table. + 4. `constant-rate`: fixed historic and future rates, when there is no usable yearly series. - **Paths:** - `city-model`, `city-dst-scenario`: all three; `constant-rate` is the city's own rates. `city_population_trajectory` gives the same answer to a caller that stores it. - `custom-location`: `un-population`, else world urban-population rates. - `site-tool`: `un-population`, else an error asking for a rate. A rate the user sets compounds both ways; −100% is meant to hold waste flat. - - `ct-site`: the series Climate TRACE hands in, gaps held at the nearest year, else Climate TRACE's rates. An anchor year outside 1970–2050 takes the rates. + - `ct-site`: `un-city-series` when the row carries a city, else the country series Climate TRACE hands in, gaps held at the nearest year, else its rates. An anchor year outside 1970–2050 takes the rates. - **Confidence:** medium - **Better:** Use one fallback on every path, and UN city trajectories rather than two rates for listed cities. - **Known problem:** in `site-tool`, a −100% rate holds only the baseline flat; the scenario's waste goes to zero or infinity, because its growth factors skip the zero-rate guard. - **Value:** `SWEET_python/pops_yearly.csv`; `city_params`: `UN_LISTED_CITY_MIN_POPULATION`, `PLAUSIBLE_CITY_GROWTH`, `FALLBACK_GROWTH_RATE_HISTORIC`, `FALLBACK_GROWTH_RATE_FUTURE` -- **Code:** `SWEET_python/class_defs.py` -- **Reviewed:** 2026-10-22 +- **Code:** `SWEET_python/class_defs.py`, `SWEET_python/population.py` +- **Reviewed:** 2026-10-26 ### `intake-follows-population` diff --git a/SWEET_python/city_params.py b/SWEET_python/city_params.py index 1cb1e2c..a9dd2e1 100644 --- a/SWEET_python/city_params.py +++ b/SWEET_python/city_params.py @@ -26,7 +26,12 @@ import SWEET_python.defaults_2019 as defaults_2019 import SWEET_python.mcf as mcf_defaults import SWEET_python.site_type_defaults as site_type_defaults -from SWEET_python.population import average_growth_rates, country_population_series +from SWEET_python.population import ( + average_growth_rates, + city_population_series, + country_population_series, + parse_city_population, +) import psycopg2 from psycopg2.extras import RealDictCursor from sqlalchemy import create_engine, text @@ -3530,6 +3535,14 @@ def import_basics_site(self, canonical_row, pop_data, usecase='wastemap', time_s # WPP2024 per-year population series for this country (None -> the # frozen-CAGR growth_rate_* scalars are used instead). population_series = _population_series_from_pop_data(pop_data, self.iso3) + # Climate TRACE may hand in the site's nearby city's population. Where it does, + # the site grows with that city, and with the country's shape outside the + # city's years (population.city_population_series). + city_series = city_population_series( + parse_city_population(canonical_row.get("city_population")), population_series + ) + if city_series is not None: + population_series = city_series # True once waste_generated already carries population growth (the # multi-row path bakes it into the projected series below), so the # later WasteGeneratedDF.create must not apply growth a second time. diff --git a/SWEET_python/population.py b/SWEET_python/population.py index 24125e4..90b1beb 100644 --- a/SWEET_python/population.py +++ b/SWEET_python/population.py @@ -16,10 +16,13 @@ from __future__ import annotations +import json +from collections.abc import Mapping from functools import lru_cache from pathlib import Path from typing import Optional +import numpy as np import pandas as pd from SWEET_python.constants import MODEL_END_YEAR, MODEL_START_YEAR @@ -66,3 +69,73 @@ def average_growth_rates( historic = future if historic is None else historic future = historic if future is None else future return float(historic), float(future) + + +def parse_city_population(value) -> Optional[dict[int, float]]: + """A city's population by year, from a mapping or its JSON text, or ``None``. + + Climate TRACE hands a site's nearby city in as JSON text on the site row, because a + row is saved to parquet between stages and text survives that. Keys may be ints or + the strings JSON makes of them. Years with no positive number are dropped: the UN + leaves a city blank in the years it was under 50,000 people, and a blank means "not + a city yet", not zero people. + """ + if value is None or (isinstance(value, float) and pd.isna(value)): + return None + if isinstance(value, str): + if not value.strip(): + return None + value = json.loads(value) + if not isinstance(value, Mapping): + raise TypeError(f"city population must be a mapping of year to people, not {type(value).__name__}") + years = {} + for year, people in value.items(): + people = pd.to_numeric(people, errors="coerce") + if pd.notna(people) and people > 0: + years[int(year)] = float(people) + return years or None + + +def city_population_series( + city: Optional[Mapping[int, float]], country: Optional[pd.Series] +) -> Optional[pd.Series]: + """A city's population in every year of ``country``'s index, its gaps filled. + + - **In a year the city has a number**, that number. + - **Between two of them**, constant growth from one to the next (log-linear), for a + source that reports every five years. + - **Before its first year or after its last**, the city's nearest number moved by + the country's population: ``city[first] * country[year] / country[first]``. A city + appears in the UN's list only once it passes 50,000 people, so half the cities a + site is near start after 1975 and a quarter after 2000. + + The country's shape fills the ends because the other ways bend: a straight line back + from a fast-growing city reaches zero within a few decades, and its growth rate + compounded back shrinks it to nearly nothing. The country's shape joins smoothly, + never goes below zero, and is what a site with no city gets anyway. A city grows + faster than its country, so it slightly overstates a city's population before its + first year. + + ``None`` when there is no city, no country series, or no city year inside the + country's years; the caller then grows by the country alone. + """ + # decision: population-growth + if not city or country is None: + return None + observed = pd.Series(city, dtype=float).sort_index() + observed = observed[observed.index.isin(country.index)] + if observed.empty: + return None + + series = pd.Series(np.nan, index=country.index, dtype=float) + series[observed.index] = observed.values + first, last = observed.index[0], observed.index[-1] + inside = (series.index >= first) & (series.index <= last) + # Only the missing years: a reported number survives exactly, not via exp(log(x)). + gaps = inside & series.isna().to_numpy() + series[gaps] = np.exp(np.log(series[inside]).interpolate(method="index"))[gaps[inside]] + before = series.index < first + series[before] = observed[first] * country[before] / country[first] + after = series.index > last + series[after] = observed[last] * country[after] / country[last] + return series diff --git a/changelog/2026-10.md b/changelog/2026-10.md index 4a7bab6..da1e24c 100644 --- a/changelog/2026-10.md +++ b/changelog/2026-10.md @@ -118,6 +118,16 @@ heavily: Yokohama's city DST methane falls 74% without it (model-output change). ## Changed +- **A Climate TRACE site can grow with the city near it, not its country** + (model-output change for Climate TRACE only). A site row may carry + `city_population`, the nearby city's UN World Urbanization Prospects 2025 + population by year, as a mapping or its JSON text. SWEET grows the site's waste + with that series instead of the country's. A city is listed only once it passes + 50,000 people, so before its first year (and after its last) the city's nearest + number moves with the country's population; between two reported years growth is + constant. A row without the column grows with the country, as before, and no + WasteMAP path passes one. `population.parse_city_population`, + `population.city_population_series`. Tests: `tests/test_site_city_population.py`. - **The city paths flare captured gas at 0.98, as every other path does** (model-output change, [#105](https://github.com/RMI/SWEET_python/pull/105)). A landfill with no flaring series used to have all of its captured gas destroyed by the annual engine; it now takes `DEFAULT_FLARE_EFFICIENCY`, diff --git a/tests/test_site_city_population.py b/tests/test_site_city_population.py new file mode 100644 index 0000000..1766b28 --- /dev/null +++ b/tests/test_site_city_population.py @@ -0,0 +1,125 @@ +"""A Climate TRACE site can grow with its nearby city instead of its country. + +Climate TRACE hands in the population of the UN-listed city near a site, as JSON text on +the site row. SWEET uses the city's own numbers where it has them and the country's shape +outside them (`population.city_population_series`). A row without a city grows with the +country, as before. +""" + +import json + +import numpy as np +import pandas as pd +import pytest + +from SWEET_python.city_params import City +from SWEET_python.population import ( + city_population_series, + country_population_series, + parse_city_population, +) + +WINDOW = list(range(1970, 2051)) +USA = country_population_series("USA").loc[WINDOW] + + +# --------------------------------------------------------------------------- # +# Reading the city +# --------------------------------------------------------------------------- # + +def test_json_text_with_string_years_reads_as_int_years(): + assert parse_city_population('{"1975": 50000, "2025": 80000}') == {1975: 50000.0, 2025: 80000.0} + + +@pytest.mark.parametrize("blank", [None, np.nan, "", "{}"]) +def test_no_city_reads_as_none(blank): + assert parse_city_population(blank) is None + + +def test_years_under_the_threshold_are_dropped_not_read_as_zero(): + """The UN leaves a city blank in the years it was under 50,000 people.""" + assert parse_city_population({1975: None, 1980: 0, 1985: 51000}) == {1985: 51000.0} + + +# --------------------------------------------------------------------------- # +# Filling the city's gaps +# --------------------------------------------------------------------------- # + +def test_the_citys_own_years_are_kept_exactly(): + city = {year: 100_000.0 + 1_000 * (year - 1975) for year in range(1975, 2051)} + series = city_population_series(city, USA) + assert series.loc[1975:2050].tolist() == [city[y] for y in range(1975, 2051)] + + +def test_years_before_the_city_follow_the_countrys_shape_and_join_smoothly(): + """A city first listed in 2005 at 50,000: earlier years move with the country.""" + series = city_population_series({2005: 50_000.0, 2025: 200_000.0}, USA) + assert series[2005] == 50_000.0 + for year in (1970, 1990, 2004): + assert series[year] == pytest.approx(50_000.0 * USA[year] / USA[2005]) + + +def test_years_after_the_city_follow_the_countrys_shape(): + series = city_population_series({1975: 60_000.0, 2025: 90_000.0}, USA) + assert series[2050] == pytest.approx(90_000.0 * USA[2050] / USA[2025]) + + +def test_between_two_reported_years_growth_is_constant(): + """For a source that reports every five years.""" + series = city_population_series({2000: 100_000.0, 2005: 200_000.0}, USA) + rate = 2 ** (1 / 5) + assert series[2001] == pytest.approx(100_000.0 * rate) + assert series[2004] == pytest.approx(100_000.0 * rate ** 4) + + +def test_a_fast_city_never_reaches_zero_going_back(): + """A straight line back from 50,000 in 2005 to 200,000 in 2025 hits zero by 1998.""" + series = city_population_series({2005: 50_000.0, 2025: 200_000.0}, USA) + assert (series > 0).all() + assert series.index.tolist() == WINDOW + + +@pytest.mark.parametrize("city, country", [(None, USA), ({2005: 1.0}, None), ({1960: 1.0}, USA)]) +def test_without_a_city_or_country_or_overlap_there_is_no_series(city, country): + assert city_population_series(city, country) is None + + +# --------------------------------------------------------------------------- # +# Through a site +# --------------------------------------------------------------------------- # + +def _site(city_population): + row = pd.Series({ + "asset_identifier": 999802, "iso3_country": "USA", "type": "Sanitary Landfill", + "income_id": "HIC", "annual_incoming_waste": 100_000.0, "incoming_waste_year": 2023.0, + "site_open_year": 1980.0, "site_close_year": np.nan, "waste_depth": 10.0, + "landfill_gas_collection": False, "gas_collection_efficiency": np.nan, + "latitude": 39.0, "longitude": -95.0, "precipitation": 800.0, "temperature": 15.0, + "city_population": city_population, + }) + pop_data = pd.DataFrame( + {"growth_rate_historic": [1.02], "growth_rate_future": [1.01], + **{f"pop_{y}": [USA[y]] for y in WINDOW}}, + index=["USA"], + ) + city = City("999802") + city.site_only_estimate_trace(row, pop_data=pop_data) + return city.baseline_parameters + + +def _shape(series, anchor=2023): + total = series.sum(axis=1) if isinstance(series, pd.DataFrame) else series + return (total / total.loc[anchor]).loc[WINDOW].to_numpy() + + +def test_a_site_with_a_city_grows_with_the_city(): + city = {2005: 50_000.0, 2025: 200_000.0} + baseline = _site(json.dumps(city)) + expected = city_population_series(city, USA) + assert np.allclose(_shape(baseline.waste_generated_df.df), _shape(expected)) + assert np.allclose(baseline.population_series.loc[WINDOW], expected) + + +def test_a_site_with_no_city_still_grows_with_its_country(): + baseline = _site(None) + assert np.allclose(_shape(baseline.waste_generated_df.df), _shape(USA)) From 4c6f05a87cddba1918b9a4c9ceb8539aeb1f1535 Mon Sep 17 00:00:00 2001 From: Hugh Runyan Date: Mon, 26 Oct 2026 16:57:46 -0700 Subject: [PATCH 2/3] Read the city series from city_population_by_year; accept pd.NA; refuse a gappy country - The site-row key is now city_population_by_year. city_population already names the Brazil city-site path's one number per city, on a different frame. - parse_city_population treats any scalar missing value (pd.NA, NaT) as no city. - city_population_series returns None when the country series has a gap or a non-positive year, so the site falls back to the country instead of a NaN end. Co-Authored-By: Claude Opus 5.5 --- SWEET_python/city_params.py | 2 +- SWEET_python/population.py | 11 ++++++++--- changelog/2026-10.md | 2 +- tests/test_site_city_population.py | 22 +++++++++++++++++++--- 4 files changed, 29 insertions(+), 8 deletions(-) diff --git a/SWEET_python/city_params.py b/SWEET_python/city_params.py index a9dd2e1..6ce1655 100644 --- a/SWEET_python/city_params.py +++ b/SWEET_python/city_params.py @@ -3539,7 +3539,7 @@ def import_basics_site(self, canonical_row, pop_data, usecase='wastemap', time_s # the site grows with that city, and with the country's shape outside the # city's years (population.city_population_series). city_series = city_population_series( - parse_city_population(canonical_row.get("city_population")), population_series + parse_city_population(canonical_row.get("city_population_by_year")), population_series ) if city_series is not None: population_series = city_series diff --git a/SWEET_python/population.py b/SWEET_python/population.py index 90b1beb..f6aa6b4 100644 --- a/SWEET_python/population.py +++ b/SWEET_python/population.py @@ -80,7 +80,9 @@ def parse_city_population(value) -> Optional[dict[int, float]]: leaves a city blank in the years it was under 50,000 people, and a blank means "not a city yet", not zero people. """ - if value is None or (isinstance(value, float) and pd.isna(value)): + # Any scalar missing value (None, NaN, pd.NA, NaT); is_scalar keeps a mapping or + # text away from pd.isna. + if value is None or (pd.api.types.is_scalar(value) and pd.isna(value)): return None if isinstance(value, str): if not value.strip(): @@ -116,12 +118,15 @@ def city_population_series( faster than its country, so it slightly overstates a city's population before its first year. - ``None`` when there is no city, no country series, or no city year inside the - country's years; the caller then grows by the country alone. + ``None`` when there is no city, no city year inside the country's years, or no + country series with a positive number in every year (the city's ends are scaled by + it, so a gap would leave a gap); the caller then grows by the country alone. """ # decision: population-growth if not city or country is None: return None + if country.isna().any() or not (country > 0).all(): + return None observed = pd.Series(city, dtype=float).sort_index() observed = observed[observed.index.isin(country.index)] if observed.empty: diff --git a/changelog/2026-10.md b/changelog/2026-10.md index da1e24c..b063796 100644 --- a/changelog/2026-10.md +++ b/changelog/2026-10.md @@ -120,7 +120,7 @@ heavily: Yokohama's city DST methane falls 74% without it (model-output change). - **A Climate TRACE site can grow with the city near it, not its country** (model-output change for Climate TRACE only). A site row may carry - `city_population`, the nearby city's UN World Urbanization Prospects 2025 + `city_population_by_year`, the nearby city's UN World Urbanization Prospects 2025 population by year, as a mapping or its JSON text. SWEET grows the site's waste with that series instead of the country's. A city is listed only once it passes 50,000 people, so before its first year (and after its last) the city's nearest diff --git a/tests/test_site_city_population.py b/tests/test_site_city_population.py index 1766b28..dd9212e 100644 --- a/tests/test_site_city_population.py +++ b/tests/test_site_city_population.py @@ -31,7 +31,7 @@ def test_json_text_with_string_years_reads_as_int_years(): assert parse_city_population('{"1975": 50000, "2025": 80000}') == {1975: 50000.0, 2025: 80000.0} -@pytest.mark.parametrize("blank", [None, np.nan, "", "{}"]) +@pytest.mark.parametrize("blank", [None, np.nan, pd.NA, pd.NaT, "", "{}"]) def test_no_city_reads_as_none(blank): assert parse_city_population(blank) is None @@ -84,18 +84,27 @@ def test_without_a_city_or_country_or_overlap_there_is_no_series(city, country): assert city_population_series(city, country) is None +@pytest.mark.parametrize("year, value", [(1970, np.nan), (2050, np.nan), (2005, 0.0)]) +def test_a_country_with_a_gap_or_no_people_gives_no_series(year, value): + """The city's ends are scaled by the country, so a gap there would leave a gap.""" + country = USA.copy() + country[year] = value + assert city_population_series({2005: 50_000.0, 2025: 200_000.0}, country) is None + + # --------------------------------------------------------------------------- # # Through a site # --------------------------------------------------------------------------- # -def _site(city_population): +def _site(city_population_by_year, **extra): row = pd.Series({ "asset_identifier": 999802, "iso3_country": "USA", "type": "Sanitary Landfill", "income_id": "HIC", "annual_incoming_waste": 100_000.0, "incoming_waste_year": 2023.0, "site_open_year": 1980.0, "site_close_year": np.nan, "waste_depth": 10.0, "landfill_gas_collection": False, "gas_collection_efficiency": np.nan, "latitude": 39.0, "longitude": -95.0, "precipitation": 800.0, "temperature": 15.0, - "city_population": city_population, + "city_population_by_year": city_population_by_year, + **extra, }) pop_data = pd.DataFrame( {"growth_rate_historic": [1.02], "growth_rate_future": [1.01], @@ -123,3 +132,10 @@ def test_a_site_with_a_city_grows_with_the_city(): def test_a_site_with_no_city_still_grows_with_its_country(): baseline = _site(None) assert np.allclose(_shape(baseline.waste_generated_df.df), _shape(USA)) + + +def test_a_single_city_population_number_on_a_site_row_is_not_read(): + """``city_population`` is the Brazil city-site path's one number per city; a site + row grows only with ``city_population_by_year``.""" + baseline = _site(None, city_population=1_500_000.0) + assert np.allclose(_shape(baseline.waste_generated_df.df), _shape(USA)) From e3b8d845a6dce6a4646a55c3ca75f60837854885 Mon Sep 17 00:00:00 2001 From: Hugh Runyan Date: Thu, 29 Oct 2026 15:41:40 -0700 Subject: [PATCH 3/3] The city series' gap-filling wording matches the supplement's every-year data Ingestion #299 lists every year from a city's first to 2050, not every fifth year, so log-linear growth now only bridges a gap. Co-Authored-By: Claude Opus 5.5 --- SWEET_python/population.py | 5 +++-- tests/test_site_city_population.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/SWEET_python/population.py b/SWEET_python/population.py index f6aa6b4..4240d43 100644 --- a/SWEET_python/population.py +++ b/SWEET_python/population.py @@ -104,8 +104,9 @@ def city_population_series( """A city's population in every year of ``country``'s index, its gaps filled. - **In a year the city has a number**, that number. - - **Between two of them**, constant growth from one to the next (log-linear), for a - source that reports every five years. + - **Between two of them**, constant growth from one to the next (log-linear), where a + year is missing. Climate TRACE's supplement lists every year from a city's first to + 2050 (waste_data_ingestion_pipeline #299), so this only bridges a gap. - **Before its first year or after its last**, the city's nearest number moved by the country's population: ``city[first] * country[year] / country[first]``. A city appears in the UN's list only once it passes 50,000 people, so half the cities a diff --git a/tests/test_site_city_population.py b/tests/test_site_city_population.py index dd9212e..625899f 100644 --- a/tests/test_site_city_population.py +++ b/tests/test_site_city_population.py @@ -65,7 +65,7 @@ def test_years_after_the_city_follow_the_countrys_shape(): def test_between_two_reported_years_growth_is_constant(): - """For a source that reports every five years.""" + """For a year missing between two listed ones.""" series = city_population_series({2000: 100_000.0, 2005: 200_000.0}, USA) rate = 2 ** (1 / 5) assert series[2001] == pytest.approx(100_000.0 * rate)