A Climate TRACE site can grow its waste with the city near it

SWEET_python/MODEL_DECISIONS.md

@@ -218,25 +218,26 @@ Every tonne a city generates is diverted or disposed of. None is left uncollecte
 
 ### `population-growth`
 
-Waste grows with the country's UN population each year. A big UN-listed city keeps its own two UN rates. With no usable series, fixed rates apply.
+Waste grows with the country's UN population each year. A Climate TRACE site near a UN-listed city grows with that city. A big UN-listed city keeps its own two UN rates. With no usable series, fixed rates apply.
 
 - **Kind:** fallback
 - **Why:** Population is the main driver of how much waste a place generates over time, and a big city can outgrow its country.
 - **Options, best first:**
-  1. `un-city-rates`: a UN-listed city of 300,000 or more, with plausible rates, compounds its own historic and future UN rates.
-  2. `un-population`: the country's UN World Population Prospects 2024 population, by year, from SWEET's table.
-  3. `constant-rate`: fixed historic and future rates, when there is no usable yearly series.
+  1. `un-city-series`: the nearby city's UN population by year, which the caller supplies. Outside the city's years it moves with the country's population.
+  2. `un-city-rates`: a UN-listed city of 300,000 or more, with plausible rates, compounds its own historic and future UN rates.
+  3. `un-population`: the country's UN World Population Prospects 2024 population, by year, from SWEET's table.
+  4. `constant-rate`: fixed historic and future rates, when there is no usable yearly series.
 - **Paths:**
   - `city-model`, `city-dst-scenario`: all three; `constant-rate` is the city's own rates. `city_population_trajectory` gives the same answer to a caller that stores it.
   - `custom-location`: `un-population`, else world urban-population rates.
   - `site-tool`: `un-population`, else an error asking for a rate. A rate the user sets compounds both ways; −100% is meant to hold waste flat.
-  - `ct-site`: the series Climate TRACE hands in, gaps held at the nearest year, else Climate TRACE's rates. An anchor year outside 1970–2050 takes the rates.
+  - `ct-site`: `un-city-series` when the row carries a city, else the country series Climate TRACE hands in, gaps held at the nearest year, else its rates. An anchor year outside 1970–2050 takes the rates.
 - **Confidence:** medium
 - **Better:** Use one fallback on every path, and UN city trajectories rather than two rates for listed cities.
 - **Known problem:** in `site-tool`, a −100% rate holds only the baseline flat; the scenario's waste goes to zero or infinity, because its growth factors skip the zero-rate guard.
 - **Value:** `SWEET_python/pops_yearly.csv`; `city_params`: `UN_LISTED_CITY_MIN_POPULATION`, `PLAUSIBLE_CITY_GROWTH`, `FALLBACK_GROWTH_RATE_HISTORIC`, `FALLBACK_GROWTH_RATE_FUTURE`
-- **Code:** `SWEET_python/class_defs.py`
-- **Reviewed:** 2026-10-22
+- **Code:** `SWEET_python/class_defs.py`, `SWEET_python/population.py`
+- **Reviewed:** 2026-10-26
 
 ### `intake-follows-population`
 

SWEET_python/city_params.py

@@ -26,7 +26,12 @@
 import SWEET_python.defaults_2019 as defaults_2019
 import SWEET_python.mcf as mcf_defaults
 import SWEET_python.site_type_defaults as site_type_defaults
-from SWEET_python.population import average_growth_rates, country_population_series
+from SWEET_python.population import (
+    average_growth_rates,
+    city_population_series,
+    country_population_series,
+    parse_city_population,
+)
 import psycopg2
 from psycopg2.extras import RealDictCursor
 from sqlalchemy import create_engine, text
@@ -3666,6 +3671,14 @@ def import_basics_site(self, canonical_row, pop_data, usecase='wastemap', time_s
             # WPP2024 per-year population series for this country (None -> the
             # frozen-CAGR growth_rate_* scalars are used instead).
             population_series = _population_series_from_pop_data(pop_data, self.iso3)
+            # Climate TRACE may hand in the site's nearby city's population. Where it does,
+            # the site grows with that city, and with the country's shape outside the
+            # city's years (population.city_population_series).
+            city_series = city_population_series(
+                parse_city_population(canonical_row.get("city_population_by_year")), population_series
+            )
+            if city_series is not None:
+                population_series = city_series
             # True once waste_generated already carries population growth (the
             # multi-row path bakes it into the projected series below), so the
             # later WasteGeneratedDF.create must not apply growth a second time.

SWEET_python/population.py

@@ -16,10 +16,13 @@
 
 from __future__ import annotations
 
+import json
+from collections.abc import Mapping
 from functools import lru_cache
 from pathlib import Path
 from typing import Optional
 
+import numpy as np
 import pandas as pd
 
 from SWEET_python.constants import MODEL_END_YEAR, MODEL_START_YEAR
@@ -66,3 +69,79 @@ def average_growth_rates(
     historic = future if historic is None else historic
     future = historic if future is None else future
     return float(historic), float(future)
+
+
+def parse_city_population(value) -> Optional[dict[int, float]]:
+    """A city's population by year, from a mapping or its JSON text, or ``None``.
+
+    Climate TRACE hands a site's nearby city in as JSON text on the site row, because a
+    row is saved to parquet between stages and text survives that. Keys may be ints or
+    the strings JSON makes of them. Years with no positive number are dropped: the UN
+    leaves a city blank in the years it was under 50,000 people, and a blank means "not
+    a city yet", not zero people.
+    """
+    # Any scalar missing value (None, NaN, pd.NA, NaT); is_scalar keeps a mapping or
+    # text away from pd.isna.
+    if value is None or (pd.api.types.is_scalar(value) and pd.isna(value)):
+        return None
+    if isinstance(value, str):
+        if not value.strip():
+            return None
+        value = json.loads(value)
+    if not isinstance(value, Mapping):
+        raise TypeError(f"city population must be a mapping of year to people, not {type(value).__name__}")
+    years = {}
+    for year, people in value.items():
+        people = pd.to_numeric(people, errors="coerce")
+        if pd.notna(people) and people > 0:
+            years[int(year)] = float(people)
+    return years or None
+
+
+def city_population_series(
+    city: Optional[Mapping[int, float]], country: Optional[pd.Series]
+) -> Optional[pd.Series]:
+    """A city's population in every year of ``country``'s index, its gaps filled.
+
+    - **In a year the city has a number**, that number.
+    - **Between two of them**, constant growth from one to the next (log-linear), where a
+      year is missing. Climate TRACE's supplement lists every year from a city's first to
+      2050 (waste_data_ingestion_pipeline #299), so this only bridges a gap.
+    - **Before its first year or after its last**, the city's nearest number moved by
+      the country's population: ``city[first] * country[year] / country[first]``. A city
+      appears in the UN's list only once it passes 50,000 people, so half the cities a
+      site is near start after 1975 and a quarter after 2000.
+
+    The country's shape fills the ends because the other ways bend: a straight line back
+    from a fast-growing city reaches zero within a few decades, and its growth rate
+    compounded back shrinks it to nearly nothing. The country's shape joins smoothly,
+    never goes below zero, and is what a site with no city gets anyway. A city grows
+    faster than its country, so it slightly overstates a city's population before its
+    first year.
+
+    ``None`` when there is no city, no city year inside the country's years, or no
+    country series with a positive number in every year (the city's ends are scaled by
+    it, so a gap would leave a gap); the caller then grows by the country alone.
+    """
+    # decision: population-growth
+    if not city or country is None:
+        return None
+    if country.isna().any() or not (country > 0).all():
+        return None
+    observed = pd.Series(city, dtype=float).sort_index()
+    observed = observed[observed.index.isin(country.index)]
+    if observed.empty:
+        return None
+
+    series = pd.Series(np.nan, index=country.index, dtype=float)
+    series[observed.index] = observed.values
+    first, last = observed.index[0], observed.index[-1]
+    inside = (series.index >= first) & (series.index <= last)
+    # Only the missing years: a reported number survives exactly, not via exp(log(x)).
+    gaps = inside & series.isna().to_numpy()
+    series[gaps] = np.exp(np.log(series[inside]).interpolate(method="index"))[gaps[inside]]
+    before = series.index < first
+    series[before] = observed[first] * country[before] / country[first]
+    after = series.index > last
+    series[after] = observed[last] * country[after] / country[last]
+    return series

changelog/2026-10.md

@@ -206,6 +206,16 @@ heavily: Yokohama's city DST methane falls 74% without it (model-output change).
   - SWEET now depends on scipy (`requirements.in`; the lockfile adds `scipy==1.18.1` and moves no other pin). It is imported only where the line is fitted, so WasteMAP, which installs SWEET with `--no-deps` and has no scipy, still imports `city_params`.
   - Tests: `tests/test_site_intake_series.py`.
     ([#113](https://github.com/RMI/SWEET_python/pull/113))
+- **A Climate TRACE site can grow with the city near it, not its country**
+  (model-output change for Climate TRACE only). A site row may carry
+  `city_population_by_year`, the nearby city's UN World Urbanization Prospects 2025
+  population by year, as a mapping or its JSON text. SWEET grows the site's waste
+  with that series instead of the country's. A city is listed only once it passes
+  50,000 people, so before its first year (and after its last) the city's nearest
+  number moves with the country's population; between two reported years growth is
+  constant. A row without the column grows with the country, as before, and no
+  WasteMAP path passes one. `population.parse_city_population`,
+  `population.city_population_series`. Tests: `tests/test_site_city_population.py`.
 
 - **The city paths flare captured gas at 0.98, as every other path does**
   (model-output change, [#105](https://github.com/RMI/SWEET_python/pull/105)). A landfill with no flaring series used to have all of its

tests/test_site_city_population.py

@@ -0,0 +1,141 @@
+"""A Climate TRACE site can grow with its nearby city instead of its country.
+
+Climate TRACE hands in the population of the UN-listed city near a site, as JSON text on
+the site row. SWEET uses the city's own numbers where it has them and the country's shape
+outside them (`population.city_population_series`). A row without a city grows with the
+country, as before.
+"""
+
+import json
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from SWEET_python.city_params import City
+from SWEET_python.population import (
+    city_population_series,
+    country_population_series,
+    parse_city_population,
+)
+
+WINDOW = list(range(1970, 2051))
+USA = country_population_series("USA").loc[WINDOW]
+
+
+# --------------------------------------------------------------------------- #
+# Reading the city
+# --------------------------------------------------------------------------- #
+
+def test_json_text_with_string_years_reads_as_int_years():
+    assert parse_city_population('{"1975": 50000, "2025": 80000}') == {1975: 50000.0, 2025: 80000.0}
+
+
+@pytest.mark.parametrize("blank", [None, np.nan, pd.NA, pd.NaT, "", "{}"])
+def test_no_city_reads_as_none(blank):
+    assert parse_city_population(blank) is None
+
+
+def test_years_under_the_threshold_are_dropped_not_read_as_zero():
+    """The UN leaves a city blank in the years it was under 50,000 people."""
+    assert parse_city_population({1975: None, 1980: 0, 1985: 51000}) == {1985: 51000.0}
+
+
+# --------------------------------------------------------------------------- #
+# Filling the city's gaps
+# --------------------------------------------------------------------------- #
+
+def test_the_citys_own_years_are_kept_exactly():
+    city = {year: 100_000.0 + 1_000 * (year - 1975) for year in range(1975, 2051)}
+    series = city_population_series(city, USA)
+    assert series.loc[1975:2050].tolist() == [city[y] for y in range(1975, 2051)]
+
+
+def test_years_before_the_city_follow_the_countrys_shape_and_join_smoothly():
+    """A city first listed in 2005 at 50,000: earlier years move with the country."""
+    series = city_population_series({2005: 50_000.0, 2025: 200_000.0}, USA)
+    assert series[2005] == 50_000.0
+    for year in (1970, 1990, 2004):
+        assert series[year] == pytest.approx(50_000.0 * USA[year] / USA[2005])
+
+
+def test_years_after_the_city_follow_the_countrys_shape():
+    series = city_population_series({1975: 60_000.0, 2025: 90_000.0}, USA)
+    assert series[2050] == pytest.approx(90_000.0 * USA[2050] / USA[2025])
+
+
+def test_between_two_reported_years_growth_is_constant():
+    """For a year missing between two listed ones."""
+    series = city_population_series({2000: 100_000.0, 2005: 200_000.0}, USA)
+    rate = 2 ** (1 / 5)
+    assert series[2001] == pytest.approx(100_000.0 * rate)
+    assert series[2004] == pytest.approx(100_000.0 * rate ** 4)
+
+
+def test_a_fast_city_never_reaches_zero_going_back():
+    """A straight line back from 50,000 in 2005 to 200,000 in 2025 hits zero by 1998."""
+    series = city_population_series({2005: 50_000.0, 2025: 200_000.0}, USA)
+    assert (series > 0).all()
+    assert series.index.tolist() == WINDOW
+
+
+@pytest.mark.parametrize("city, country", [(None, USA), ({2005: 1.0}, None), ({1960: 1.0}, USA)])
+def test_without_a_city_or_country_or_overlap_there_is_no_series(city, country):
+    assert city_population_series(city, country) is None
+
+
+@pytest.mark.parametrize("year, value", [(1970, np.nan), (2050, np.nan), (2005, 0.0)])
+def test_a_country_with_a_gap_or_no_people_gives_no_series(year, value):
+    """The city's ends are scaled by the country, so a gap there would leave a gap."""
+    country = USA.copy()
+    country[year] = value
+    assert city_population_series({2005: 50_000.0, 2025: 200_000.0}, country) is None
+
+
+# --------------------------------------------------------------------------- #
+# Through a site
+# --------------------------------------------------------------------------- #
+
+def _site(city_population_by_year, **extra):
+    row = pd.Series({
+        "asset_identifier": 999802, "iso3_country": "USA", "type": "Sanitary Landfill",
+        "income_id": "HIC", "annual_incoming_waste": 100_000.0, "incoming_waste_year": 2023.0,
+        "site_open_year": 1980.0, "site_close_year": np.nan, "waste_depth": 10.0,
+        "landfill_gas_collection": False, "gas_collection_efficiency": np.nan,
+        "latitude": 39.0, "longitude": -95.0, "precipitation": 800.0, "temperature": 15.0,
+        "city_population_by_year": city_population_by_year,
+        **extra,
+    })
+    pop_data = pd.DataFrame(
+        {"growth_rate_historic": [1.02], "growth_rate_future": [1.01],
+         **{f"pop_{y}": [USA[y]] for y in WINDOW}},
+        index=["USA"],
+    )
+    city = City("999802")
+    city.site_only_estimate_trace(row, pop_data=pop_data)
+    return city.baseline_parameters
+
+
+def _shape(series, anchor=2023):
+    total = series.sum(axis=1) if isinstance(series, pd.DataFrame) else series
+    return (total / total.loc[anchor]).loc[WINDOW].to_numpy()
+
+
+def test_a_site_with_a_city_grows_with_the_city():
+    city = {2005: 50_000.0, 2025: 200_000.0}
+    baseline = _site(json.dumps(city))
+    expected = city_population_series(city, USA)
+    assert np.allclose(_shape(baseline.waste_generated_df.df), _shape(expected))
+    assert np.allclose(baseline.population_series.loc[WINDOW], expected)
+
+
+def test_a_site_with_no_city_still_grows_with_its_country():
+    baseline = _site(None)
+    assert np.allclose(_shape(baseline.waste_generated_df.df), _shape(USA))
+
+
+def test_a_single_city_population_number_on_a_site_row_is_not_read():
+    """``city_population`` is the Brazil city-site path's one number per city; a site
+    row grows only with ``city_population_by_year``."""
+    baseline = _site(None, city_population=1_500_000.0)
+    assert np.allclose(_shape(baseline.waste_generated_df.df), _shape(USA))