diff --git a/SWEET_python/MODEL_DECISIONS.md b/SWEET_python/MODEL_DECISIONS.md index 6de28fd..70ffaa4 100644 --- a/SWEET_python/MODEL_DECISIONS.md +++ b/SWEET_python/MODEL_DECISIONS.md @@ -218,25 +218,26 @@ Every tonne a city generates is diverted or disposed of. None is left uncollecte ### `population-growth` -Waste grows with the country's UN population each year. A big UN-listed city keeps its own two UN rates. With no usable series, fixed rates apply. +Waste grows with the country's UN population each year. A Climate TRACE site near a UN-listed city grows with that city. A big UN-listed city keeps its own two UN rates. With no usable series, fixed rates apply. - **Kind:** fallback - **Why:** Population is the main driver of how much waste a place generates over time, and a big city can outgrow its country. - **Options, best first:** - 1. `un-city-rates`: a UN-listed city of 300,000 or more, with plausible rates, compounds its own historic and future UN rates. - 2. `un-population`: the country's UN World Population Prospects 2024 population, by year, from SWEET's table. - 3. `constant-rate`: fixed historic and future rates, when there is no usable yearly series. + 1. `un-city-series`: the nearby city's UN population by year, which the caller supplies. Outside the city's years it moves with the country's population. + 2. `un-city-rates`: a UN-listed city of 300,000 or more, with plausible rates, compounds its own historic and future UN rates. + 3. `un-population`: the country's UN World Population Prospects 2024 population, by year, from SWEET's table. + 4. `constant-rate`: fixed historic and future rates, when there is no usable yearly series. - **Paths:** - `city-model`, `city-dst-scenario`: all three; `constant-rate` is the city's own rates. `city_population_trajectory` gives the same answer to a caller that stores it. - `custom-location`: `un-population`, else world urban-population rates. - `site-tool`: `un-population`, else an error asking for a rate. A rate the user sets compounds both ways; −100% is meant to hold waste flat. - - `ct-site`: the series Climate TRACE hands in, gaps held at the nearest year, else Climate TRACE's rates. An anchor year outside 1970–2050 takes the rates. + - `ct-site`: `un-city-series` when the row carries a city, else the country series Climate TRACE hands in, gaps held at the nearest year, else its rates. An anchor year outside 1970–2050 takes the rates. - **Confidence:** medium - **Better:** Use one fallback on every path, and UN city trajectories rather than two rates for listed cities. - **Known problem:** in `site-tool`, a −100% rate holds only the baseline flat; the scenario's waste goes to zero or infinity, because its growth factors skip the zero-rate guard. - **Value:** `SWEET_python/pops_yearly.csv`; `city_params`: `UN_LISTED_CITY_MIN_POPULATION`, `PLAUSIBLE_CITY_GROWTH`, `FALLBACK_GROWTH_RATE_HISTORIC`, `FALLBACK_GROWTH_RATE_FUTURE` -- **Code:** `SWEET_python/class_defs.py` -- **Reviewed:** 2026-10-22 +- **Code:** `SWEET_python/class_defs.py`, `SWEET_python/population.py` +- **Reviewed:** 2026-10-26 ### `intake-follows-population` diff --git a/SWEET_python/city_params.py b/SWEET_python/city_params.py index 97fb7c8..9affe1c 100644 --- a/SWEET_python/city_params.py +++ b/SWEET_python/city_params.py @@ -26,7 +26,12 @@ import SWEET_python.defaults_2019 as defaults_2019 import SWEET_python.mcf as mcf_defaults import SWEET_python.site_type_defaults as site_type_defaults -from SWEET_python.population import average_growth_rates, country_population_series +from SWEET_python.population import ( + average_growth_rates, + city_population_series, + country_population_series, + parse_city_population, +) import psycopg2 from psycopg2.extras import RealDictCursor from sqlalchemy import create_engine, text @@ -3666,6 +3671,14 @@ def import_basics_site(self, canonical_row, pop_data, usecase='wastemap', time_s # WPP2024 per-year population series for this country (None -> the # frozen-CAGR growth_rate_* scalars are used instead). population_series = _population_series_from_pop_data(pop_data, self.iso3) + # Climate TRACE may hand in the site's nearby city's population. Where it does, + # the site grows with that city, and with the country's shape outside the + # city's years (population.city_population_series). + city_series = city_population_series( + parse_city_population(canonical_row.get("city_population_by_year")), population_series + ) + if city_series is not None: + population_series = city_series # True once waste_generated already carries population growth (the # multi-row path bakes it into the projected series below), so the # later WasteGeneratedDF.create must not apply growth a second time. diff --git a/SWEET_python/population.py b/SWEET_python/population.py index 24125e4..4240d43 100644 --- a/SWEET_python/population.py +++ b/SWEET_python/population.py @@ -16,10 +16,13 @@ from __future__ import annotations +import json +from collections.abc import Mapping from functools import lru_cache from pathlib import Path from typing import Optional +import numpy as np import pandas as pd from SWEET_python.constants import MODEL_END_YEAR, MODEL_START_YEAR @@ -66,3 +69,79 @@ def average_growth_rates( historic = future if historic is None else historic future = historic if future is None else future return float(historic), float(future) + + +def parse_city_population(value) -> Optional[dict[int, float]]: + """A city's population by year, from a mapping or its JSON text, or ``None``. + + Climate TRACE hands a site's nearby city in as JSON text on the site row, because a + row is saved to parquet between stages and text survives that. Keys may be ints or + the strings JSON makes of them. Years with no positive number are dropped: the UN + leaves a city blank in the years it was under 50,000 people, and a blank means "not + a city yet", not zero people. + """ + # Any scalar missing value (None, NaN, pd.NA, NaT); is_scalar keeps a mapping or + # text away from pd.isna. + if value is None or (pd.api.types.is_scalar(value) and pd.isna(value)): + return None + if isinstance(value, str): + if not value.strip(): + return None + value = json.loads(value) + if not isinstance(value, Mapping): + raise TypeError(f"city population must be a mapping of year to people, not {type(value).__name__}") + years = {} + for year, people in value.items(): + people = pd.to_numeric(people, errors="coerce") + if pd.notna(people) and people > 0: + years[int(year)] = float(people) + return years or None + + +def city_population_series( + city: Optional[Mapping[int, float]], country: Optional[pd.Series] +) -> Optional[pd.Series]: + """A city's population in every year of ``country``'s index, its gaps filled. + + - **In a year the city has a number**, that number. + - **Between two of them**, constant growth from one to the next (log-linear), where a + year is missing. Climate TRACE's supplement lists every year from a city's first to + 2050 (waste_data_ingestion_pipeline #299), so this only bridges a gap. + - **Before its first year or after its last**, the city's nearest number moved by + the country's population: ``city[first] * country[year] / country[first]``. A city + appears in the UN's list only once it passes 50,000 people, so half the cities a + site is near start after 1975 and a quarter after 2000. + + The country's shape fills the ends because the other ways bend: a straight line back + from a fast-growing city reaches zero within a few decades, and its growth rate + compounded back shrinks it to nearly nothing. The country's shape joins smoothly, + never goes below zero, and is what a site with no city gets anyway. A city grows + faster than its country, so it slightly overstates a city's population before its + first year. + + ``None`` when there is no city, no city year inside the country's years, or no + country series with a positive number in every year (the city's ends are scaled by + it, so a gap would leave a gap); the caller then grows by the country alone. + """ + # decision: population-growth + if not city or country is None: + return None + if country.isna().any() or not (country > 0).all(): + return None + observed = pd.Series(city, dtype=float).sort_index() + observed = observed[observed.index.isin(country.index)] + if observed.empty: + return None + + series = pd.Series(np.nan, index=country.index, dtype=float) + series[observed.index] = observed.values + first, last = observed.index[0], observed.index[-1] + inside = (series.index >= first) & (series.index <= last) + # Only the missing years: a reported number survives exactly, not via exp(log(x)). + gaps = inside & series.isna().to_numpy() + series[gaps] = np.exp(np.log(series[inside]).interpolate(method="index"))[gaps[inside]] + before = series.index < first + series[before] = observed[first] * country[before] / country[first] + after = series.index > last + series[after] = observed[last] * country[after] / country[last] + return series diff --git a/changelog/2026-10.md b/changelog/2026-10.md index dce0df1..98fca53 100644 --- a/changelog/2026-10.md +++ b/changelog/2026-10.md @@ -206,6 +206,16 @@ heavily: Yokohama's city DST methane falls 74% without it (model-output change). - SWEET now depends on scipy (`requirements.in`; the lockfile adds `scipy==1.18.1` and moves no other pin). It is imported only where the line is fitted, so WasteMAP, which installs SWEET with `--no-deps` and has no scipy, still imports `city_params`. - Tests: `tests/test_site_intake_series.py`. ([#113](https://github.com/RMI/SWEET_python/pull/113)) +- **A Climate TRACE site can grow with the city near it, not its country** + (model-output change for Climate TRACE only). A site row may carry + `city_population_by_year`, the nearby city's UN World Urbanization Prospects 2025 + population by year, as a mapping or its JSON text. SWEET grows the site's waste + with that series instead of the country's. A city is listed only once it passes + 50,000 people, so before its first year (and after its last) the city's nearest + number moves with the country's population; between two reported years growth is + constant. A row without the column grows with the country, as before, and no + WasteMAP path passes one. `population.parse_city_population`, + `population.city_population_series`. Tests: `tests/test_site_city_population.py`. - **The city paths flare captured gas at 0.98, as every other path does** (model-output change, [#105](https://github.com/RMI/SWEET_python/pull/105)). A landfill with no flaring series used to have all of its diff --git a/tests/test_site_city_population.py b/tests/test_site_city_population.py new file mode 100644 index 0000000..625899f --- /dev/null +++ b/tests/test_site_city_population.py @@ -0,0 +1,141 @@ +"""A Climate TRACE site can grow with its nearby city instead of its country. + +Climate TRACE hands in the population of the UN-listed city near a site, as JSON text on +the site row. SWEET uses the city's own numbers where it has them and the country's shape +outside them (`population.city_population_series`). A row without a city grows with the +country, as before. +""" + +import json + +import numpy as np +import pandas as pd +import pytest + +from SWEET_python.city_params import City +from SWEET_python.population import ( + city_population_series, + country_population_series, + parse_city_population, +) + +WINDOW = list(range(1970, 2051)) +USA = country_population_series("USA").loc[WINDOW] + + +# --------------------------------------------------------------------------- # +# Reading the city +# --------------------------------------------------------------------------- # + +def test_json_text_with_string_years_reads_as_int_years(): + assert parse_city_population('{"1975": 50000, "2025": 80000}') == {1975: 50000.0, 2025: 80000.0} + + +@pytest.mark.parametrize("blank", [None, np.nan, pd.NA, pd.NaT, "", "{}"]) +def test_no_city_reads_as_none(blank): + assert parse_city_population(blank) is None + + +def test_years_under_the_threshold_are_dropped_not_read_as_zero(): + """The UN leaves a city blank in the years it was under 50,000 people.""" + assert parse_city_population({1975: None, 1980: 0, 1985: 51000}) == {1985: 51000.0} + + +# --------------------------------------------------------------------------- # +# Filling the city's gaps +# --------------------------------------------------------------------------- # + +def test_the_citys_own_years_are_kept_exactly(): + city = {year: 100_000.0 + 1_000 * (year - 1975) for year in range(1975, 2051)} + series = city_population_series(city, USA) + assert series.loc[1975:2050].tolist() == [city[y] for y in range(1975, 2051)] + + +def test_years_before_the_city_follow_the_countrys_shape_and_join_smoothly(): + """A city first listed in 2005 at 50,000: earlier years move with the country.""" + series = city_population_series({2005: 50_000.0, 2025: 200_000.0}, USA) + assert series[2005] == 50_000.0 + for year in (1970, 1990, 2004): + assert series[year] == pytest.approx(50_000.0 * USA[year] / USA[2005]) + + +def test_years_after_the_city_follow_the_countrys_shape(): + series = city_population_series({1975: 60_000.0, 2025: 90_000.0}, USA) + assert series[2050] == pytest.approx(90_000.0 * USA[2050] / USA[2025]) + + +def test_between_two_reported_years_growth_is_constant(): + """For a year missing between two listed ones.""" + series = city_population_series({2000: 100_000.0, 2005: 200_000.0}, USA) + rate = 2 ** (1 / 5) + assert series[2001] == pytest.approx(100_000.0 * rate) + assert series[2004] == pytest.approx(100_000.0 * rate ** 4) + + +def test_a_fast_city_never_reaches_zero_going_back(): + """A straight line back from 50,000 in 2005 to 200,000 in 2025 hits zero by 1998.""" + series = city_population_series({2005: 50_000.0, 2025: 200_000.0}, USA) + assert (series > 0).all() + assert series.index.tolist() == WINDOW + + +@pytest.mark.parametrize("city, country", [(None, USA), ({2005: 1.0}, None), ({1960: 1.0}, USA)]) +def test_without_a_city_or_country_or_overlap_there_is_no_series(city, country): + assert city_population_series(city, country) is None + + +@pytest.mark.parametrize("year, value", [(1970, np.nan), (2050, np.nan), (2005, 0.0)]) +def test_a_country_with_a_gap_or_no_people_gives_no_series(year, value): + """The city's ends are scaled by the country, so a gap there would leave a gap.""" + country = USA.copy() + country[year] = value + assert city_population_series({2005: 50_000.0, 2025: 200_000.0}, country) is None + + +# --------------------------------------------------------------------------- # +# Through a site +# --------------------------------------------------------------------------- # + +def _site(city_population_by_year, **extra): + row = pd.Series({ + "asset_identifier": 999802, "iso3_country": "USA", "type": "Sanitary Landfill", + "income_id": "HIC", "annual_incoming_waste": 100_000.0, "incoming_waste_year": 2023.0, + "site_open_year": 1980.0, "site_close_year": np.nan, "waste_depth": 10.0, + "landfill_gas_collection": False, "gas_collection_efficiency": np.nan, + "latitude": 39.0, "longitude": -95.0, "precipitation": 800.0, "temperature": 15.0, + "city_population_by_year": city_population_by_year, + **extra, + }) + pop_data = pd.DataFrame( + {"growth_rate_historic": [1.02], "growth_rate_future": [1.01], + **{f"pop_{y}": [USA[y]] for y in WINDOW}}, + index=["USA"], + ) + city = City("999802") + city.site_only_estimate_trace(row, pop_data=pop_data) + return city.baseline_parameters + + +def _shape(series, anchor=2023): + total = series.sum(axis=1) if isinstance(series, pd.DataFrame) else series + return (total / total.loc[anchor]).loc[WINDOW].to_numpy() + + +def test_a_site_with_a_city_grows_with_the_city(): + city = {2005: 50_000.0, 2025: 200_000.0} + baseline = _site(json.dumps(city)) + expected = city_population_series(city, USA) + assert np.allclose(_shape(baseline.waste_generated_df.df), _shape(expected)) + assert np.allclose(baseline.population_series.loc[WINDOW], expected) + + +def test_a_site_with_no_city_still_grows_with_its_country(): + baseline = _site(None) + assert np.allclose(_shape(baseline.waste_generated_df.df), _shape(USA)) + + +def test_a_single_city_population_number_on_a_site_row_is_not_read(): + """``city_population`` is the Brazil city-site path's one number per city; a site + row grows only with ``city_population_by_year``.""" + baseline = _site(None, city_population=1_500_000.0) + assert np.allclose(_shape(baseline.waste_generated_df.df), _shape(USA))