From 08626ee42861dc782f87e90384d0643c3d1eb0c9 Mon Sep 17 00:00:00 2001 From: Vivek Vasisht Ediga <140938681+ViVas970811@users.noreply.github.com> Date: Mon, 26 Oct 2026 11:20:03 -0400 Subject: [PATCH 1/4] Remove process/canopy; the canopy data pull now lives in tree-canopy --- process/canopy/canopy-process.py | 85 -------------------------------- 1 file changed, 85 deletions(-) delete mode 100644 process/canopy/canopy-process.py diff --git a/process/canopy/canopy-process.py b/process/canopy/canopy-process.py deleted file mode 100644 index 893736a..0000000 --- a/process/canopy/canopy-process.py +++ /dev/null @@ -1,85 +0,0 @@ -""" -Pull recent forest canopy cover for every US county from Google Data Commons and write -the two files the Forest Canopy parameters use: - - forest-canopy-timeseries--.csv Fips, Name, , , ... (percent forest cover) - forest-canopy-target.csv Fips, Target (1 = growth >= threshold) - -This replaces the one-off Forest Canopy colab described in github.com/ModelEarth/tree-canopy: -variable LandCoverFraction_Forest, relative growth between the first and last year of the -window, counties at or above the threshold labelled 1. - - python process/canopy/canopy-process.py # last 5 available years, 5 percent - python process/canopy/canopy-process.py --years 4 --threshold 5 --out ../tree-canopy/input/targets - -Needs DATACOMMONS_API_KEY (environment, OS credential store, or the .env named in -automation/paths.yaml; see api/keystore.md). Request a key at https://apikeys.datacommons.org/. -""" -import argparse -import os -import sys - -import pandas as pd - -HERE = os.path.dirname(os.path.abspath(__file__)) -sys.path.insert(0, os.path.join(HERE, "..", "..", "api")) -from run_models import require_env # noqa: E402 - -VARIABLE = "LandCoverFraction_Forest" - - -def fetch_county_forest_cover(api_key): - """Long table: Fips, Name, year, value for every US county and every year Data Commons has.""" - from datacommons_client import DataCommonsClient - - client = DataCommonsClient(api_key=api_key) - obs = client.observations_dataframe( - variable_dcids=[VARIABLE], date="all", parent_entity="country/USA", entity_type="County" - ) - if obs is None or obs.empty: - raise RuntimeError(f"Data Commons returned no {VARIABLE} observations for US counties.") - obs = obs.rename(columns={"entity": "Fips", "entity_name": "Name"}) - obs["Fips"] = obs["Fips"].astype(str).str.replace("geoId/", "", regex=False).str.zfill(5) - obs["year"] = obs["date"].astype(str).str[:4].astype(int) - if "Name" not in obs.columns: - obs["Name"] = "" - obs["Name"] = obs["Name"].astype(str).str.replace(" County", "", regex=False) - return obs[["Fips", "Name", "year", "value"]] - - -def build_outputs(obs, years, threshold): - """(timeseries wide table, target table, first_year, last_year) from the long observations.""" - available = sorted(obs["year"].unique()) - window = available[-years:] if years else available - first, last = window[0], window[-1] - wide = obs[obs["year"].isin(window)].pivot_table(index=["Fips", "Name"], columns="year", values="value", - aggfunc="median").reset_index() - wide.columns = [str(c) for c in wide.columns] - start, recent = wide[str(first)], wide[str(last)] - growth = (recent - start) / start.replace(0, pd.NA) * 100 - target = pd.DataFrame({"Fips": wide["Fips"], "Target": (growth >= threshold).fillna(False).astype(int)}) - return wide, target, first, last - - -def main(): - parser = argparse.ArgumentParser(description="Forest canopy timeseries and target from Data Commons") - parser.add_argument("--years", type=int, default=5, help="how many of the most recent years to keep (default 5)") - parser.add_argument("--threshold", type=float, default=5.0, help="percent growth labelled 1 (default 5)") - parser.add_argument("--out", default=os.path.join(HERE, "output"), help="output folder") - args = parser.parse_args() - - obs = fetch_county_forest_cover(require_env("DATACOMMONS_API_KEY")) - wide, target, first, last = build_outputs(obs, args.years, args.threshold) - - os.makedirs(args.out, exist_ok=True) - ts_path = os.path.join(args.out, f"forest-canopy-timeseries-{first}-{last}.csv") - target_path = os.path.join(args.out, "forest-canopy-target.csv") - wide.to_csv(ts_path, index=False) - target.to_csv(target_path, index=False) - print(f"{len(wide)} counties, years {first} to {last}") - print(f"{int(target['Target'].sum())} counties at or above {args.threshold:g} percent growth") - print(f"wrote {ts_path}\nwrote {target_path}") - - -if __name__ == "__main__": - main() From ed6e201024162b9a8e75397336b6f0a8f4f30c4b Mon Sep 17 00:00:00 2001 From: Vivek Vasisht Ediga <140938681+ViVas970811@users.noreply.github.com> Date: Mon, 26 Oct 2026 11:20:59 -0400 Subject: [PATCH 2/4] Delete tests/test_canopy_process.py --- tests/test_canopy_process.py | 23 ----------------------- 1 file changed, 23 deletions(-) delete mode 100644 tests/test_canopy_process.py diff --git a/tests/test_canopy_process.py b/tests/test_canopy_process.py deleted file mode 100644 index 5e27901..0000000 --- a/tests/test_canopy_process.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Offline check of process/canopy/canopy-process.py (the Data Commons pull itself needs a key).""" -import importlib.util -import os - -import pandas as pd - -path = os.path.join(os.path.dirname(__file__), "..", "process", "canopy", "canopy-process.py") -spec = importlib.util.spec_from_file_location("canopy_process", path) -cp = importlib.util.module_from_spec(spec) -spec.loader.exec_module(cp) - - -def test_growth_window_and_threshold(): - rows = [] - for fips, name, cover in (("01001", "Autauga", [40, 41, 42, 43]), ("01003", "Baldwin", [50, 50, 50, 50]), - ("01005", "Barbour", [20, 20, 20, 19])): - for year, value in zip((2016, 2017, 2018, 2019), cover): - rows.append({"Fips": fips, "Name": name, "year": year, "value": value}) - obs = pd.DataFrame(rows) - wide, target, first, last = cp.build_outputs(obs, years=3, threshold=4.0) - assert (first, last) == (2017, 2019) - assert list(wide.columns) == ["Fips", "Name", "2017", "2018", "2019"] - assert dict(zip(target["Fips"], target["Target"])) == {"01001": 1, "01003": 0, "01005": 0} # 41->43 is 4.9% From 53ec79f8e60645268247c1a621907d4f3fbe19af Mon Sep 17 00:00:00 2001 From: Vivek Vasisht Ediga <140938681+ViVas970811@users.noreply.github.com> Date: Mon, 26 Oct 2026 11:23:27 -0400 Subject: [PATCH 3/4] projects.md: canopy data pull now links to the tree-canopy repo Updated the status of the automated pull of forest canopy data, including the new file path and added links to the relevant pull request. --- projects.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects.md b/projects.md index c78ea9e..5a3581e 100644 --- a/projects.md +++ b/projects.md @@ -14,7 +14,7 @@ Write Loren when you've submitted a pull request to show your name. 4. DONE: Send report files from the local python run to GitHub (`api/run_models.py --upload`, `push_report`; the Run Models page pushes with a browser-saved token). Set `GITHUB_REPORTS_REPO` to push to your own fork of [reports](https://github.com/ModelEarth/reports) when testing. - Vivek 5. PARTIALLY DONE: Add output files to the "[report](https://github.com/ModelEarth/reports/)" folder and add smart formatting in the index.html and javascript report display. We send its content to GitHub as a final step in run_models.py. 6. DONE: Run the Forest Canopy and all-years parameter files through `api/run_models.py` unchanged: join column aliases (`id`, `join.key`, `targets.column`), 5-digit FIPS matching, index-column drop, readable NAICS names in feature importance. - Vivek -7. IN PROGRESS: Automated pull of recent forest canopy data from Google Data Commons (`process/canopy/canopy-process.py`), replacing the manually added target file and the missing timeseries file in [tree-canopy](https://github.com/ModelEarth/tree-canopy). Needs a `DATACOMMONS_API_KEY`. - Vivek +7. IN PROGRESS: Automated pull of recent forest canopy data from Google Data Commons, now in the tree-canopy repo as [pipeline/canopy-process.py](https://github.com/ModelEarth/tree-canopy/blob/main/pipeline/canopy-process.py) ([tree-canopy PR #1](https://github.com/ModelEarth/tree-canopy/pull/1)). Writes the timeseries and target CSVs the Forest Canopy parameters use. Needs a `DATACOMMONS_API_KEY`. - Vivek 8. TO DO: Decide what the `timeseries` section of a parameters file should drive (extra features per year, a trend target, or a chart in the report). `run_models.py` currently reports whether the file is reachable and does nothing else with it. 9. TO DO: Pull 2-column targets (location id, Target) for UN topics directly from Google Data Commons using `dcid` values in parameters.yaml; `parameters-gdc.yaml` and `parameters_un_gcd.yaml` are the starting points. - Vivek (Nov 5) 10. TO DO: More parameters files that join features and targets on county Fips, including the [all-years](https://github.com/ModelEarth/community-timelines/tree/main/training/all-years) features for states beyond Minnesota (one wide row per county, `Emp--` columns). From c37aaa35423370dbf8094796901a0c3c17bea175 Mon Sep 17 00:00:00 2001 From: Vivek Vasisht Ediga <140938681+ViVas970811@users.noreply.github.com> Date: Mon, 26 Oct 2026 11:24:05 -0400 Subject: [PATCH 4/4] README: DATACOMMONS_API_KEY line points at the canopy pull in tree-canopy Clarified the usage of DATACOMMONS_API_KEY for the canopy data pull. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 67f28fe..cae95a3 100644 --- a/README.md +++ b/README.md @@ -127,7 +127,7 @@ Settings are read one name at a time: first from the environment, then from the - `GITHUB_REPORTS_TOKEN` for `--upload` (GitHub token with write access to the reports repo) - `GITHUB_REPORTS_REPO` (default `modelearth/reports`) and `GITHUB_REPORTS_SITE` (default `https://model.earth/reports/`), to push somewhere else, for example your own fork while testing -- `DATACOMMONS_API_KEY` for parameter files that use `dcid` and for `process/canopy/canopy-process.py` (request one at [apikeys.datacommons.org](https://apikeys.datacommons.org/)) +- `DATACOMMONS_API_KEY` for parameter files that use `dcid`, and for the canopy data pull in [tree-canopy](https://github.com/ModelEarth/tree-canopy/blob/main/pipeline/canopy-process.py) (request one at [apikeys.datacommons.org](https://apikeys.datacommons.org/)) - `ENABLE_GPU=true` to use cuML models when RAPIDS is installed (falls back to scikit-learn otherwise) When a key is missing, the CLI and the Flask `/run` endpoint explain where to get it.