From 3c32c7c28dc9dce73c475d4e407dd766d1f0e69c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 31 Oct 2026 06:12:05 -0400 Subject: [PATCH 01/27] Grow alcohol and tobacco quantities with the OBR's clearances per head The importer now reads the OBR's tobacco clearances (Table 4.11) and alcohol clearance forecasts (Table 4.12) and writes six per-head series under gov.economic_assumptions.yoy_growth.obr.per_capita, one per product group: clearance growth over one plus ons.population growth, to six decimals. Dataset projection grows household weights by ons.population, so weighted quantities grow with the clearances. Each quantity input declares its group's index and is mapped to its series in uprating_indices.yaml. The real-table fixtures gain the clearance sheets, with each kept sheet's XML and the shared strings unchanged. Fixes #2596. Co-Authored-By: Claude Opus 5.5 --- policyengine_uk/data/uprating_indices.yaml | 19 + .../gov/economic_assumptions/README.md | 27 + .../gov/economic_assumptions/yoy_growth.yaml | 324 +++++++++++- .../tests/fixtures/obr_receipts/README.md | 10 +- .../obr_efo_march_2025_receipts.xlsx | Bin 48486 -> 53880 bytes .../obr_efo_march_2026_receipts.xlsx | Bin 37689 -> 41807 bytes .../obr_efo_october_2026_receipts.xlsx | Bin 46977 -> 53369 bytes .../tests/test_excise_quantity_projection.py | 487 +++++++++++++++++ .../tests/test_import_obr_forecasts.py | 496 +++++++++++++++++- policyengine_uk/utils/import_obr_forecasts.py | 274 ++++++++-- .../input/consumption/alcohol/beer_litres.py | 1 + .../input/consumption/alcohol/cider_litres.py | 1 + .../alcohol/other_fermented_litres.py | 1 + .../alcohol/sparkling_cider_litres.py | 1 + .../consumption/alcohol/spirits_litres.py | 3 + .../input/consumption/alcohol/wine_litres.py | 1 + .../tobacco/cigarettes_per_week.py | 3 + .../tobacco/cigars_grams_per_week.py | 1 + .../hand_rolling_tobacco_grams_per_week.py | 1 + .../tobacco/heated_tobacco_grams_per_week.py | 1 + .../tobacco/other_tobacco_grams_per_week.py | 1 + 21 files changed, 1604 insertions(+), 48 deletions(-) create mode 100644 policyengine_uk/tests/test_excise_quantity_projection.py diff --git a/policyengine_uk/data/uprating_indices.yaml b/policyengine_uk/data/uprating_indices.yaml index d21df43f29..e0fe004ba3 100644 --- a/policyengine_uk/data/uprating_indices.yaml +++ b/policyengine_uk/data/uprating_indices.yaml @@ -196,6 +196,25 @@ gov.economic_assumptions.yoy_growth.obr.gambling_duty_bases.bingo: - bingo_net_receipts gov.economic_assumptions.yoy_growth.obr.gambling_duty_bases.lottery: - national_lottery_ticket_spending +# Alcohol and tobacco quantities grow with the OBR's clearances of their +# product group per person, as household weights grow by population (#2596). +gov.economic_assumptions.yoy_growth.obr.per_capita.tobacco_clearances.cigarettes: +- cigarettes_per_week +gov.economic_assumptions.yoy_growth.obr.per_capita.tobacco_clearances.hand_rolling_tobacco: +- hand_rolling_tobacco_grams_per_week +gov.economic_assumptions.yoy_growth.obr.per_capita.tobacco_clearances.other_tobacco_products: +- cigars_grams_per_week +- heated_tobacco_grams_per_week +- other_tobacco_grams_per_week +gov.economic_assumptions.yoy_growth.obr.per_capita.alcohol_clearances.beer_and_cider: +- beer_litres +- cider_litres +- sparkling_cider_litres +gov.economic_assumptions.yoy_growth.obr.per_capita.alcohol_clearances.wine: +- other_fermented_litres +- wine_litres +gov.economic_assumptions.yoy_growth.obr.per_capita.alcohol_clearances.spirits: +- spirits_litres gov.economic_assumptions.yoy_growth.obr.private_pension_index: - private_pension_income gov.economic_assumptions.yoy_growth.obr.lagged_average_earnings: diff --git a/policyengine_uk/parameters/gov/economic_assumptions/README.md b/policyengine_uk/parameters/gov/economic_assumptions/README.md index ab0173d409..0efafedc02 100644 --- a/policyengine_uk/parameters/gov/economic_assumptions/README.md +++ b/policyengine_uk/parameters/gov/economic_assumptions/README.md @@ -36,6 +36,8 @@ GDP and population assumptions remain supported. | `business_rates.great_britain` | Business rates receipts, England, Scotland and Wales (cash) | EFO receipts, current receipts (cash basis): Table 4.8 in October 2026 | | `business_rates.northern_ireland` | Northern Ireland business rates receipts | EFO receipts, other taxes: Table 4.2 in October 2026 | | `gambling_duty_bases.` | The base of each of the seven betting and gaming duties (receipts over the model's rate) | HMRC Betting and Gaming tables (2024-25, 2025-26), then EFO receipts, betting and gaming duties forecast breakdown: Table 4.10 in October 2026 | +| `per_capita.tobacco_clearances.` | Cigarette, hand-rolled tobacco and other tobacco clearances per person | EFO receipts, tobacco clearances: Table 4.11 in October 2026 (3.11 in March 2026), over `ons.population` | +| `per_capita.alcohol_clearances.` | Beer and cider, wine and spirits clearances per person | EFO receipts, alcohol clearance forecasts: Table 4.12 in October 2026 (3.12 in March 2026), over `ons.population` | The four receipts series project the corporate SDLT and business rates statistics (`gov.hmrc.stamp_duty.statistics`, @@ -60,6 +62,19 @@ model's base year, and its 2025-26 outturn, which the importer holds in for each duty's rates, the higher-stake share Machine Games Duty's rate assumes, and the approximations. +The six clearance series under `per_capita` grow the alcohol and tobacco +quantity inputs (#2596): `tobacco_clearances.cigarettes`, +`hand_rolling_tobacco` and `other_tobacco_products`, and +`alcohol_clearances.beer_and_cider`, `wine` and `spirits`. Each is the +growth in the OBR's clearances of a product group over one plus +`ons.population` growth, to six decimals, because dataset projection grows +household weights by `ons.population`. They link their outturn year to the +previous EFO's outturn, as the four receipts series do, and hold the +clearances flat after the forecast. The importer reads `ons.population` from +the `yoy_growth.yaml` it rewrites. See +[household excise duties](../../../../docs/engineering/excise-duties.md#projection) +for the mapping of inputs to groups, the arithmetic and the sources. + `national_living_wage` projects the minimum wage rates (`gov.hmrc.minimum_wage`) after the last rate set. It is the growth of the rates set to 2027, then the OBR's estimates in Table 1.13 of the detailed @@ -483,6 +498,18 @@ the forecast levels, though not their growth. and extend the Machine Games Duty measure's receipts when the forecast runs past 2031-32. Then update the published levels in `policyengine_uk/tests/test_gambling_duty_base_projection.py`. + The same run also refreshes the six alcohol and tobacco clearance series + under `per_capita` from the new EFO's tobacco clearances and alcohol + clearance forecasts, and the previous EFO's for the outturn year's base. + Each line's table must state its unit, in a row label or a note, in both + EFOs: million sticks for cigarettes, thousand kilograms for the other + tobacco lines, and thousand hectolitres of alcohol for alcohol. The + importer stops on any other. It divides each year's growth by + `ons.population` as `yoy_growth.yaml` holds it, so rerun it whenever + that series changes. Then update the published levels in + `policyengine_uk/tests/test_excise_quantity_projection.py` + (`CLEARANCES`) and the real-table fixtures in + `policyengine_uk/tests/fixtures/obr_receipts/` (see its README). 7. If the new EFO runs past `PROJECTION_END_YEAR`, move it on (see [Dataset projection horizon](#dataset-projection-horizon)). 8. Refresh the household energy price inputs (see diff --git a/policyengine_uk/parameters/gov/economic_assumptions/yoy_growth.yaml b/policyengine_uk/parameters/gov/economic_assumptions/yoy_growth.yaml index acef315895..d09bedb97d 100644 --- a/policyengine_uk/parameters/gov/economic_assumptions/yoy_growth.yaml +++ b/policyengine_uk/parameters/gov/economic_assumptions/yoy_growth.yaml @@ -1299,6 +1299,327 @@ obr: - title: OBR, Economic and fiscal outlook October 2026, detailed forecast tables, receipts, Table 4.5, divided by ons.population growth href: https://obr.uk/efo/economic-and-fiscal-outlook-october-2026/ + # Alcohol and tobacco clearances per person (#2596), which project the + # household quantity inputs (data/uprating_indices.yaml). Each is the + # growth in the OBR's clearances of a product group over one plus + # ons.population growth, to six decimals: (1 + clearance growth) / + # (1 + population growth) - 1. Dataset projection grows household + # weights by ons.population, so the weighted quantity grows with the + # clearances. The outturn year's growth divides this EFO's outturn by the + # previous EFO's outturn for the year before. After the OBR's forecast + # the clearances hold flat, as road fuel volume does. + # policyengine_uk/utils/import_obr_forecasts.py rewrites these series + # from a new EFO's receipts tables and the ons.population growth below, + # and keeps the 2024 base year. + tobacco_clearances: + cigarettes: + description: Growth per person in the OBR's cigarette clearances, which projects each household's cigarettes_per_week. + values: + # Base year (2024-25): the alcohol and tobacco duties start in model year 2024 + 2024-01-01: 0.0000 + # Outturn (2025-26 over 2024-25) from OBR EFO October 2026 over March 2026, receipts Table 4.11 (March 2026: Table 3.11) "Cigarette clearances (million sticks)"; per head: clearance growth over one plus ons.population growth + 2025-01-01: -0.102653 + # Forecast (2026-2031) from OBR EFO October 2026, receipts Table 4.11; per head: clearance growth over one plus ons.population growth + 2026-01-01: -0.105009 + 2027-01-01: -0.102347 + 2028-01-01: -0.083831 + 2029-01-01: -0.092617 + 2030-01-01: -0.068657 + 2031-01-01: -0.06894 + # Beyond the OBR forecast: clearances held flat, so per head they fall with ons.population growth + 2032-01-01: -0.002195 + 2033-01-01: -0.002096 + 2034-01-01: -0.001896 + 2035-01-01: -0.001797 + 2036-01-01: -0.001697 + 2037-01-01: -0.001597 + 2038-01-01: -0.001498 + 2040-01-01: -0.001398 + 2042-01-01: -0.001298 + 2044-01-01: -0.001199 + 2045-01-01: -0.001099 + 2047-01-01: -0.000999 + 2048-01-01: -0.000899 + 2049-01-01: -0.000799 + 2050-01-01: -0.0006 + 2051-01-01: -0.0005 + 2052-01-01: -0.0003 + 2053-01-01: -0.0002 + 2054-01-01: 0.0000 + 2055-01-01: 0.0001 + 2056-01-01: 0.0003 + 2057-01-01: 0.0004 + 2058-01-01: 0.0005 + 2059-01-01: 0.0006 + 2060-01-01: 0.0007 + 2061-01-01: 0.000801 + 2064-01-01: 0.000901 + 2071-01-01: 0.001001 + metadata: + unit: /1 + label: Cigarette clearances growth per person + reference: + - title: OBR EFO October 2026 (detailed forecast tables, receipts, Table 4.11) + href: https://obr.uk/efo/economic-and-fiscal-outlook-october-2026/ + - title: OBR EFO March 2026 (detailed forecast tables, receipts, Table 3.11; 2024-25 outturn) + href: https://obr.uk/efo/economic-and-fiscal-outlook-march-2026/ + hand_rolling_tobacco: + description: Growth per person in the OBR's hand-rolled tobacco clearances, which projects each household's hand_rolling_tobacco_grams_per_week. + values: + # Base year (2024-25): the alcohol and tobacco duties start in model year 2024 + 2024-01-01: 0.0000 + # Outturn (2025-26 over 2024-25) from OBR EFO October 2026 over March 2026, receipts Table 4.11 (March 2026: Table 3.11) "Hand rolled tobacco clearances"; per head: clearance growth over one plus ons.population growth + 2025-01-01: -0.17412 + # Forecast (2026-2031) from OBR EFO October 2026, receipts Table 4.11; per head: clearance growth over one plus ons.population growth + 2026-01-01: -0.069369 + 2027-01-01: -0.073763 + 2028-01-01: -0.083858 + 2029-01-01: -0.064999 + 2030-01-01: -0.041711 + 2031-01-01: -0.048922 + # Beyond the OBR forecast: clearances held flat, so per head they fall with ons.population growth + 2032-01-01: -0.002195 + 2033-01-01: -0.002096 + 2034-01-01: -0.001896 + 2035-01-01: -0.001797 + 2036-01-01: -0.001697 + 2037-01-01: -0.001597 + 2038-01-01: -0.001498 + 2040-01-01: -0.001398 + 2042-01-01: -0.001298 + 2044-01-01: -0.001199 + 2045-01-01: -0.001099 + 2047-01-01: -0.000999 + 2048-01-01: -0.000899 + 2049-01-01: -0.000799 + 2050-01-01: -0.0006 + 2051-01-01: -0.0005 + 2052-01-01: -0.0003 + 2053-01-01: -0.0002 + 2054-01-01: 0.0000 + 2055-01-01: 0.0001 + 2056-01-01: 0.0003 + 2057-01-01: 0.0004 + 2058-01-01: 0.0005 + 2059-01-01: 0.0006 + 2060-01-01: 0.0007 + 2061-01-01: 0.000801 + 2064-01-01: 0.000901 + 2071-01-01: 0.001001 + metadata: + unit: /1 + label: Hand-rolled tobacco clearances growth per person + reference: + - title: OBR EFO October 2026 (detailed forecast tables, receipts, Table 4.11) + href: https://obr.uk/efo/economic-and-fiscal-outlook-october-2026/ + - title: OBR EFO March 2026 (detailed forecast tables, receipts, Table 3.11; 2024-25 outturn) + href: https://obr.uk/efo/economic-and-fiscal-outlook-march-2026/ + other_tobacco_products: + description: Growth per person in the OBR's clearances of other tobacco products, which projects each household's cigars_grams_per_week, other_tobacco_grams_per_week and heated_tobacco_grams_per_week. + values: + # Base year (2024-25): the alcohol and tobacco duties start in model year 2024 + 2024-01-01: 0.0000 + # Outturn (2025-26 over 2024-25) from OBR EFO October 2026 over March 2026, receipts Table 4.11 (March 2026: Table 3.11) "Other tobacco products"; per head: clearance growth over one plus ons.population growth + 2025-01-01: -0.13903 + # Forecast (2026-2031) from OBR EFO October 2026, receipts Table 4.11; per head: clearance growth over one plus ons.population growth + 2026-01-01: -0.053308 + 2027-01-01: -0.042058 + 2028-01-01: -0.083858 + 2029-01-01: -0.050588 + 2030-01-01: -0.039029 + 2031-01-01: -0.058055 + # Beyond the OBR forecast: clearances held flat, so per head they fall with ons.population growth + 2032-01-01: -0.002195 + 2033-01-01: -0.002096 + 2034-01-01: -0.001896 + 2035-01-01: -0.001797 + 2036-01-01: -0.001697 + 2037-01-01: -0.001597 + 2038-01-01: -0.001498 + 2040-01-01: -0.001398 + 2042-01-01: -0.001298 + 2044-01-01: -0.001199 + 2045-01-01: -0.001099 + 2047-01-01: -0.000999 + 2048-01-01: -0.000899 + 2049-01-01: -0.000799 + 2050-01-01: -0.0006 + 2051-01-01: -0.0005 + 2052-01-01: -0.0003 + 2053-01-01: -0.0002 + 2054-01-01: 0.0000 + 2055-01-01: 0.0001 + 2056-01-01: 0.0003 + 2057-01-01: 0.0004 + 2058-01-01: 0.0005 + 2059-01-01: 0.0006 + 2060-01-01: 0.0007 + 2061-01-01: 0.000801 + 2064-01-01: 0.000901 + 2071-01-01: 0.001001 + metadata: + unit: /1 + label: Other tobacco products clearances growth per person + reference: + - title: OBR EFO October 2026 (detailed forecast tables, receipts, Table 4.11) + href: https://obr.uk/efo/economic-and-fiscal-outlook-october-2026/ + - title: OBR EFO March 2026 (detailed forecast tables, receipts, Table 3.11; 2024-25 outturn) + href: https://obr.uk/efo/economic-and-fiscal-outlook-march-2026/ + alcohol_clearances: + beer_and_cider: + description: Growth per person in the OBR's beer and cider clearances, which projects each household's beer_litres, cider_litres and sparkling_cider_litres. + values: + # Base year (2024-25): the alcohol and tobacco duties start in model year 2024 + 2024-01-01: 0.0000 + # Outturn (2025-26 over 2024-25) from OBR EFO October 2026 over March 2026, receipts Table 4.12 (March 2026: Table 3.12) "Beer and cider"; per head: clearance growth over one plus ons.population growth + 2025-01-01: -0.030668 + # Forecast (2026-2031) from OBR EFO October 2026, receipts Table 4.12; per head: clearance growth over one plus ons.population growth + 2026-01-01: -0.020398 + 2027-01-01: -0.021805 + 2028-01-01: -0.008938 + 2029-01-01: -0.007436 + 2030-01-01: -0.011936 + 2031-01-01: -0.012643 + # Beyond the OBR forecast: clearances held flat, so per head they fall with ons.population growth + 2032-01-01: -0.002195 + 2033-01-01: -0.002096 + 2034-01-01: -0.001896 + 2035-01-01: -0.001797 + 2036-01-01: -0.001697 + 2037-01-01: -0.001597 + 2038-01-01: -0.001498 + 2040-01-01: -0.001398 + 2042-01-01: -0.001298 + 2044-01-01: -0.001199 + 2045-01-01: -0.001099 + 2047-01-01: -0.000999 + 2048-01-01: -0.000899 + 2049-01-01: -0.000799 + 2050-01-01: -0.0006 + 2051-01-01: -0.0005 + 2052-01-01: -0.0003 + 2053-01-01: -0.0002 + 2054-01-01: 0.0000 + 2055-01-01: 0.0001 + 2056-01-01: 0.0003 + 2057-01-01: 0.0004 + 2058-01-01: 0.0005 + 2059-01-01: 0.0006 + 2060-01-01: 0.0007 + 2061-01-01: 0.000801 + 2064-01-01: 0.000901 + 2071-01-01: 0.001001 + metadata: + unit: /1 + label: Beer and cider clearances growth per person + reference: + - title: OBR EFO October 2026 (detailed forecast tables, receipts, Table 4.12) + href: https://obr.uk/efo/economic-and-fiscal-outlook-october-2026/ + - title: OBR EFO March 2026 (detailed forecast tables, receipts, Table 3.12; 2024-25 outturn) + href: https://obr.uk/efo/economic-and-fiscal-outlook-march-2026/ + wine: + description: Growth per person in the OBR's wine clearances, including other fermented products, which projects each household's wine_litres and other_fermented_litres. + values: + # Base year (2024-25): the alcohol and tobacco duties start in model year 2024 + 2024-01-01: 0.0000 + # Outturn (2025-26 over 2024-25) from OBR EFO October 2026 over March 2026, receipts Table 4.12 (March 2026: Table 3.12) "Wine"; per head: clearance growth over one plus ons.population growth + 2025-01-01: -0.034606 + # Forecast (2026-2031) from OBR EFO October 2026, receipts Table 4.12; per head: clearance growth over one plus ons.population growth + 2026-01-01: -0.002797 + 2027-01-01: -0.018867 + 2028-01-01: -0.008149 + 2029-01-01: -0.008291 + 2030-01-01: -0.012466 + 2031-01-01: -0.011764 + # Beyond the OBR forecast: clearances held flat, so per head they fall with ons.population growth + 2032-01-01: -0.002195 + 2033-01-01: -0.002096 + 2034-01-01: -0.001896 + 2035-01-01: -0.001797 + 2036-01-01: -0.001697 + 2037-01-01: -0.001597 + 2038-01-01: -0.001498 + 2040-01-01: -0.001398 + 2042-01-01: -0.001298 + 2044-01-01: -0.001199 + 2045-01-01: -0.001099 + 2047-01-01: -0.000999 + 2048-01-01: -0.000899 + 2049-01-01: -0.000799 + 2050-01-01: -0.0006 + 2051-01-01: -0.0005 + 2052-01-01: -0.0003 + 2053-01-01: -0.0002 + 2054-01-01: 0.0000 + 2055-01-01: 0.0001 + 2056-01-01: 0.0003 + 2057-01-01: 0.0004 + 2058-01-01: 0.0005 + 2059-01-01: 0.0006 + 2060-01-01: 0.0007 + 2061-01-01: 0.000801 + 2064-01-01: 0.000901 + 2071-01-01: 0.001001 + metadata: + unit: /1 + label: Wine clearances growth per person + reference: + - title: OBR EFO October 2026 (detailed forecast tables, receipts, Table 4.12) + href: https://obr.uk/efo/economic-and-fiscal-outlook-october-2026/ + - title: OBR EFO March 2026 (detailed forecast tables, receipts, Table 3.12; 2024-25 outturn) + href: https://obr.uk/efo/economic-and-fiscal-outlook-march-2026/ + spirits: + description: Growth per person in the OBR's spirits clearances, including spirit-based ready-to-drinks, which projects each household's spirits_litres. + values: + # Base year (2024-25): the alcohol and tobacco duties start in model year 2024 + 2024-01-01: 0.0000 + # Outturn (2025-26 over 2024-25) from OBR EFO October 2026 over March 2026, receipts Table 4.12 (March 2026: Table 3.12) "Spirits"; per head: clearance growth over one plus ons.population growth + 2025-01-01: -0.016504 + # Forecast (2026-2031) from OBR EFO October 2026, receipts Table 4.12; per head: clearance growth over one plus ons.population growth + 2026-01-01: -0.009907 + 2027-01-01: -0.0181 + 2028-01-01: -0.005966 + 2029-01-01: -0.006394 + 2030-01-01: -0.012237 + 2031-01-01: -0.009692 + # Beyond the OBR forecast: clearances held flat, so per head they fall with ons.population growth + 2032-01-01: -0.002195 + 2033-01-01: -0.002096 + 2034-01-01: -0.001896 + 2035-01-01: -0.001797 + 2036-01-01: -0.001697 + 2037-01-01: -0.001597 + 2038-01-01: -0.001498 + 2040-01-01: -0.001398 + 2042-01-01: -0.001298 + 2044-01-01: -0.001199 + 2045-01-01: -0.001099 + 2047-01-01: -0.000999 + 2048-01-01: -0.000899 + 2049-01-01: -0.000799 + 2050-01-01: -0.0006 + 2051-01-01: -0.0005 + 2052-01-01: -0.0003 + 2053-01-01: -0.0002 + 2054-01-01: 0.0000 + 2055-01-01: 0.0001 + 2056-01-01: 0.0003 + 2057-01-01: 0.0004 + 2058-01-01: 0.0005 + 2059-01-01: 0.0006 + 2060-01-01: 0.0007 + 2061-01-01: 0.000801 + 2064-01-01: 0.000901 + 2071-01-01: 0.001001 + metadata: + unit: /1 + label: Spirits clearances growth per person + reference: + - title: OBR EFO October 2026 (detailed forecast tables, receipts, Table 4.12) + href: https://obr.uk/efo/economic-and-fiscal-outlook-october-2026/ + - title: OBR EFO March 2026 (detailed forecast tables, receipts, Table 3.12; 2024-25 outturn) + href: https://obr.uk/efo/economic-and-fiscal-outlook-march-2026/ + house_prices: description: House prices year-on-year growth. values: @@ -1445,7 +1766,8 @@ ons: # published. # The per_capita series under obr and orr are derived from this series; # re-derive them, and the petrol and diesel spending proxies, whenever it - # changes. + # changes. import_obr_forecasts.py rewrites the alcohol and tobacco + # clearance series from it. values: 2021-01-01: 0.0036 2022-01-01: 0.0105 diff --git a/policyengine_uk/tests/fixtures/obr_receipts/README.md b/policyengine_uk/tests/fixtures/obr_receipts/README.md index cf59073008..8322203625 100644 --- a/policyengine_uk/tests/fixtures/obr_receipts/README.md +++ b/policyengine_uk/tests/fixtures/obr_receipts/README.md @@ -3,10 +3,12 @@ These parser fixtures retain the original worksheet XML for Contents, Other taxes (3.2, or 4.2 in October 2026), Property transactions taxes: receipts by sector (3.7, or 4.7), Current receipts (on a cash basis: 3.9 in March 2025, -3.8 in March 2026, 4.8 in October 2026) and, from March 2026, Betting and -gaming duties forecast breakdown (3.10, or 4.10), plus the original shared -strings. The March 2025 tables have no betting and gaming breakdown. Only -the workbook's sheet list and relationships were reduced to those entries. The published unit cells, fiscal-year headers, markers, row labels and +3.8 in March 2026, 4.8 in October 2026), Tobacco clearances (3.11, or 4.11), +Alcohol clearance forecasts (3.12, or 4.12, a sheet that also holds the +alcohol receipts forecasts) and, from March 2026, Betting and gaming duties +forecast breakdown (3.10, or 4.10), plus the original shared strings. The +March 2025 tables have no betting and gaming breakdown. Only the workbook's +sheet list and relationships were reduced to those entries. The published unit cells, fiscal-year headers, markers, row labels and numeric cells are unchanged. These are minimal archives for the importer's XLSX reader, rather than complete Excel documents with styles and charts. The October 2026 workbook titles sheet 4.8 "43.8 Current receipts (on a cash diff --git a/policyengine_uk/tests/fixtures/obr_receipts/obr_efo_march_2025_receipts.xlsx b/policyengine_uk/tests/fixtures/obr_receipts/obr_efo_march_2025_receipts.xlsx index 3e6ca8b1e8782c08f280a9da292737c59654d1aa..2de2958e06b2d0f9edb4829e8748f0f1125ecfce 100644 GIT binary patch delta 5876 zcmZ`-bzD^279NH!NeL+d=>|oKp_Lp$Kw1Vzq!F2+os zf$&oy5Xh~H`63|@PLcZkPS4PaaVf=J)8IQ^`@K;heYJ&xZZg%s|j~ z7H!~5iogQ0Oi{jJ%xl6kJ_>A}T0r-Y`AQ z@t9*2i5r9l*z>#FxR88op57&}dvb|5O_QtEvejP`Y9=#;bb6aNeSKbn(qtwG4@WEMi7cuE+trWNhYh#|FBGU=Ox(nikzR0Vb1bOFWN%!nBgexAtsSCz&v3(0jP zAP^%H2*irYPGAD0WW;^lT%Vg;+%LVs7+f)Yrs8)sE2t{T{RQkT z*b^~Sa@G>{riThqioVHooRJ_iV{r}fd9kY^isvB7)XTjaN2@7F^-B5*t~s!g8aNZn zDv~V9Ku_^U3Sn^DVx=8iTdilCiz6oec%`0nkZ3aAtsc0aKJO}DHWcKowfc@7Lx0P= zK>iYSU|CiHzN$OP&1WDtjXt7tFY$~0xK*0zR+XK1lVTqee~{c=`ysaCS3FKCKudoD z{{19(P(p+HO~`9@fnquKwodr%FMU=Hm#NOJxbB*y3;T&;w<^;3oVIJixWKRXs-aN{ z-tu3Z?41FO3s0q@BWF`Zp?^BsK^J~DV^1POhD~jTetyGvm|c4?!PLO=;%q4G8kJx9_KGQTSTMFpgkL@* zVqg?5(4A{nBa6dG_OQ`pg+5n@W}}-j+|Ioj*kl7tCN^c8yD-^;PC^{>35o$_6-uN4 zE8-xN|N6eJ6)W)+37LC%EtV_J&s~U(HeNiy+Uil0Gx#~qHMNqzd8)zGMyV(}2F39e zu|%K^EN7K-5YE30Q_F_J_CC2--P;Pyi9RW0b`uTw)+8vN#NW_V-0NcKbvo9C-VpMA zIIGJhsl)r;AS3$)iyFa~mEG`_){vzAo#TD~193aN>`Q`EPfdK^o^I0u*6$jkfN;k+ zfp$>UIMH)dUD3ZX{*%yBDDq(glbL_z#abZe@UfRsHH&(W6k|f}2#aMxZXfwg-l*b! zyH&)&7haCA!v4i2NCa_{e)z^--qNF5_Yds%5Gl+bCKjZsFQ~6cRm-XRevv+|?0M<) z>~?Q|z$2Rg-sg%Ef$S^w>$8G)5p$v86>|E%W=ipxvB{3MWE{Frg=Y*ca{bn&!zfq9rxM$e=Yt zig%)7(GlW3Y5Ykl^N1CZjB>uEC$x{qyQ?(7Da#axWsiKi916&=T*hZ}RH|dJG`YGr-MRC%wX{pg3~H0ONulde`gGBZ zQ>RG}SPI@0F+R14P8MM)yJTjm0#M#cI&?95tzhtpTZ<+`N5m<9ewRU)+Cd=xvbrTb zv6HdhH9MPeWSe%xrIb83TrBO_(O|AIV;gIGKAKU!p5Fbea)mZqi;_Ha;bZ!xSb6Vwh;I+bYW+TdtbPvVN^eVDm&{*J;`!IP$;ls_z z&||WI$o>1*Ip4Vsy=iM>M%J!8weFpoX8%^uo340bLabmcyFp2}d|h?dJ7Vc@uoo)V ztl?TQMQC~6@6g&dNU0@;JiHw}eJsCCq-Sq$eD<*&*Q~?@yk`^C9bC$h%4xA}@%Lk- ze7HIL#49g#qlfqUp+<+%RaL6R8_{HYYKH~_r+pCzOJ7~4}!ki8UE_-|rf${N~8F15fb+O{Ym&+z@rZ@0W_@kqR)jY;WF z?lES{8}R#nJhS=QETaNf2fTWzP!ch+pL=|3Y^@20+gLBN)~{=EwkaQelIo1f99MwB_=jam}n)gwdWv3ZK6H0K#rv zf@v{}!lls#@l6Q`UOpALd(-zxVB^;##roO!^%+GY<;FwCqwGL~#!EW~T+?00HmWb6 zn$WkNk~Sq$2UBFTz^XaqWUzv1q`Pd#>-!R?Q->`jDcKG~*X11Vy;ZXziEpIRZ&YxTICOTmRfQ& zsa2A`km=%K5nP?&QgXh3p=U41sMPXI7AT)F;f9E_K%Fqo zRnsSRn9QqV1?fpcTuo=dSj_mSa)!2!eVt6e%;?CD2fe4DgDn`6tYW}tILeTzGxn`q zJc5J5u29XyO~Up#e`<7~`IG5hxaUDG`(bK^dL}%fzITa>X(+a@Rl(;4z24f&{7J#l zr-}8F1{;G6K!K%!ch!B23IuN{(A*bLj?*!N)*Y^|6*_;5Dbe2DDO|*_`WSEoExVrw z0`Bgd?pq)8bgJql>F zUNlrkPbWbmjzne{mU|)4l^*x)ymIR348wy%d#*YF$FdHOShDTh_SW-_m#g`R9!Gbg z_p}Bbx$oC)kBbL8GDGTI48D7AE0)wyzVWqeQiF(e-T06!P&cB`Z_@tAetS9DRa)Iw0Z;yAE&{v2T zB~KQh&S6HcU9+u1Q|R9B(9Rt%ptwnwM_tae>8qdMAHSEjrjXTDEfu@xA>=P^-)dm} zepq*p`>OE#USO911-qJJ`3Pj7BXWo#)<5hxnS9lGc10?aE8wlES78=od18{2m{^1y zx89%GJg(FyV>TdT)+gKWp7F720W^l6S1n1Wdan3qp(oFucN}PkLsAj#5Y74$s~RP0 zd^<;4keJ`{(J*9b`KC6n2uC8Q7Kt zd)aIr#x8#&uDilz(sckS$U}Dw=kvsp&!#i9qeHdTQ8)PGpy~CaX;0Fye+>b&*W zefulV-Z_;UY6kG*Thvn@b(AG5K;|S9jBW8P5;eu=muDmP*$1rst&di9hTjSdcLYtK zR@=QfREe46WtE;VePG=xK2F=B(;v~@8%V%=7{s+2nAWGq=&d?wD%tQf zUoov7SmDYOR49DFAIX26{|!GMf4=6NfN&rtY+X$MUp}ec2mCOrucpXUC77E$$~Lhi*oW8~*tb}IH;5Jmdt-3zNQ@dj(x<(Q(F9K`DkCW4qyKI|Z>DkiuvX9mg!Vgs`gun__o)n(IL zz67VLh2g|<4fn3=ms=z;RPvKD5$cBIqie*KW5I+kV3VM6+HtaRmT|P(hS6t7M&E?s z%d{}kP-{#iD0~k>#CX@0C(*^54wl#&7ZTPRMC3Vg15BVm{(nFMGT%ZT*{g?I*S%1Q zr*r@S`{FAN9^yc7EZyk5&5TQXQN`#~XYNfdS4`j>t$1Kb%3WxCMM4v&gei}Bx~Og0 ziD@>AaSlhtEjn@29@2he7n)+uOpR=6e?^AYnU@ZH_eWLYhnL<+-R7K&v^*xg5`>oQ(lMYX#@DF)z!$2k-}Q5Vs4yo&3gU!0f9))e#ln%4lnN~gmOB%ifKlVqPU+BqA|^-1k}qE zboq`Tj(ro1gE&K;6T<#f6cy_Cxo)ZLY)a*#BdO{#8(E>?gQ`nXVnqLwvVmEfbzJMO z;Fi7o>?`K&XT8`ch}H!`l%dzLcVOg|$ss|+U|FUR>JaMCxRAI|sQ$fpf_xiaQuHDq zTu37dVgkp6yus4Lges}b&%Mmqf68i~Qjnn=qf`kpCldw#F&41WFuBUG$yD`ZpSGt0 zBMi-W`0Fzx8ORnPWk@8J2PRcXWKJPUm_jI0bRwoBd^)2f%`p61fIPPiCC{b#UN_KPAjjdY4nL*_4*BMf#Kanw-lZcYu`Z0u& zPEL)7!yEsQOB~ta=K`7C>5$)Y37eQ{17MwlNCai;ZzRC>>!{QClL}9@&?n*LqDp7(Ia8JkrmcqTCMy~2I$G=!={?&rH%z+O2J(|JswYtk3 zcM-o^`hmrIzKr>c8HB%*>=7W_B{soMI>G)MeK?Gq`}@Uf?-oOTc_DB68j zee+jx1iymvghf_5Aj%gSHaD}QOcHl>-6dL^?JV4H_XIbXyDDH-gG{e-ku-fgVFrLb z@)q51*#jTri6W-+j}4q1HI?Yj#HY=MukmaRp4eIpCKL@}F-0gWqR1F4Q^bJXA#ZR| z8XMZ{Y?w{>wpQnCI=VaIIOJD+%WD^Msqdlr3BpDRX=eJ`lB0E^&~G7Qui3*?Fawjl1cAvZ`-wg`;*qqmxU!_*-NiekNdf0XBc-s;!Vj%?v=|ge|#3RvTDI$ zm0H5*1HJDyxHPS@DMGjLVf2~!Vnyl9A|98k9zP?1I7-UzIs+T(5`Y;A|DhgiO$ed^ z{b;BsO``1oFS~+37k>8sw}lY23!!?2*irZSpvLg^~Ymt zvm`-2YODD=IPTTY4gxt8wwaOrZ#B*T1^jUjA$ZdC(+w;o^M~KxPK1B+1KNN3u}l4N ZKs3}P{!t(WfyjSc?qCpTum9)Ue*jsF;Wz*Q delta 534 zcmeydg!$PoCh-7oW)=|!1_lm>jZWdQ(Ong}&6*4hj2p!jnCrE3v-ZRuZ8R>iO`lM* z!@ueEB|M*}vsRyu;tD_2emb${&mNtnn`?H+99zC6 z)pDxoxy6}syhT$i-6s}LiP(4G)6=Q?>q;`0a%;zi-CT5PTFABqlb-5K)fR8Gm5z>D zcKmqoog1<|GW!31J+M{$!5Ja?!$;6!=Dn+%FEamE<=WAZyMH&(slLpUZ)qylU-sr| zHV|NYu)0NNL7<>xVEEk~xz&ve7IV&GpE+gQ_WWHN^WQAmA@zy<-%qpPte!4epF7@* z13kK*tWoQAJ)y0&XFF^14Xywg^J7hX`}Y=h{r*tGym^o8PX?XNMLj!@%SBCJe|O1C ztyi=3PHI@j==|Aq;LF~e#Ozy(ncmvJNz}04(VX{w?iJ>?q<3r|L~Fau7fu!WHdo|- zaEVc#3*&?Lvn+R29`2a#x_a_7-=&K< x=Yl@d7oHsOFxh)HM#Gi7E{h9u;}@V>>{jeAk)}^@Bt$OLjfpg830|w}9p6c$Zx2tErltE1oL*c8)L&M-eKtLcsG>T~(4)rUCL(xM(3?|{z0`CJw&Xuad z5*@7AXo9d07Bwm@FbWXKw6HwC>ffx_Q*CIGUfSl~*HY0okF5?Lc+I~Fy*Pb(vnTqs zDd_u}?l=9M{LDwF9O^Zj5XesyzJ0nrCObp?5u4xpWHQJ0_UPLUttUai3A%Df=Ed@h z$uHh*3xBom2h++lZTy0I>HrdJp@PHPgumy{8yXINkBjw2Vj=1V)bflSg`asBup4Cl z#4EYpp}}kJYSZeY`8Y_lp3cw=vLQ;!{s(=i2F{GG<;gWST=tCN)|sZ^esK?4qYu&;_PeYHuI z1UGv~`FlwMm~-}ee$T#5759ew-zr}nzVz;Fz3(tTDBc|z8((SGoy!~S6%rbcJ0s)# z?R0nx@OMV`+`Z4uC4Icn{d68*>>QoitX6U{2DePO(!)=(zw`NPW_T{!` z=}UEW$ilU|VEqM?zZW3wbZPt4cg=%*(c*gesLt=<;Y=hdGQ@Kq+(VD0JQXPjCt z6^E?WRc-)j%c)CB_VaFdktmhzVOO8+l_Q2=F2~K!Be5T3a%wJa>W=w?31_$N)v?im zyy(_9Y@;i4fpuKHq+Q*TdAS7piY2~R0!x`gh=P?L%^&)%@|A$^zUubBgow_RUdmQJ zR%WhmQOORC?FXlCM*h(7ZI5o$I+VT^oVSmoBjvjnFZn*SqFUGM-{wnYvGUuw(%{Fq zWw)DwfFR?${5&H3k<_9jVQ#LM8Us2Tz3tOK7raDDn|A_!@8i>SNyGtuzHSVE4@bZ- z!0q1Y{GR9aO+OZ(X!i_M<*VC^DIY198sn|Y%U_JYL$0k#owU39s`lG2x62hS&?9%m zgr|9zbgw7Stvuj@Jv#T==VNqk`NG3UxY_NQ`Sm(&J8AsR<#>J_;JMc^pn7{4-W>CY zFaF#^Vl7K0)rcc}?ek}Jws=1sh%fGK-TlSL@f-A~`S&Ssp*dWV$3?HRW_(a4>%ij9 zz+gP4ysJQmgI&q8G34~(3t+FWMU=J=IcAR^#VaBT;AY zPvjEKrg;pG8N@UrbNcwWZ{#Wj1+#PD(yYnV6Yg z$h8Hh(#9R!aA9Y;;gzgSge}7fQ{=J+JF}VkP;;ar?gSLKt_8w!NpfN$%UbxH)`G^; zeF%V7A%^ZDLd|^`#DG8!Z;{mDDiYGw`Gs4^&d?GUcd&e1B~m#xA0nTLA5!Rx09DKH zPbX!yELso(sdx~13~GbkO(K}Pm96B#TvcVd&J4)d5XuX1klqZXkOG=G(N`9#bi?ZQ z;t1+IEn$x>t`$x;h*!3hs_QduljTK*3!i|}S&CoSQZb3j1k7SWrY(_=DYH#mb2GP9 zoidmWV?`3*XksuRvzhpQICPFiu27YfmBnOvR(*rG)M$jN z!&YNJ12ek{C3;x{R%Nh^ApnZo(57VNubDb|7TS_FmPK&gA@bzlA(U}R%mj3UBCqCG zG5R|7#7$Eo4?V=dg`J`RJ0dbji3tUkQSb*vG^(YzvgNLlWvG%RfK(3u!iymBXoyB3EEL00Ui77_Qs|3JoO|p}C4Z}tR z!nJL&uOp+=}$E+M~ww8GYW1^N*WW8 zWqy!isuyBbbcqWqSs3kiq44T)&G8o}`HTflA^O0LaAcx_&Hzp^Ze4?lHEBuXeJAF$;N(73(RZ{EUe@od!j)~r6J%8doB~N z1`E0tVmbhyii=(ZvB=e}Pp9c@Uu8z%DWc)kj~3Yv2_`f|;4K=VJ{D+4!j${2V0(zr z@SZi5*Lkd97pM9ElIo zObC9GC*@oD@wx6d4BRrLW^2uVt{7E9!GLJFV6dn*Z!0sntI^<9LIt*iKV42Texock zIXQh!9ad8%l+w0GD%id*NafWM4r;P=GUHMi5|q7J$14cSkgBsmBp9&Jr$H5oL1?39 zl2;BAah}{oWxm`{|2pg-p?Kr~BT^oXTH1_h#%IZ`HYqhfKP5F!XH97bM1wy{nlZV; zn=!?>HOEwViOJ1CdwHEJkwhyF5(;W<)CMTD-}x-F`_f&oQjVYAgKuD<+_wkbQv{c> z>cG#gb#qjW*|8+(GMDX zdPE8-mh^0zKp>$6!2*Xy19VY83V%T~Dwr;SZqiW|5VxF}QX#Cet^kig$Q+oIv#uEQ zkfX1#@dVgcS|Qau+@E z(K007@Hrw1?n){gD$Aj`3XzN|T0|3g2}x*A5gZh7m&Ks?$YOAZKrl3CRKb-V3QT1n zSw&tN?uGa$#c}q=N^b>+WH1?SZbqJ6RY3pqiR_-xKk3~J|z{o(1?weMWcsvVE*kB6&08O zoBbPoP87z!Num(;*X*!IYMRHawrp{B13y-!%Jbe^V{@(f31HyH3%-xsF-?Fk3W4$~ zK+14L*Z5imT#rHuU`U~D1+%$9pob+<-ir`^)a)PqqP_rC-Y4-T0}k8L2uUH;nw~8e zi;6(;g5lkrxOb5qbicYS%2uVBQY>wZ0DGPuF_8{9yv%D#L*ywSvF9-Xkgc(hihC0eO)ORw=>~!E(5U6efiH1VCUkj6ym>Xqk8}Pi{!|X z&nsG+_(BF;aO$}K<7BAS!MAT2&Y1b1+h0I70T1bndN%|de*uyhnDuG_muOZKs3#4= zsz)fANaeI#J64lU%I=^er88KCf6fP=4H6tQRDB`mNcLa>1urE01+cqVl-@4t1W#5U zFe?dOV-+%1-IY!_h*yZv%A>RzdZV-`BAEFf5b*sOvTQuF*>p6NUZB1ERa8ITu@0-l zA+7)V169_dg?VjDDQR0v@oR24`iGunUwL6=SDw-+O$^qjr8&eEi`l|+-%kFflcV#j zES669WPpJULO|02!n^7c98xeThC#4nNXoG4E3}Z{_)*lP47G*0D`A;$5&nOoh?eK( zUK>0`-YPQRg$?WBSM?52{n0XpP$V83I^|S>LTMfiAa!PJEGSg zP`-}QgcOq(EuQOzbF5N8bre7jFC=u17|t{|TvG+-6^#z2O9<=%US6sUQE1SHHdFY^$XwhGM54~V&9aih_cKiPA+6T zsKXM{zyfmL)=U-!=s(y@4&`?DO@M)wk4(Z44@;U63+uyY`di}?ma)(zP6<|3`DZl@ zxCRz36FWjaQ0@kM;vi38s7Q$};+4 zrwk1&B7=i#h9Sf-wO6zwMuy_*h|rM;p9X90xuXXIxR9gQAOsDvqk+G(YfmrA1yK?B zk5KlR#Mwo-+|VMz<%hw9a(#$nki-fj@W;YJSsf)wumy*gK?Qp`F|#siZcKK4M0`v{ zEN17oE*gLHP?*q!NpO|l13nai>Dm(C(=L#UzP_~HXHsN?HE>fC)TrLU$z52#kScQ+VS1K|Y6rA%pIkXElck_37Kt9RCSS%tGC0zEM#+dB zT4i9tr|1-OqpMpf1FDph%u)a}8Gcw)iTvZD6|@-CEG*4bQlqDJdB@s>McZ+@xRggL zW!x-krlQo6_D4&_%(N|<=#y!1ZQ~6XE;r6jLvuKx1iv*l!s*o?DAE{hq7_qw^Q1Mo z1yrA`p-&MwuGr|TmZ~-%E6z3^Eiop^*sfHgcR{Qz1bB+n{EcLQR5%ZfU<& zcc|!hA*NVQ&B29=^>M`yJY_5Foo@p}kkkMc`BG<-~+8GjlvfwXOfSk>zP< zA!C9g!D>tH=}~eEqwMm6WPNLr2~}7O`v)l)b#n-57R*DGme8JDmYlBKx3||~pQ4wn zKjwCE@mD9R4|=`87bnfBgjd?T+15qe*PruQT#Apf-7AjvirdnP%pWQZ5QNxom8Du4 zuAQS_P7F7XB;(dM2}{42*pk_fGoex`hL=K?d^ER&u8s!1b{z_svodk)P;s;$P@{TDuQzz0XxRp;@aOke_>R%*P#G~bTaL`1 zeIy|02L>Qz2%QqhEIYWMQnpng$WYQtTiI&FYeI(=ZzoE(xrtID81YnyoJz@UJ2E3u zws%@oBf__axKcj+R1hC`X@#O6h~q3kX`c;ZGe}$PZ$Q(-SPkSQE^0^FG{QPH#uK^B zPNziJkZ=Mfe)psGTE6tTIRJKwKP&g5jqeIzcO?Dc2AyDyGt|F6Ko(IMnd=tLT=@HQ zGKt&s6sE@^*god0mKD0M=vs3~Vw@MsN^IdVht?`Qo-Zqt+!nf>2qgIgSXuh59h*5` z%)VH>G_BU<97V!Wob9S4fWD3NQTx!7&JQ*UzFY%)$Vc`o_Ey{>JW8#IE2NCUa+FoG zyZ()QlwKG`C%bmVpCtYI=aVzW^6ry&sM$e+XHU`cW;D<0*R)O2oYr}R3(#4ei=h%1n1v*P~2zE7{VhrL{Q0DAg$e)bZLTy z1jK4!%|LfA(`oLFQIp3d$wB?bcms3N3`HzsMKae^1}5bYY6`zo9S-Z5wJFtm z+39HySbtkagQ_;+I_X@vUXLQdPeh4T`ucq?y&T6py5y|#t7D&?cS1a{8Gno*P*C`Z z?^AP~{rYc3>}*#-=MT*RJdx8(1AO;oqKDIdg>&s=EUouHTx=HLnyYt9k9pQxZH1B>vrUk{Q=!*fz@KR&8&Y$+`K`Sfx14B zEIS(=-fpm=qMZ1iU6zIm8S6*931*JT9`U;8d=2@+GK$3Z$s!k$W5p+r=IT-wuC!5< z$gpV;E~4#H`@n5d`yo9IIc#l2Vu%(+p~*swLror!RJ%2!_$>_Z=}*dG)YI~z@CMHM zcJ+$#p^;TD>oKY->w3)B8E1P8Z`FoI@++}v|569e>7kuVSHEzaHFPjmy<%2s%Mg-&yRm#eq1C)`z_u>1X+%@^yoLgEeB|mEK($! z@@`>&G-5#|9fOn`K*gp*ZI?^r^k;xCLBXe$A?#_kE=EC4LLm2Dh-(dQ+1^K(kN7OJ zw%UNz>Py^hboiSqYEn3W$G2_f2Go6NlfXFF_)KQ{KCH(N5LR}&3^dZV@aYX?eKf#* z6vOpymwozj+otkL*k$cZ+MB$iP4|@9W^e(Wj_Z13VPLBOAXD-!k-3YUdWm3Q`jJ>5 zMOj<;D3P^4UW&H}C-VLb4;|VT`Wes5BuOE8>otSkk=4gI_-PU{yrW;1anP6Pb_)rf z8f}TIf!AOXIE@@oyCoe-`|fAO$ZsDclxFF;IXo zZ!gwC_729oDWuK?YZ1MTmD`Wu1TLq~+HB@6?Bc^fH=%l&jERwMxi2<47`HHkyHiSVHKusxu4=197!Ltrev>GZVmjfDSby;>Jf{~pv_({@U0t?RU+Op{@>c@bpd8&DtWpxT{~ zV>xEY$~%!Y!NG=H!> z&_5J8@QH;9O}Y01Y>5oGz7;TAci%MCQ$00$*NY^n2E<}+%PABZvW zgFyG+K!RC<(k>bFL0}9DVZag*BZ5}%Z$t?tSH9`hA#e&C?7wueVMdu?BZx10b1e;- zi8#SkA+FPFYc-`-SV9-84hziZXna0h7V1M|%n0AKZZ+Lz4jpa3!&%Ye07ksJg?uLr z#9ZFz(EnbaVphLZ5_Df}77FW$T@j-z$;BVahywyXS(f|lK3Oo>uiwA7R9N4hM+$iE zBb3)s?pE<2#>YhHMf5=I&uX0!ed44IaMv81xDF|f)Gpk!^hIRfmen`rQy)oSpz5~b zjNjPIuo^jhzPK4Ywq7C=t(6U-Nds50FY@8rh<+jj@8==>GNHI&iK6Mgc<96{yq6 z&h`Uxu@Ua?YkCX_>vX}0!f1tVfG-vgFxox~9L0X=dI_S1b-u3`#B{G0m~Hf2-l&HN zd0!PtcIYaHBAQ)*4@-&tAiTlZGJ9gLy?zo=>K9OMiJqJLoyml7ZsuY5vle-L;NnOd ze`C4n<&Cm)m2>n3utDG~XQ>c$dZQJn>72TBd;b-s`p)w{YYOKe3(9j6(YJ?R-q6o$ z-qZK|`zH$jDdJx(Y#wC+V6f)4>U}CmV~opyAHHiQlrP->#7WcEm7PUNwLRYW9G{Ok z+Yox`iASo{8$*w$e7%PqrNLE1Y8QIBV*Y8Z9WGD(6I|W}U;#68w|~*}Q3)v{)Zh>t96q7Sb+DYS*t^gzVSLd5jB{ z(~Ln4o*;2rKveef*Fw&By`IlK*M%GNwZ5Q&6NKQIb!mjKbFn%Roo3t3C%X~m6t0F{ z`xs{a7QQK_7OS3{E#Dim6$e^1lrJ=tCy2e8s*Bixl)LhRgx~*&?qNb;2I&Zs2 zT*B^;l*F}f#aP=6V%QGN8Vk0tH``#dA1ur?6_TSQ0dPE9#G`6ihnMe-hO-N=Sh~Az z6KwnKa{G#6fAjG=?32daaojYrF|YWI-4UvFs|^15FLh?6)v0eh*T1j(F0xc*M5DcB z*kS2S$*mZ;4hJ)i1`FoQ%H~3p#y^wv6AF=&B26P4TKLlmQEp*1GI0`QM=WyjtqZFK z2U!cG1L&U*jYzv$&0aj~>z%XK*M|2?}IO2it43{l`Vq*_D_b_q%^aN7Vzt}*sy6uEg zf%;%BV%s4R{KG}rN}fn-r=C^W{k=A2$u=Bfa)h-{dxTI9aOrQcadJoe`lM8GMGjhz#Oie_L@G?&z zrScBU(XxW~AxEIHU}V2@6};-B;hjCsqRZ*BsIR{8?@v(?{H_B*V%Y35Vo9(Fj(AUs zMjk^*Td;k{skk9zYvQ084|GMbWq_Gz@m|kfKF*kuKX=Q4j~z}hUf;wb7Gb#P@80Dx zwn2RJCHg^<&6O%dri4h)5hqh-_|&@cW4gUz^KbVNxAG!>xLIl^bX4Q}cH0kahPANW z&T=v!`K9QH)?8jHLh8XS^<Av{ICOZJ)1fjp))!S5AZ4Z>mHl=# zd$^YU!;(4W?MhSB!U)`=!#D>#D=6F=O`_@Yd!HH}9{DSl{xhfjU4nK$BurB&jL4&> zFQQmBp^fgjoHd-=Ym~ae>mP;zX!Ld11c&`Y!iP~i8uK~O<-*rl=Paw@I#VOEh!mYeoL;-5_=ob3yE(6mUU6({b4F|K3AG7gDeUV~(%^NZ2VhtSr?2ZjnxO zfe2IngGFU+CCE3Geub^;OP1%*q$N;u=kIn=6OZ@Gb<@jw??-vsK|yK+BYJ`_eo3Qd zX=eM`h_+W2t>j<;R%Pds%{77dXUsY!8ajrltG5S&{XjUnL;cbFWeLn_IhJ#e68AXl z$j-KHT(>c#@*HCxv}JDSYZb;5y0Suc4)k{=zS<-y03}zyka_lpQb*qG~$K~Y9J0tJqH(C;xX{+5LkqFvrKrY8xA{OjNRs2ZI#9u9> zy-E(@$?T51XsI#Qjw7>982RF7P4snguhUXh8F*fDPjKQLpY*@zLoF%?)&yahxqCNU zt-hqqMv(wHlyE32ztMU~t{lrRF6{|7^?vsx%+It1H!CQboV?mzX({Zx(#DD<2=LI= zY#+bx3FgmvM)R%h+UvS%EnI#VSu+DdUR=8JxaRgG(9615reDBw+?f{cp54IN=&tOg zPdQqaf5dy)OE)12!^giY!t%C-WUbxO4!ax*#peMIe@w96^NmiiZavka2o@K!erVCr zSvOX!d=k4(PL>f+TrM9oXQiSQ&!;_f$5r)CDqz(ZH2El+u6?+Q+r+mto(!;}(y5qF4o<^MLtZ3kbFIeZGiF&7yy`90)u z;%F!|@Ol*f3t=efFRNG}QC=NJL_m=~=u#IMYH(5qkIZ3sr=f3Loo~M~UZ{Cr{4gQk z&NI>o{?VnnwM&ZMF9tPuM)4kufV0;E4!r0n^478amtmjtNkk)maz&%)ISL5|CO0+RP$o_7S-x_ z)`;59yNj6-?IvXY+cg$fz#7kfj+_KkW~&QlXsefY#zU`HR@MRps|xJx;#S_GrM4B>ej z#JEc@EJnlx78%Ne!C#o2geLA3VN`>{BOdg@|NXrb;Uf8BFe6?GuR;5_9hbW%N;LT- z8CRQPM*D%giI|q{V@rKvEs8_*P}TfrRyx;?z~G^6BrEB6X+`EOB}zg@tA!6ClAbUJ z65wQsp)VpJ*&ne*EM&^IrkDr;i+m8< zSGf+nh);(GdI3>9q5PCx7q^8RzIe{4axaWGF!j66`?m&ur*MZcHGDVE-%!P*cSdWc z{CCi5Hg*=+YK|PZJ!z&mTV7}xTP{W(-TV2JL4MAFL#hYz+y1~RjxdH^_Rffy{hWG@ z?b5Lh&o;uFk>bOxE=bSA2WMdg8m$e1 z#hLOn$qgd7zD#8ZX_`BB>0vpBUE&COQ%~!oN9^LR`=w=t(clV)QZ68F^?3P zCJj=K)w`1cfAmIHF%01&UXPB^U){<9(xOj4t1%Lm(dY)D+grqHq-)^wZ-NpdP6%a) z>P-G(5j^-Q+JmP0hxh_Nb!4fLE7G5N=9y?c((eVm1fag%4^dE>hxDil-F_apP4)c> z1I`a*SBTnnpu&OkgapYE>15zEA~-jBi8Uxih*xydH9#KtW|Q$d)M@5+!Vc(iD>MIa z)-V^^9>M71`^eyoRX)hF4+zG+@4&xyv+cl#k{#fH{F|<52XqSO&)nPRe6~t9{*1Sz zwTW55i9W!xki}k5nez~Qlk%$bjnTAPU6ufo`ZC8HpbelIeU^Zy> zosu2NU~cQzc9vDXfWe#q2>3jt6YW}|+V7QuH&YXt9DTrGDLvzraj@d|9kb{=Dx4gN z6YG7&z~oRJU(JxT-|0PQf^~L{o?`B!ullLTgx6cg z?bcQegcsGLT)UjLa+R_n98Qc7?_`6f!d&d3kVtq?z>< zq8(Yj=yT`-n|;09^8l`(y>huAW37^HKyAxY_u_fc{#)XG=OJi6_eriFAk}f|0;FB zcV9%2`fQ^g@}zvZ>U-Bp>~Zo4`XJ`EawiCrnI|~ zNzc|9DMo8`rFuWBzerOcv(R5bMPhGd=#|92wZA-KEBv-iPgqBXr zxpAuv;#d0bRA5DtjAx3Lf5miKeoPs0ErBKd!X-~O>-Ab-HgK$xzh(CqrQVk{f!hM; z!HrqhXW1jgEcPzo)SjCKjIC}Q-mSAZ(IxKt>Dm>f?{{TBmM88@<52j}qHNl{VO%7i zVMjWfeI&!^9rF_*3oP{N74hep0ORDKv!%}$wmx;}6Ruvh_Gyd%&R#&Bl+VrP&xh(; zc@dOCzhZXb&08f+o0V6oQDvI+gJW+D=IVa z37HC2n_2(I9~m)$H`#_6n&>#AiRq!YcDj$@{_G5bv@y!L^`?{vOU4KLJAVQMefWd=$J*W_Pol zdwOypsIi7MXu0wWH|9coOp7 z_BL*MzP<_n_tTYBT{531!OvR4!-BZz$$@!Aq);{t*sW?Av>8?8BDJpLvb8pR1U@2F zP@=4@-*W`Zsqj&e1@1KzD=GGC9m=pINv z%xq+&9$rnGez*Rmo_rejVgK{Rh{H8AeRa#Ib2+cboNV$%bU>Eyy3*T|Jg@qktmep* z&{VX*m)sueh1<_vy{~)wt#tX-;>q&>c0;0e`<%F8KHiN_D?imp2U4_p^1!&c>ib=y zSEl^^g-V5;fSWRNZd!z>Cw8S9TYFPgBeP;}e+93xs^#u?ps!AGDYAG=Tfiq7WliU^ zj;_I<>@4}lO3j9@0>M%BwrPvbakEFiI`k!!eN=~yIU=9=t*)G|yTbsO9@}0(F_fR5 zWX@1mW5}E&Xm84z$6s*i#+ygSZMSz6#VRq#xsrZs1?MTW?a?GACvH~QMge75p>upl zzOy&?)k~2py|5>Lj9Pz^;K#&@pz8LiY3=)zd8p6HFGAg2wkR(eN(F=<(p}2=tr$_Z z&{n?JPNt@!ocS&En#iL+=T;j)_wKn>zLXs%=T6CQ<=(5_ z0bC(|>*d|0Zer89WQdBf0BVIg~SA6u@0$SzE278xKKiMnG z=SzCl{3}Lq_&%%)3gb)yq+3xw58Xa~UEch)5HFt6Pm_PLS1@`^6HF<0oWd3?y95%g z@LBe3NWU*Ps1E4eGc8b!i!kErb*UVR5#z6Rs?3Xb%3ks69Y>{k&#jKgtSPARWf-?5 z?(ClSB={_-@#0d(r>5U!a!8#1kNl6Q3_}w36AG(?NPZwPwkT;3X=P+vYg8!ScN+2e zO!}SWS?iSxTceCVsqEH6Nd;viW|YaEFvCxf9Fc#r>;^%(M1tWrt$HS(1KJBudfd;PG~x zB0e96%m*NBi-nQ8xvBe;{{dv7n#>??a*j?d#+}EkC0wvxN~B9H^(lo=M^Tjh1eJg# zMH`3w)sz=q0$PCpk}6XpTz$<4T@NMQf`WS#RVCbHEo^X`!78m~VIRDsq6}$1J2|mI zUB{O~DO%>^XHOxt5Kv2qI~G4uTts?4j>d_nLqaa~VbV|yrQOvk9?Y2HD2R_Be6I+|@a;T4 z3;`f=kM|!{KXwlGFoLevg@jm=kI%oF?**p@r0}JQSxat-+2bP_+U$zy<45}<@IOcg z7zTB6QO$QCSFVQ(%I8CS6t*IVv$VJt)*?%iP$gj(sFtKl9BO(`X2-32@yK+wr$NW% zL*JX&qvQED#KEumHrRX%BW{xxZO6}W%>iyW-lh>uPqq7hDuJYt{2E%}kp-8BLmdTm z!i4b#AAKcb{bc=}dM!V@a;16RGrLOgeLglTgkSS+sD+P#i|s5lVhwk#Czn3%9S}-= zBl*mDAN5L_G*#I3A0?b7KT+3d4d0XxyB~?-AO-J_-fdB@@Tjql<-x$=3lRgb{#0Ig zmrXgQ1=fa~s9V*aXzmdez1|<%WE8fwLyeshek*l>il-2UQ@0g=Npj&CSP7&Il~bo>9si6=HVs6Lvg9(PwwKz05w<*v%B{d6)1PGhb?N@v ztnXFgP3H?{kgX0=c0zqCt4~V5-Zd@gNE6@DRy@z*H=kJEV3Lc4!hMDY$iB=?+b;=e zl2x?s$2CdaU+!NbD#($ooBC4fsnVbNuK2?pgsY*6!@q;E(w?vi^$xX~BP!e%aqH3( z%=hpo;4Pt=nPdA#9At2_8m5H^_a_w13sQ)f2Q>cYCVaQ*M;(Ilh@HGf<@Aack_w6zBfM| z77nu0=OkWmvrq5>t}wkLD2TCN6=(SHv@B^ncoVz_?zO+)JkX9N0MlP%DHvryRv+ce64b(K18Re=^>=kC|!dDikV`Ust#6762GymK+IzOA8hmCFSg0mS#esE!UcU+w8i02V0@5d4!bPqx|ZUu+njUdlmqlRf&SM^v@U^9{YQW7$HLm|aiL2Og`%H0 zR@!jifS-c`3dRX{ys=XX1AjIpLkV%V+|7u+3n_$0ho%jFiAe#83OQpAU#`d8n{uDt ziWbCFI&%_QcG@Y2kIoL!)|sUg$o)U#DO}>d9&aiz7Hztr4-x<<2{WPa_I)_R0|lIqpt9RUkdz^aj3pJi2PeFlp@v z@Ub;QSupNnF}pHeQ}DN>IC`bsQMpORSKG=rBj(GYZb`3ved%gBkHjhNjqqt(#BD&X zZO?T7BnXoQm*%H4Jixp;lG zX`U42k^I^LIJYG+2b`_j(@YdW^E;t-z^i2ESFGndbE)UN1LkPW)aJ6GnU;51THlvhouI`f5di8z zG#hS^k1T|h{BC@x9w7|wtr&fyWWvuC$r`PNPBlSY=N(iHms%{sI%-*6Gc;{ja}aw~ z%PpdBH zCMqd?e=Z-(JKhL|aI#35s)AZ4<@5mF1uS@x3HCBE6Y8Atns>Y)fvu1a-lA%BAnfnm z(B;waNHa2Wq7^goW{YS!7&taCqT)Rbjd(o3C3159hc?pp_7Q*QtH+Y=*Ufh#3M5jjiOTEpL^Q~ zmREv?9bt#P%^>5+XoWr?(+7&!QeA3;LXc?};*O6J*{I105}7PhE~my9PLczi)9dnF zJ(>p!j3#mdY&MEqxL8bN!%H!Jt+X|#66dzVq4D{GENok+uqD6vskv;a!6Bug>Ey#K zz#z6;2Jp@c6=aFA<3f|>@b&%VKTUygaitpcL_zj;)7$SiU7*C^tN~1*K74AwQbeZZ z%Bd6~A60QpB&w<1)Zwdwe!N^(%QOWIr<=>0A2TCpJ4`UPE#R)b;I93ScARTp=u6;- ze>Ts#*!9UNgEe@`YyOef!oT=i&P$pm``B!28Q-I+5-n#Jd2;gBgH6|f2GgL|lZNrk zFL`B_haT2lg^MK+T@txsTHY*c!2)rNN>cG&=?YH zoRu$ysM8OVSRk+zJ;{OMDSQ0?03j&dl=xLQ(C=W;dOWmvhK#! z3Es3BR*`ZU!*n1tfMlwI-+bH|HU6Z6bm#i%SFYIT9{Cg25^{96^#B|M3XO7sh$8zO z+gkrJeS`u{o8FU^epD-(nWoQ8Kj(^de)9DU&fvzlWA_xP)cPNI6?sS~9O(c5FPqTd z|7?Qrk~rV}-y8!31Q9sb@IS90NLG5#NBVc5kR%2uP6p7n9xiAgi52RK;~$Bb%nEfO z@sCsnley~u$Od&dPn_g|6ozvLb=|H{2%{})JA%JHege5Gc~(QmojQ=Xu_Le&>A2 z*^`~*zW1KJc6P3r$6?^nFS%SXK|vX$5i$Zy+o*jR zb)^*ed5Dm}Bb|J@%*1f^FgPs8!aKE37wz;rdIFCvYR`J<8SB=s;FsjTyA`{&7XW{Y zx9w8Opw5R^j?Vh#D#AW1@xSNRII_3Cc6NK_`_pGvwc@@a_nT>y(`T{4q=|Dw{@}G? z_kFAMs(lP>8aSN=)YcWI)T>VaiXp? zo|p53^x#u*p+IpYg4>Ou!X~%)m0O&p2WQkX)w6-SvCc>8&7D70rww*?4wPeUQ8V(Z z@HMyx2ygR-e=pRxu9g5Ngf;2}I}S_Z$RffXUb_jo(No1IxW#d4r$Xsgbj|?^DNr(PDT@q-6$CUe7{iHMr8C8;vr?L&3gvedj%%>)cEQNJntE% z$2XtD*7xmwtNPJDaYO(YLSq>IsLp zLk^wJ#zlGwa&ODae{Pk4B@R)y3H?9qnfgf|?`iVCHwi_*`DpbWyqt`^b-m3zbw>$r z{e5>!lXZRFZ}Kt%%n?pi*pym-`N2MUr~Rr``}h6 zzS!99mw3}!m9dV&nMiHFI2+!2d&-1w@sxSre%&xtV_{cv6TEv| zowdo9nyGo>&LW;fRF}VuEG?~nhnuSK^r^ompSV!edEC5J0KCv|E9e73Mkx9-(NpF@ zX#v?&&K+oD0Q@bxwAoVDeA@-8VIDib12lEk_ko)^miLZDb(YtL#X08Z$|iN@=aTd} zL#FIHm_V_{7M(ad^wAM=404VOr(StDGiDe<)c65>UUn2YC;sP+!lsw~p9>?0KO5;x zmo&}0)}2~Tl2RsV3=W`F&uU@G$Vk^^;Jwwmhkg#!0qO$^15PYZ3FDva8>zuoS$Np} z)!8Y8tWIJywUvTKi$PoVinpIj>1RMm*?5#6jp7YjWU(5cd%7x2ZZ++r9hgs9uhtB> zW~{76c~X+_r^xt?tlift(@Rq8x}rQPY^$9Dqc9qz)PF%KI|E_AQ{jD-NEoSw#oO|J z9(%0317zvy(1__p$3Gf8eE^r*RD`CmW1`f4UUKkV8I6ZZ`4$D0g8v~%#Hi%zdTpa* zAfIo^xg|3y4k~2@0r|>63H8cAa~xDEOHG+li%z4mJ+CzaN|N@LR>r0jz*#|O69{76 z@RrsmVKWx9Y;k{zT^Kr~l|xNQA!bt+!%(fS0wmyJkg_QWI=3M0H7tCkof%6=%uSMV zbP%d{0HL5kQ?T&E`F+2_0yok`rpTkF^HHFVozP@`9vY?{L&i)YViXav{NjF<`#Bw~ zXZGrNaDa> z4WMHx3+evBw@6CbV5t@gT9gb0#bbz3ut_}nIlH8q2iI0{D*1VdQ2TlH5be!lOGiq} z6CgxnQ8-YKMmGT~nm$)FNmevr+o2>@z@f<9-3tR@dBZ-p?~ZnCitALTm0KxcFMT#p zYN!6e32At4(s(xuU!~6ZWvD4@ZghG8=#9}`=#Axmq+e50NZBTB-=iQauF3ywCcPbP zpxyJopWR7ma+2_uP}-?I)(slJL&TIMW@Hq_>uhdnN@e8HPm?l1EmWG_-@Zq8efwVS zqnw8XS-K528yLvSF>Pz<4R2}bIsI7XvxERZJ7!U80|6k*7nuZ*Q0C52f|QaQjG~|A zl>5A`$5~?&r1jFHOeiSf{cPXi+|r@Zo)SGZ=BR;pQCbB&FRiLFm95*WqglnPqw!of z(ERR>pEorYZy8lHF2Y|;$3*vSDgs6*8$~Gx@yV+NtkEfAAZ<*cqbHIIW3c)s1gO(p zTN7{52wT>reFg>iKsJJr8XrFoZS*~s49pT4uHAd~!7G$O@S>@r&TCy;0u+xfMqcxa z$)sroJIC3(Rk=;*D807M&rnw0L~j9~y~cgrfL?lU5iunxJMbi1%&X-arpX;LnkAya zO4A80(FjIExCTTPx~3Ng>)de)Z%3nk-DJsV_=O{jbK( zGA6&BBSVw%NtNtqAHAlXNAi;@YYLMpt!Cwx@#o~0HSK6mQ0!8>#@0I&dP$6$+_N>{1NqFCZ?5JQOFF?i%*^?8ezX1Xo~J?g9#u41qG=t4q7^pd3Djej8!?T@n%Y5sImx% z1JQmOGIAp@AX@1c6o5?+Z%%`_gGVR&gqRs(yMv$YD$&02=mZIs;~6xIL~j^UzY(chPbp;;;Y)9plB@)O3w$UM< z2p28UKzE}Eh<)HiEs+@W>jg1=`2lX!q?e@r0O>RV8e!D-S`hw&p6{Qza4Jm3lWK)= z>Z;@X{~%8v7%8OV%u0@X^5U(+291<&a5ct#WZ=h7iBnzXYOHcsnYOXQ=ZzAk^mA_6 z(iX@TSQxu0-)QVXN~w$(V!?v*n}(7sh~w0qlZhoaZv|*p+tEO@-j}^oiHsA*wP5}d z2mTje6}W!^wxjtUz;-nM0gOBNFTi<7{{p<2RlRJsXP)0${>?kdlvZ?B&an;CPlA~Y zQG{xA7#dRnUiJ){#oR~K{MvDnRvrw^5^TAJc|`FZ~@HRT_d#QMG}!*T_o%ZnR8Jkxvj2qxeKhX3)I8x~wY?Ba)8?@y)sJ z*M1g5qgGB4FxuTT?lGM-?iG*R{&OI_v*j4SfbkE(N42tEC4o7Gl zxj8)W1inFd3(`gjIgy_Xh^rrA7_ceXgjY-lykY{NmkKvfb7lNk*_KTQ=mtvR;}=nL zFE}~OyR#8G2=38gb2-LwfSA_cn z=B<0m2viDypCCzr*IsAD z24nw}u_=}bl~2UY3Wo3u{Ew=3S2ssF{q^9SZi-YtM~+`T!@VW9YI_I}A5w$Hyj3z| zW0fTqMcU&3?-uOs^U+<-pS<{mCp@qy^$UtcP6lHhM)m;8j9TOiu&xN_TJsQmrTG&= z<>)!QlqM8-M*Sv$Q)jl+op(A5 zA8I?jumXHmwR;Ke$uMJc31*^HW*E4ik~iFF9TWWF>sXqn_q9VGU9qt{2B61QQo2AU zi_Hx;OkqV|$47ExL=GQ(HJRfu!cx#iERmnEt;_)ox4KuQDcr?CYzf=u=$^m>Vj~Kj zE#%A%W(_TP+RbJZV=&ZpO*OA5Dr#xQlZ*tFxX{wMB3UJVtD_7oP)8N<+j@372xUU_zWoNk&7 z)il8>1gF+xs(EU-RB3VF2ArlHteHH~y&qC;q(!~uNN%Rpw7p-_OM}WZJ%cZu{ zuG(z(=RMEhE#K~a+unETGUTcCYHG@moGlp(5|vDv+K`2Zv80#~t+f=K{Z4OTQC#Em zt8WFG(?uq5amyiTTfrn7(+m*X$4CpC2tEkEzrV_#syy>OftK?{f39Y{CGGV6*5Ocl zW4x;&t(|-AtZD_EkUR(^>GjvNcJwU%jfQ(DCOT^hwJ|BQx=23bX zwxTw*gUq6iB1stUaap1SF|!(t%}v>~95Ig6=}K#n{~0?MPD~dU2Y!s>4(z^$z8out zx9!M=25@5~x=i6OhgsDHyuGw;y+(oEX{C4ve;Hhj{Sp9lzZ5G`hK7zPGpP|srxk5P z8^+Cxf`ue)<3Nk{8|$=06Zh4_p)!d;Ng6*vgT(Qh?4kw_n^tuEk{&TmIKisKsu0MSp6>t zES3y`TgvhJBP#bD$1HWRd1q6<)H4s);@2o1zR%%vMZDIg_0k_{4Y0XOr8OuPGcaa_ z^2#y8YZ<1Bsp@j0JN#n_@)00CB)7~PE-eSBlsZe=i$O9XqAI#j#DGT4>d6dcDG~r} z7Tgu}8$?Xbgow0Rk)Fh~7wb&_UZy^goW%Y)J#|cheot$RDa8B-2w`0Z$RB0}ApV;I z*0efr3$ZD4HA~0Jn+vPyIS8%o$a%5UpwpU%D~>9mHD(#N2rV{2G&`!;^b8bgt%G^z z%BPwRB0n0e=)d=OKON}u0l*^GDIFxff{~poBG^vkaL?@tg7?*8^k%Nt-2Cj~!w-eW zGsdSyDc1gWEqo_L`JI44H<^zG&*4Fu`5ib-_v?p&Wt*DYpA#dq2)lu04Cxy4R+6q2 z;0S6I9UM37vOwq-LVCZ~k+@pMU`kIF24V!5G zZ7noMZ|junj#c0CBzQZ2-K>+vHRq~9NFu}D4P&0psDPYgtL@uLz<6hy%p|OgKuyQ_w|bZtdJFq?N(ql|5=!j*_&MyQXpuo9&|YLVr> z)_|k?2XlHH6dh4pEA*ES9ipE|!x<}oR=}xGgXG??McrfST>;`@fqBE!^#YN-UURqM z{k35~We3NiM^MFXvzqp@ZuYXcrBu~=Ol^BiLwZcJnWnS!XY=)Ei}h#e_&(|i4BSpm z%Q*U8x$~Ak=$SI3^|lQWMq|)!AW3s`5P{V}L zqK@5QTipf)smX_lj_vrSwS zkeASh#+k!*k5$gZgndFAhLxtnFPk@-z}eKdE}~qYoG#$%#NpiXl#ym5eue>d&$qbX zFPC|G!|te2tf#Z3pwEK{toUEOM;9_`Z_0n+V~Hqfok?|xO*dWugYrc$>xv^qRLj2jwYVl9PSg}#BoRJ z{qJ{^eA-_wa+Zebqs#`@*x3O-B2NjnXJ*W3s%xl`K*=NtB9euxDByZPAG=QKW7oX# zs}W)P-ejR1wO_8^%5TGRSaGZ)=7t!s*t4$wsj(epz!eub41_~#wLBGqlO0N1A)8Oj zD2M4dI;HYN4a1?1?U08czDn%Ocu(ZajUrW{Lar3bzg&UmDBXO}y#YwY$+!+jsDYS` zBS9FqPQ7y$H95nSc>R@6|2Xbj_bZG#%FfxYpVX1*bC1<|Nv|0OoQTez6_}(;wx~e5j1OROHhKwf%5XZQ8zZD_^#@F)|mqZ@b+8vT=jaJ z^rvbS405XU4s-oUE#PndPEn3U>&~p%Qr^C?_CTHR9+mrXmD8nK*X%%HH^%Vsi>6uv z4TyAOXJEo%sp>s9NVn3IKrP`Z=nd{@DR=!M#8E9Y(g{Q%B*`GNgi*TbA7t<5ZZA*S!qLA4Wc5MvkOp$xVT&`-1WOO-K110z$i}gOC1U0=x!?hNW3nMFqGu9tD z?;r(HgzqHUHuvf*@q5hxAd$hmxQ?*1hcEzmm>&a8 zm_QF-^jJ5u-ouDn6O^vPdZWnQ*hUw2E@tqTkF*jWU{HU0=o09Bw|DURarubn7S3B| zm1Jg4*^E>AHsBE5wpE32{6Je&4FZ3TfAAC7cbKV+N>;EeH$1Zhizkc3-E!K?a=?LW z$pM=2aF}9`tSA*T$_~z*TMVxl+dR$ypCndQ`;@fzjj~E2l&5G4QTmUY+72Nw3!aOAQ+bi;7%0R<0Bh6^DOrMrQ}YV+(Vyw zx!X$6a%aP!Wk>Wb`$@%&9gS_{Its|woy18uY?i_CR*bN%dL!QEmWYV7>{6|X(S=P|bf#V=Ik1)nj4qc5un zzk&-+qL?1*{-UEe3VNS$_SRCqbjd6gc521~ktykb<2Z2qRv{ULCP0`+i;fT*ccA-kr+ zsX|aE<-(oFeP!i4_9%7w;l5{32kN%MGTz6S2ipL07_2r|g}y$hVmo;JHd-oa<3Xe^ z?I7|~!a);!Vn0Pl9K@aHxqoFLV;b>}0OVuMC54wpX1k^W1^Niv#|Dg3k^lpCQw(~P z1sXglwiu=is54|&*1)`oELJ+H7y6cahZR!F`c7CRbL9_EgPXtck(_PqxAnTeIf-6D zl^x2|2}qpiQG_bmH%3XSiXNwtWapqrtD>3_IU4g1rJ~hFv+LrvZkb~UIC=@|)o+Ha z1}>#xUWD@$!269A8qU}_BLHyvj9hxRxeL&H)$0?X?trUu7^l*O;lqD5cEKfLe^4eQ z{VTmhGh&wQpLpDFFXxOgL#iJp(L^%VZ(h@Q3e;aK9^-4z653+?ajecFMB*$UWa>(8 z_>g>>R4e<7imYMH5is4yFdh-Q3TK71CF7{tu(DbkEg+%L0ZB$*04Q{JwxU9_Ilpm@ z0%Szit!w=?PDjsn3@O=OoQ?oyXYDiR9Wazdh|s5m(qINH8%rBRQ${yN#KomO7^=_q zxqcF4C!-WtF$l4ipY6Sveng{W8ITHDn(LFM52Xxb{GEmqo4b$)e|WujF}q;}*sk}V zw_ypj!C>`xcEdo=0EBJ=Df6nJ`pqCA&C7|OFvHbGK>@wxEryxk;^qbXT=)?K*T zoiQ%_t_R=gDEZnl1GIYUA6k zTyYWP8gc&OU+r~iHl9`J8#X=VQ6QRr4ITIZ*_gDGy)#>#Y7*A`*r|M=%A; z?@)sjWi!|}37dV8ILFIroPG|>ejC0sl@-aRJYo@eyGIQFTv zFPt>fr1xUugo0J)QRuA76%BtYRH=I&;9wTZpuMD>#Ss>|ZaVW0A`*_VZ-21c)Gxp8 zM#^{i!wv|yCFxEmA)@ugCbimPH@72gMJ>r2JA<(;$9|5^gTKzmBx8&zhQJzJHXZuvDomI8l_y_TNtJuHmGBzE4Z|v zca$J(K=qykj_=c&sB86Zeg-8j7kLQ{P>b9^ zyiRn!pV>7=6CBINvu9sPVsOh`=btHBct7^obS--IE!}48e?D9t;@}h^nI;yz5D6xu zIJ&JfF^BDbqZM@evPVh;GKlAr8(Y28$xJljY%V(+F+*Axz4<+ z;S9*ViLB{DpxpP~zDn4tayJ3&p<7{}6F)Oc`N19&X?{fzlgg|NE5_ctPsz>>X2hU` ztqXr5a;NPdQ6O4z&s4A~`?pn25akIfm2qkP+d>6M$K>?QOVwaa(S*@YcQv8vt#f?9 z?I!!PtjpywLJX=)O=w+grRpm zG2Obi_O->$;{3xGO3~M@wR1xC(8sF}&dU$$t?J}ZHef_B{D`~Y*N5hR^1lLltY}ke zY$WMRYX~boG(L7|Nba9XU0T$7QxoT+E!+8baxXhze~`|qRUkfTLjq(1LlPv2n3W-M z_Hh#=NiY{mtRfC*| zex5jf_LpHFAJbj^G;U9Fe?({kQN zQ)@(ZJ?mZdz2nHcBHM?zW#-=mFEcSoAA+b_xS~=3QT*jLl@7ZnhUfM$fckQ?kWb!K z5ATqQmj$Ha)oYx_%R9r687&6|Nl3~euvvxHa?{DMj+iY+`FHZ_G$MTivL`B#oDBWf zA+C0S)X@lSYtWDvRvr1S=yaVU8~#j#LUW38h$=Nj8tqJ01}n0;+*D!rrWv8-831=& z@?%QOKMSHiSoqlezQQeizF%EaIr-7BW>fl2xzi+%L4RCwyGcGfz!+3yc%1|8JQ_zX zePmf~4s6qKkm+A=BN1(|+AaROLv;3idsXXLQz=uC9l?|i7gF|hx_gm@GCN%(3O^lS zVyb^1;PCxRZ3K!yS%^K4mEv_*W&y|qj0DVo#DsFFEW%cl#-IKt7P%RCIxxPU*}-;q zbDa3~aWF#IopmYfVSL>aL1mtarGh#w-9h7pYL?=*` zR>t~Yk$^t-AF#yTK~|4%3ru&2TP45sTV3yrW2`qmSIE1e>x2IQ9(8w^)Y42?-vaVQ zG&l|p7a>+ahdu)biNK_`-YjIb+UK?d66O_qpRyd-T#}}dm{|NOS#!eJk&jV|PDFV2 zv<9_pI{(l!aLRT zta_E#fsMS&H;@R@)(w*8(h%%(g$iJbw+QNV$BqL30&E~vi*QBBm8q#L$POz((`AL^ z$1&=*FSESvMEInTdA~Reop?IoJ%Vo~DPb>N_G<6=Z`Q?a?pNUr&2~MXN%j0*z@H$-5`-DZ9sqx`k%nOcMe1o*>e$1sZpQ-@k-B#bd#>*O#bsReGN9 ztgl);MqfBL-^5n!eK7!`=yULtcFF%)UO9jCTGu z6Ax=2#vI$yca4z&+JtWi8yn;kX>R@PFNKHWzVk_MN1l;?04A@b-@jS>Wn}EW=cfe} z-XZ-Gc9zvqaf)kJbg&J2MjwC3Al{YSyk@gQP^?$^n@!&6wW(e1D-xFKf5V_ZeSEt% z2lZuK^BQelk^GCD#yQvTT>QmwN?}V4OV3AWHtK_RDK>8o^)vX25)SKMKBZDJ0OgY0cBUkVW$`!XgjIS z(CFX4T1-xo&*qZJL)F6qNz}m)-7i?gL%PvMa$($F%cFY4i+c{-#G>;yBUmn#w%Cr+ z-cyhw;mDHpwM-!aqyW?ossB{l$mdfk#vY3!=WH~r5qX6hzW}DvwZ{n!e}DY+_5<)l z!Z&Px>+Oo&(O>w|?qgbYsJyc8u+ExWaVW(HFsd=hRG&Mw{r%FUv6|$0dzrCToqnYX zO5ROH&k-kP>;H5(GEEK_@+;+QJ1pS7?vq)k*h^#4X;Q^(f|z4S&;sbs57l>urvt;k zRdae1yCGSz=|Q)@5X~-v->076VKqqq&*8}$vZ;qb$Wh8&SSTnGsEmseGC=Q=DoO+= z7Cg8{0RwCguc@iO-Qf}#R#J-1hn$z$*6#Le=*^07RCgS;=ZQ#x{lmDApvNYQ;@8!K z!U|2TqnmJKCvOCa)`7T;dS0c64Q8Itr*-A9GpAv`NCi}}gCh40+<^1@>T>gPN1P*a zb`rpEeV5JE4&A%YQV!yYVR9MVu*fo}N}a#w37nIWT;=z>^^OL87Z~0cGu3KgLeVFPn)(0)3G!xu Lfr5HX{YUzLB8kT5 diff --git a/policyengine_uk/tests/fixtures/obr_receipts/obr_efo_october_2026_receipts.xlsx b/policyengine_uk/tests/fixtures/obr_receipts/obr_efo_october_2026_receipts.xlsx index fed8a21a1b9daa5d5159bc93edb58f926e19d6a5..d4a90ce2d17b8b59e5ba87a5e8538bd4477daa77 100644 GIT binary patch delta 17443 zcmZsDb6}*+(r>V_t&MGCW7~E%6B`?AV%yHfwr$%^HpwO%+q`+-?|kRH=id8APvdD+ z{i?dFr=FVey8wIo2@X`00sn#y0s;aJQtqUyhah3`C`%0jawZK9g7n$yX3gko=V%46 zv$JAwv#~zQg8Q*689&?&dvX%G8kX7wx~H)%Cjm`tZOT*x+PsN{mK!VNTOVA>hwNV4p8Y9VTpRYf=qXlZ+Radx{-U`^peq8Q88RYO@0Ikl~-F`z7<`KU6u)+6_ zh9o5X_C7(9_Ybqv>HKcKzT@VkJn?5*VJLZzfSx+Yo62bX zgFCC^!ecKCgIi&01jYbuC0s?;VMu=Y0j!#kK*EJmzx%@JOw|MSDM5H=;J8Y#IR58c zn4JKR-(ICj6_9L{+n{dA(JdKAeWcp_Pkci?d1Z^g-l<9`D`47Qh&Jm3%+dFF5W(|xsdA5oPi3wx|Cs<&oCa<$*%ooSaXCf)b321QvuFx8E+|L3-!q$@p_Q0hUqd?BC3 zTxU9tCa$Z~z0NW&IOU*wy39S_!`b2KU74r43&?2UzSrsEqT~Zqn;Cnmhdb+Iiu@Sx zQ+%GetFYk8OmUoeeJgPyeq5})v$@KsM^Y(ncKo<_eckK)HNU$jFzz2UeqL|X{Uu}n z>j%}tS+9fV*7$|Q1)M{r*XHXz`O|stQBePOO!#2L`2Mu5wHZ^SvOZN-QXXOOk_s(D zBv3M9*CKQGhHKDHSv!rq4%eG3heO4KMAA~ZaH)H$RZ4YgoVrYFRAG;W3LUMGfq%gE zP1_D*`6Re-Z#P};>eZ>g=*x<3#rkg6>h$LeII zx8cZ6Q~ChfgZdp7?405{mo$Ww-p6(M0nN+&?! zJWYn`YT_7$Doa+-?op$iTyRlx*_D|@YwrBXsx5E9%cIOfSBg*WZ-b&oti)G<128}V zs>>teeB64JLaIQ$+O=UA-$XKsGC?!4o1`eA=hm`)sbqn_9MNn9DeZt3imk=m76Wsk z&?UOqZ&SB9BS{qihgC|#JFOb%i+3yux>2xJDqIu{YmbJIy4}{Qyc0|pi|Hk-eZ>@) z=#ozsKWCdLsyr5zE-Brf8%4&l23&Zs;Z#jogUAUqRV^PI!X>#c6y{iD66x_ln`^Kx z8$qs-A}1!+YlbZ}Z7u7VhI~WjZ2U>M+k|;7vbo~#Q689J`bEtw^ zJqW*60QgmDINGm-aS`G2O^yrgV*o^tiUsjdf*7oCF)TWBwl*HBW~XlD5)gN+?@Nyg zLKFyZ(j0(S3mwd@j0%0Pw2?TFIwxM(Hgs4j7H`)O^;fV*;+NK31Qp>mIXMAQGMLR7 ziTD`=Z?dors!+0Ayz9E++#{2HB{F*;(hR0@)W6wFZjP86#Fy(S$)Wtc&OX2|?oNax^e7r9U)m zagJ{ARGFcEl{z8sjpl*(s6d2bD^wz&ROByWh=+yBQwI9VW0R6_>j6Wd4+r`bYvz6k zt$`<$71x*jsVk$b8#Aq?W;*gy$b)fw68wWKBw%bgpDhyU@BkzbOl&}t76Wi2$Q;#9 zuuaN{BgR3&qT{F4s~6csOS}m_*(@c zY|Y5&b5bXh#^FOE-Xu)Rm{}pWGC?Je$SPU)$SPgN@{<8k>oR%S9tv81(2TNER5lE$ zohJ!s1x+tl*QW7Av)2QrdP80+`HKk%1jQ#DqFlJ$*3@Sps~`#1!Vjh(Gv6`e&s64G zH{QUrY7)e=D$JoaUTjaPGeCjILcKAHS^JkFQg7ycIaH;g`aA2*b&RN!Z4oQ2@IZ#B zsXEH)vv}&tqX{;I61AZ`^`-hM2&Jk9;Ub7Z8w>6cBxw(|rU~&DdnDgpTSf>l!mtcN$5EB5v<)O!uEh98F+QlkVY*9|an~j|2&am}%Tq0xT7V~H}SWxOpr4=@l!{nPx z@(2$p;8u_v61;zA& zLFL$=7#@}C43@-J0tO27pyn)g62*#7FQ2~3`*Y85qnw5STU2;I3x^?XR;r!ys=0cU zWmLM9sCA_W6tK%-pR~G&?{LTciwEwwzj^q({gVfIvcGv?QBBf@l~psgU{Z3&5yf7l z)u{o`8-wTh8IJIGxlB{3qt0Ze${Ghsdll?&{Oe4<0!(C11C>#LWp_j(XCARatpvtY z`baF2%K`8~Cqg6~X|TsjYVdW=NvULQf8rDkf;1QzuE=V@>3w^ALf5B0b6=lOEWz&r zs|kWUc>*T7;4eiH3h-UoqTVLp=HOw{Vyw^N{hbu$PnO%zGd+}h$R5AD@WlZmv6Zqp%(QkFloDi^9N;X7C(|4 z*L2l*ZK~qsGsrS#|0zb7@1`fpK1WYlC*=HS~?8S64-9 z&;)%UKCeam%%6$%D}3J+#n8?GVrn~ppjbI}L7;kzCNUtPksTI`R@7QqN>cK(5NKWY z3GZL+5Se|0uzxWEJJ(pYVKD#wc)zW*G17&eZ@Ktvu9++EZ2qs01w)B}&Sy*s(cXr0 zzi`C0b9+auf#)j@kep5=$j<0>|%9Muc%2f7;v6i$ba@W zx-aPqv;tdrz@;i~#Fg=1Cl`rAWbslkJCdNCK8=oQcM`N?K1F;YANI54 zvwEJyAWu$A!DoVX1@Z@3mkE{j>8YV)SZC(iYpDGtodiKg?6nT1zzKe z`F#jJ#2J=~1@-|30~8UwL3pnu9HnHc1(ub$UiUDq#6SX6A=^+1(NE(Eit;;{NzlxT zCf&R?n?Kg&f=h8=_B;wIoN2A|Ll8xbrkNPiO?ZV^n=}`XT*)bPN%FV2+*fLwC{r)o z^y>#$SUp9F*v~v%H&!CL?E#iQzEk6}HuoqGD-0D1;a3osEePX4F6q56KA$gE8Ho0; z((h2%Kyc7ct8)lqL{y6f!Cc@7-_VoLlb$s6X0vKe?b&m@ay;&Ve*5`PHdWlV&r5SD-q&YCt249M|Heu9(( zDO(yJDAwKjxi2)>C`dgGPOipWSBPOR1YTfQHg`)$7hWVS*vqLV7FJZ2%7{MMSNbGH z@&|67WNYC(8YWmoI5O~&Br@<7EOOwdWMHmTq_K95c(MPGZ;KhIt}*C=1wabEFJ(J3 zfTO;h@e77bXdJRgVBCx_&^N9`lLTH4KsN8sP6Lyd?WPLTG6F_crU^jqdX;v>II)() z)|(`PwU2{HEiN4fAnU(;gR*gP_t}d&yA2oNXiZUdA8P$M657t)BZf(&q#7#pSi+SO z5f10aG*C(WOPIioI<+3HnZy^){Z|r*kJC?+cW+6YdRd9zyJfUKu9380CdgP2(b0Sg6&h2HKXgu*L15R4Qh z0|wISw)m~>M93Hn(BzndvVUgR$cR2j)+dRs;@^wQe$KCAEEE_wBK%p|Ji-sB3uQvp zl4LR`?KXSK$OUTh@`$Wq?Vst~_cRCeTZH^Kc<07umA0<-PAv#no0**u zxYn?;r>?KUk`t>128w!>Z>Y1V;Y%fyT0&ouyRDBHHwHW&x7!$-0FW((`kwMC)C)#c z8%gAf$}lWg)QAc`JN#Xtu47?_ZjAjyHvImjUm18h%ZoxukRt71Trm~q7`H2VAiS$; z>_`<)fkho{%y1dPR(YC`i4G>vTU!IE-Fy-?tUnE#PZPwMT<7Rjzo#>O4{f7Wow3%a z#B7By3vsT967A=|Y+U9%MwI{BAA<1sS`k&G=| z2}0mS*<#b|lNsPihyVFP-80ks;lwV;;K7W&kxO_Y_$sz@bOo%Rli-A@!GiYNJh_D{ zjiCk_-;%gr916qG2sR2`RM@xpG(oJN7`GKpa@${DKVG_VDc9^qi_6sQ@VfDxZSUIG z4;G(hfyF+~<<5Zu)s|7=6t8lXvlbQzJJmp3ZDU7g;>W050qp&{5unOFnGGzQgcp+$kJ1fxGLVYf?p2e3%H;c%mg`+ajkhsA`%u2(x( zqF*G9e)sY_m!-kj=&Wn*DscSO$qz{akQTDDK*AIh>m~2zCEFs3Z{Q zJO;$1h0{NAM+<}qez~xasY0Y8U`X%wea@Z`LAoCSjtt>p?kC%ZoXD8Hvo>|V@Z|xn zG|qVuu}O;-XyT4Mu5$2;xgd6xjEtcs6g@P(KqmZ*MDRs5%oJlWF=|K(DfE`;`N;DG zBGCA4wSj({eK^{Rdsl)_(hrrS3ls>5@t>Zc&VKP#WQnPzQ#>@|=f^9`ZN#1F7k6R{ zi`OpBm08utWEDg1V@re9%{Qq@g4ss!ToyN=X{JL<8#cpidYD99x`*MDU#cFjEN8`^ z6)NyL{pz6nxY9_@xF3+tRku-X&>hd8+-Yn|>Xz@66Z=gIMio~nI{s$S8VQ)?6KWeX zes_RyhM@8TK;gNDMK~Rnc4~2XCpeS8}Dm zE+Jt|u+3j)U$6{%uDMkDfY_V1T$iGt0y>)t1MvIWH(PH|yf5M#G?Sy2pu#O6Up0X; zCCF)eVQmHH|aa_2qGzmBc_RROCHty0E9V}88bIK%=!MrZEZ^Q)Dd)C$32?s~a z$U1QnV`-?lvYPJ#g3pR>Tcav!VLgRR6w{GRHQQlEy)WezRngc{QDQsjCt8O((_~Mv zo<#SALNmpZxv~DsmJAn}=F{3(g68|$@108To1rV44w_uP%U6wttY~0U3AS@xxz=qi z>v5XNdXx`on+iJpHb=Isq3z|&5@(}Fp`v8>p5LFmWv*;-97@dDrB)R&@t#*5! zGn*G^WCaluoN0bhhTJxejswJKkNNOAFQ2@^U}cSC67uKvKsDV7SOX#`>58p2sN&a?VM`5D|)gV$@JlP=dskZ%-%)HH%O zl6UUf23^lOM6N~5Y3*6C=_@JQuk#-W1{N0E@ug<&tVLV1leKqpEeRpry!LoY|W8N~ab56!9pKFH*=2n@@AvO-CMGfG|CB+C(uO)@(Uu-3HtPnVSfCRny}cyvUR#Ho^xWzo%0AFDwQL(Zez(!>+QJCANM?^h{c zx6urft_<2@bb1XR*vZL8E2O`D{VmD6w%6zMn^z_kD))j{E&`%2+%6+I+RmHqQJJo|v;ga>sF*L}HsANgH)kX(+mkXN(wHiAY5PXb+$x zz6z$BhWX7w=o9`9g&Y!Mzdo=NH{QlYCq^Q6TfGg}k|snuU^6Kn)Y2W5M`wJ3^ z6tS88vzw`@-q+;32No8SY*w9>H1FFpI+=OXcOZ@T3Ei`8NhQ~7f+rr^HzT}{(6v5A z{i%^;RRv#4+Cwl(!#C?EW&a_Uz;s_OYAJQL)3Sw>U>BYFdVwQYzss2Sa3NHM_bDMs z)>}a1CPJHjY*(7p{VqnRD|zcrs7ruQ;nD>rZ-s0Vd)iD`;>?UxD1VK(_+7M99LvtA z4p1PUzRUTH=op5Ma61lrTX@^@S$`X6_2@^p76+gyq5gEhn$m1Fd{v%8(_{PC7jWd= z9w4jK8I@_mV@k_4?lXbMgdmYyqBW;ksrqQ*`a;_;VM%P_RtT*aoDxQ4^2kg7zo&XP zXXpcUORrdHzs)&^gPVoVUw{BDkvGOEeScvc06x#@gF3?^e z1WSX+#LXQ-Ihco35a&GNvl0h^9DIUG1h4V8RRZ>mzrMisXCafpD}-S7)xf{?GCdP= z(!5yppN(PKv{-1(2Pc(q`yi(mADZ!@wKF#Dqh;2BX`%k9P^arKL4M*eAl3N+c!^#X zAxx8jzOO&aBfJ9YvWn#_l(*^Ks2q-`P6i6Exhq9#Jr-+XxwLVnN9$>wBtMWC9mDIg z-d372pb z@vTH?j#m8ba8Ivsjf49`7P)&r|HHO%c-6JLi}6+c5hDs{GYf#~?{**bF)(!NB|r|Q zWCs|!iGxG;m)`;OT1dj2cRU(qf=pUm#0`D3DfBtlO&5m@pnsg{@ZF}qwH314)hX}= zV!sX;1;;U4M%a#qUl_LbZ1scvGPZL=-78>pOW%t{4U^`&iQ>w;p7Mf|$4xi!aZ2cQ zF-Mr8@D>2p$nYL)7~m1O*vtuADZ>`K| zTdvPo>e-^&0ZZNq?&4f)^mMq5;IGT9-4E_%{BQ$m_-MRdPPw^ygoUNsY*is1o!Jzs zpH{j_+WiS}Dr#9kuc39BxyzC|R`ZacZ$FX!%G39!!$%%YWc^Ibc6rJoKkgykORc4i zs)O<1=7}^@!N~esM(a0WR9owEvzIhvXIkhB$}5P+j8woC8PJ_1uH55Rq>`{pO1;?G z?$tm@Nt4(;=G(U)jU> z-K*H2X|c7tHeY0Zcy_(Tm#@n&Bj#Pdp7i=WkJm+q>|j%WABNfX$jYJ_Q>(Pa_Ci)K zfAe0T0mfH#lk-|wnX#qHrz;$Re?Ls&rjybHwqI4@6_B}ITSimLZHsh^=Pk`lYKw4X z7l7uv!hs* zNM}Hm5F(w~7!_vs+xcVkB3?;SDBMkw8RFZZa9QcbPgWHSy%pKP2 z(#v4qsiERsJTL~d8|i@BQY-xY%_mk2Ahg?%&XaxU)YyHAqXB_CQ5``{bp6}?)&6E~PoOZ0xIUs_iNT&EpC&Eh)1(Q(e3~@iKWv&9 zK|fK$ju3nW0%!YA4&F$=#vKQEDuiV!McgVytGOCfAkaJOy2Ar#(IVnsxC7f59~|`9 z@5#bbjdmNb1>9IJbSXP3j#UbeBi^!WlRI{KoC{Ku?n#I>jn2OwrL} z^;B7R(w(J+on=QXl3I9kPLH&auo8~RnwurNY5ZwuGt40+6vl-5O^WI8lxNobyRUu- zfy6f9uR3C293BKG!lyUifR*is2z=+=MhmBSvWInGS2{=Sv+i(uYlm-53SEaUQF0I# zvT0on^i|5rQn6C0cNdC0Q7m5}HrcGgw2(Qgh!y)eZrhT$f*`KTVT)d#Avt`)EC8ZE zz3m>(mCh5kUk{E9-jxB)P%~IuF2ynoH(H<NFmujU9DF&hGq+cQ4Iw{0z8P8oaDwtE;^~M*$Gs8AZuDx%F$OQsidTWy9|qg7pqxgfbKq zu4@jk-VOnl=&`aF+n+OT4cFSqss6wPhJ^uA%Gd`no|ViB14Sfm;WTV9A5q zrw$7X;UoF!X3OUya8JW{7XrD%hxv;gi5lgB%i-B|`)?1AE=v${#A(dt(Eb1|&9B*% zwkRQ#VKp4OrQdB@ z;dJ)>7(wwO$af4ZR@(FjpNiVwba+GHS~LL6t-7_jnHax`NwS-))G_iV{r05#cv(wp z>LlgPeAkAe+4%YfiY7)aQq1W6_^9k`aSF%PiFk=8cyZIV*`7a+hmo111H$X>394s6 z{_)FnQj`n-R9CTaPuBj?sqyRh(!i8K5AJlnNm1PrWDcWM;T0M7mC2Dfk1!fpAew}7 zwc4ieG`1QDhmjsw369l5JPkscTP}tN>NJhp-l)TC@c5d|DzxL0d7@|C<}a?46Wws} z=M!MIr$e8i?XKsvJaXZCiwYRMI&cfvMA=$28;zeyN&zqU?jk^TmG#H&&pghy&bv?K z#>~B6f~?;6no!v?=Z!@Mlo7-g9dDq>PP=E|koiHOq>bCg# zBbN(ydb*ts^`fcw?x%q(?xgi{)9}=2Av3}{xMY8RuO3)~5y~$2zKzB)Y5B$3Q-sEhQhEoz=pnffAGg z^DHa|uTOqB4)Gl0`X_LaKc}Z!RT4FKkF#>Pn%98b`-Ah2U8hl;k|4(9JZOL@jNcqF zo_W5Y!YOZ~m@B@ur<&wh-{2m25Mg>!rCpwX!5pc_P1@^FCyV9=oo?}rD8wqaMNBog zfd4%~c7xcZ$btuClFb4tHLd7W#uRg4+l4dxE2{MnM;LgkT`PPm-c#)P=_b@(^|S^Fyr=_W5ml@%T#Kwr zOeg9jv$F7X-C5^P=MUR`Ly{doOWQw08*%aiD>A)S)m|eL@D2 z)QrtZu`|^~*?q2o?z+G{?Kk!KhzfZUrQA4)LU^(i!L8xo${{Y$W}#xo%Awx$o%Z(1 z=5_+{twEL-b7-x>Ro0?Vm;!-3b-L&$di@F;o6N(;6hLPp!yKPn$VF4zlR3P;oVbiwRg}C zR2PWhV=HtcMRbQ;A3<^ z3|kmL`|Q=UVf(p7x#3&z)--U=g9S|wyFngj9VN>391xEz85;_!R-=Sg7S z<6;6y3`H)JZHUDgpbqoko|6xKysO{=CT=On%E1!EzAU_7{1SKcg#U6n`7-GyCHK`J zo>|pXUNty)n%|ugI9*MjN{NBi!=7*3+UPFCwYb-V@wsSFy z$uN|o1SZ7sHEqsZ8%)l4;O;OIT6IU~bi}!rWjs<6CKo-1&CH<-U}>i(*A(H`+iV4i zy!LZs&0Qt+=YM>De~HG450+@HVe6QHl%k;JjJ_{1z%>(fa#ApANhlKR;v7$QD6 zgsFy&H_)$OhZw#9zL_nSWn-q|(|^aU-5LqhF9?r*Gilj7k*0#~cJCcJua&Og>Cc_QMCC|Axxg&cY2HOs1a}l-5Es^hID&@i@xm;he2zUO~xpRYr%1 z6-*iyRc%a`OC`39u9`0kg#o>QW?V+aNH~xxbV}CVCTM>L(XO3$F2PCq0Ckm_H>wfOZ(ghZM&@XKX6# z-y96)t{614`h6RpVvrZ3B3h(!o4Ydo_!c-q=9EV@wAT6c`#SNyNH2rP8q`}+AKbUA zUfdPG{rk10k?Sn3wphvw18bl_IL7(weNwO|zb5Yaz4}}YaxF3=f0yTD+ThNTNe0(O zS4Bj>vZ%8COCsvUj1_$JnTX$E?{K_O>LZV;g6-sLEK+j~|TkTob^k3ZjW9KF+VO zs^{n`!0p>kGfI1sBM6{R`umT3`se<~bbnpyyX7SHDtl*V>Y5n=rPeA&} zxK(#Mj_FyFMR$F!+1XKpS>YmeC1X<6dr4(`$;)tH zwEm;7)J~!?-$a2|VBld0D698!O!F{3Z!j({f^1s8{m0UE@(hS9H|-}dGWn6`jpwNp zJDrua9TPvT=RN6~F@3?SG4)72HFcUqF0lJEwQOD5=Lf=j5g70ta(&%~F^=$i!)t!h z$b&%S`?*Zuoj}EVV@Yw?t%9wK4Wk)%1kMc7N<@j~l7EDhiQy{<{s@vP_BL?)iVq7? zrdswn7(@K2Njz@r3QYK5FhiF)XBw>-v^_ST3}HkJKb@`QJ{5if_DFc!dGP1J&L#a8 z`q$Es&QdPj?m=LNztMe7v)*!J95;`7DK!I|+Uc%Pd2&xipRly2idN&7Ep+wD2F@Tg z+OyN2g+(nhshyk`K|zW4MRk@i9n=-t7Iv`Z>T9b)GI9>Nr2ql^hQFHW3K}?* z)MS3i87_X#q}ez(g{W;A4kw54VgOaC4Q2-mM|~y-G+9J-`|IJ~%hU%Xr|$}@^qf!J zr%c=h=nI@4Kz6bTOsEHdS@Q6znzKmkhsTMzn*RG=|l{Ed}x0~hagc7_5_M6 z;&dGYO5sl~xxe1$JedqY+q9Uh<{>oG|7A0(`;TamwC_`b*D*D5ZMD_sUS)+l-lUl3 zDZ-;{=cflhIIUD+p8yLrnG-%%K)4g3mRD}hyHB~l>+!eK{ZXh6zmQoJzVj}%PXZb+ z7{O-A#YS zaKb<&0I7+A3o@{MOvsSG$`il#NLeY-HPLd~F2V)6tfAYWvY_1ELt@>9IS^|F$^OcT z!5L9TVdQW_u38Q^wxjxn1SJ+!og&L^1`AR~j_6OZ_Vc>BvMIp+-I9kXZz}?kc+U&+ zo;gHqeEHOPGjXCS!)YvjbMO}xn^fwNi#CF><>?Dgb1ys?J`0l{JVObNvCm_TOMzk*gP)3sFgBvF00IRPI5 z%NHxo<)R>XQI&9OL%yzmv~0d`)zF8~Kc_}LUFg)2>8Pcp7n&f-$@N}u zUOX0G($DV4y?(3~gU?-J9RfR&P}Scjd8A;d`L81eB2yg0NWpv zFShQhfm-Ufk-?MibsyU3>o+_0-ENahM?SH3^-(zPQ?4fsOx@@D!7pbFGC7D?l^P=A zp}1A^`$Yy8T-)BF67R1ay2g86S6x#aVui*jm2-V7-9SCthp48zUDD&Fqw_Ow&!&-O zo&4Q#$3{AzuZj1bCxhTWbu=dcGj}G8FLt3K8SZT z9S9<)yyyk8U`7S9^b$_4(q;nn6KWM%3V^rO42a>T?b*~r*6CGphy*gitvIxEjQL(k zzzdI{+TqBO_se5|rh$X=RRi0g(pGcxcsAop&f!z$xbknJs|%BRo9*Y;2hZ$kLaZa2 zjO3(RmU-V)3q{Pw6kZ^$7^)H1x6S-YVqVpR+8b(5`E8A$+LPvel`D_c2dJTMW5(S$ z9wXg@tJnLDOs>BQo5Z@-2;a{tQ}$0;fnq5>0@SbDn2%&!-<#LVL+BZuiG`{x@%zv`e<1BUanH=J*}W$$fFM0 z1sYiGuR3evTMnn#J6+V#`@3h{01ZVDfRy;9a_!P<|P{)^4E7f@IdjR}pkUbYXNMbdf|h$sEVw4MlHA zW3UnQSRO%B-wp6cnh;NJTQK8mdtc0B;3Y+M?<<5Q!Go& zV+;_KYEC}#<6s9nVOAa!1!vMattQz53*IDJF%Sl38Z#V)cwT5lSc-l@A83^7$hN62 z9AB^EEy(6)4m*o4${Ko3?1B7~A0#cJ>N!?Gh0yDga-aC*`n$S+Vm%F~a)%TV8hB~L zYE2sIhU@h=mw~YAFW6XO**ohmJ5f{G*SZYUC|mp^+=CQ^L=X9 z`egsnXUwhmGAN@PlND*_J?Gd)F&M)=!baImbSsxjE;0mi5V9EZATs$DbJj4! zP=M5bMgl*F{_-mTWqD|eF9K@KIpJsD56PFZ=14qaFZX}h*F}OR&DK);ydCiD4MlUY zXC+><-A!N`HbEV5R5o$VYa_5F1j12FWQEqp1xB}^zWX}Zz%#GY-fTpXISvGwn)0(! zPqMO*_i5<>B-T#L!Ry5wrc<<)b{WLN_|=z%fE(78;PpIG;)Rq^Eh_KLxatI=XrV>O zn?h^>hXi?02sB8st>V4Hdxu&khXsxcl1Ke5B>N}`gGX?h?HYfUY; zBR%t`Xn>*z6lodP`On2Uk&Dn9SBbxn7nwMY&7e?D7-y%388=}v1iG}7>*%_fLgKgr zy`>A?`;={I@p*PlBWMdJb}gH9<~drOS3+NsT2}4Y*=;gvwKFzXy%}2W4Dp5ksiK!c z_+Ts#?Z=n)H1~eO*07Q4B*zU1k(S0b;?Qu&m?U(+vV zGhZi0fjc?2GR^2;%e;gCDNT@BQqdo4?rRcRdtFBK>LIf6v~FRZ6RKHX(iQO;ZKF-J z{~$u+ku$4`CMO z%GC829w^mxHjS*2FFC^0ffy14`$z%@p=GR1Gpe~K?6Rqh$6wQ#yG{6wLHmRQ#h@cl zl8LMx8!lAwRRTAOt%GYkGggVY`QVvpMdd!Kw27}{k%cgc=tMOGYp@N&CarIrL!RW& z4jU(yL!V+Hp#QZzfOXxgyZ^Pwb%y@Nd%$Z1EHDT<0vVH(t~L$$W*hSBfOW;4t;B|} z?;j6r_!=ue4BpW5XJ}zxs005!mw$~8|0W5h z&lczU9?AVJq~k+GAB4mXBu>M3gh%i^JFF-;oQIn@gMBLAK^1kUPeMA4$ zD^8Ajy*zvdrGdh>9gD2*Q7Hdet_JEW!Fi<`L3Q{$rvg^K0Wh#%`g!8 zA|~L<2CwNzhC|AgaeT6?c`REFezXTI-5yBeB{p$?RDVt7ppgZZZzR@5FX2U>TF^BY zZ=d5iu&|fX!K5I9A5g9JoOm`SBvo3J$W(<*+DFUt`iGaX4KgHYF-GgE>&!-FZ+!3i zp~B-s&8OPB$v&TWn_kZM_n)>wu)k>0MZhzVK}F^ zF5eA5Zqj;glf`!)eoWZy?z|D{c5`gH6Q18@XUEs=y$Kwb=O0!guS&o8R}+RU4Y)gN*Zpzjtz7-!2juzu>uA_CBdn-~TA2>sBkUw-tkeYkQv8tNg}j zd(b?8F6qBJ6NV%xB6Z&mU!f2ckf+Kf040XBd$^zz{&ln5A_H9z~A<^-Z@k}gl`3f z&()e-izhcQ&J!X;^Xr1@P6cTV+;8)?tSdJF+!F(PBZrJg`hjZ6Xj`m4AVA z8>;St#+doVMf?1Hm##FpyK?B_tG(0S;f0;(>+$2madgy(1%2Oc))NK}Dhml2F&Q}- zDH&w}(eKFEoAiN>DsQYbZ#OPzB-ZIj1SC5rZM1S@PW`xha1QH2?yFp?T&hB`$ump`n>U2dXv03MUK;ol#MuilzBTE>bf^zW=ol$PLYYEAr zFGpZ3!7nTpr1O>VL*TjaFr&X_$NXY-^t^NPP?YtEG0cJ7q(a;&a)^()$W(1V&1GlF z#n8g5cz>JliECj6P{tD^6fBFX!c7wA(YcX@oN?(7hl!c~6eA;{j>NDPLXajwj!DGF zE-h45r=k~oWD42UMGB0Jnr96B&cQ^1FsPSHi$iT%h@mVlahpd=Gj6YeG^dIbDxKS71gpEpqajs%B4=D?005H2KtA6R{AS!cphB%5W1c+ zTkc_ao>lmk(rtDg61sQ6A(~hY-*2@>4+k|`;VN0uQLg!5nBD-{ zn9v3S^Zuk=TbZhS`(sIA2ONZ!arG0z0>O{oB=%a^GH&z~>z*&HMP+uCpPncY2NWdu7*ZvCoz^5jH}Qdd zq0=JQ%7P4yH2BC-5eHMqIuDYd7xL&m`!d_XJ#pT;08O0y2G6xdcE3j)sh-v%HM!Y; zs0zE-1(k=wEL1qZ@@-h&^z$sRjKhBY$xRMVd7izt_Q+PispZYQAdx>aiAtJEntF_C zjG9K<7|{$2@YNZrGs7Ajx3K zpvmCsr=Qm`vxSHoNcV4Gr(iYA-&1_$K-#aM6YZO3q+UyB)G2!5f5{FE!{j@HJm+{QWS$-tBe> z`2dTk+Ci-0SIEDk^;CHsm>t`(eZe_{KA$B9=?~UgJbMVj4~m{4=1DB{HSyi|#h{5^icTfJpm^au&g)UNc0+ZThp%SA=tGOdC8#lz=W7u6K)>i8-; z_C^`SGyMbqs759e&E56aGKI3Pl+}j5T1e31ScbYjPE|RF@Net6pwYefe&CHp*Pxh2{dedxxONDr2O7XUf=c{Not+7 zFPcsLhy2euiqbg_{2i93`~1j{F3%PO&Y?A`*T{oN)7?xzAMF1igziyL z3-k(Q4}bk|h7r8Wa5jVM8j+PEQ1jey*;T+%fX5PF-HocRJO*KRL@g UA4c%2Fg zZrVPxuBUsH8+xW~u>a}`^t=z^@l%UG0@M6od_UMjptTuEEjuGi z?tIC8|8++3-&C5dJm0^d=$+^+S-64x>s)=AXjc6$LnzQ86($aEazkwXhx85w_#Ngu z^>>`_-@XS;d&ywjS7uSlQNm4nl^w;CUeOWBaSBHz-&>r1UHm7cIx%LT7JhgC7cGrg zyl?|3jr`I7MN4_5z8^D zAJrBlo|7_2C{v75>}?mXb=S6chA((xGcdP3GL^TqE7|YVS%a&MW%o*I5ddNoST85I z9ym`C%h7DZQ`)3x9(f}{1e8CMGTec7QC?W5K*i$>>}B3qoyeWsu-Lm3bsTxm&_HtC zx9rlJpg7sGX5jnR4lFrZa1Q zc-H%xS@vYv{-hCPhnW$we_LE{$o;ako?P|&xI8Om!sc21N`*LDej9NyPFCXnkbW

CL%z?Kb&FM`LKwii!gDGd>rC+Lpe$tFfVqbK`QY z$*ZJFlhl2pg2jQXVrG#C(O|4{y;8vO%9g(~t-3sd*{I0lOpI!Zw6y#)#9zynu-#2p z$D4242YARQo5i>sA@Jw!p}0I{c{6#@f@&EK>nBC(_|fr@o@XkLf_#~IWkiM^i52PG z=BCH|{_e@~9woqY`AdQKp!dfGf5#J{q6{b)I{5$RZ!r?|8rksv^I`?$|BOKjIKct~ z4UCnO*KBoXYS5a@zg=`P*ER#D22(IIaGO5k?#YeY+?ZCCg9YvZ1ym>VZFhCaSO7j{ zqc|h6C^aRxq$o2ly%^LLM0Wzp>;}w6Al<;2ycJ}m5oEFhT^mYI6Ga=>0gyHsLvY74 Uz?+o~B+CVaTbURb)}00M093MPN&o-= delta 11073 zcmZvC1yEhfvNjIE-Ccrfu%N+RgS)eF_rSv4HE4pnySuwfa1X8lg1vpt{cpW`RsY&G zYkGQi&FUWMue)oWenJOaLIKKh&@k8#5D*9u<*s`A^|`wpeDn|y0kY5#Xkf0VEvtuv zvyF*^gAI$Po$YxxlFOQO({MN9X=d4KX~uy6HRFR~S#Z3q1=~k|tfM8hQZvvOtavS% zmFtbQOF!LRt8yaV%6T=K&p9#6)wlc`??SKkjc>zok(WOLuTI6#fp5&;GNWIQ1>a{c zrSf@mx`c&dy{WjJ@;!Q!pI(}S$g^75-?@oNrRL*XUqJ&}0t94fQG@k+5vNa5jIvf_ zV*@DNQ$-Fwy=Esvn?XE(c8{=|4wFA+MJyu)(cuqVwrn`D;d=_E$|#k3>M@YCcXAzz zW2k*bUCqby>XkfX03s{uacroqKlkIU&)vo}Qi6Xs_h1WJ6~U`3{Biea-Rjm(lNxI8Vhl zvtjQtfj-+YtF*WuR~cI7t>&|-0U;rV=ncB2zXlV3yUDU|0f!*kKfCZdwmR8frf-un zwa$I_f(Fq1R6KVt#uPI9mBN1ZpK{!D9{fynbzXnhnKbr^vL4sBUS5nu39P3rYzSlLrc*Q@(o3W+oGkm6YEcvgW1Xkx@z%aXSw z1l(*1J>T@^fN)ECzOKrqxFoZBKVxm>m#}(Iz9_2U`)5U)KWu-zZ#npz&UCB1pC;HX zu5EY8mRDAM%yw?CEz8$RJu3H-JVG)yVq)isiL0^NA0?B2Bt48jS`SjBUDO)bTyY$? z#OdKwIRFV2e9KpTPY90*D*pT&3w4m+`u{BpA^H zUsgxUYmmy=hueqzvDR%(o{Q!K2gR-Fj<>bf?x?7^=0K(Kxc3S~Zu)j5QlChJRF|9l zzUJwiEwmgzVQ*bo>x0{IhaYp7LO!pach4%V-~4L-u5#UmQsvyJ9yIg;`0Z%t1#B(2 z2T9a}fTc{NVPONeQo*F~oD%C-ZzgGLokH$zmPac!ma&NCio0SNg5OVDa(9zi1U`v z)0VFVT^2yy7yc+T14UYy)85SDDLywTvbSqPz;66)IIGjRGLRAPcE=h2hk2cQ0$N)C zBNPww*pL_JWWGo&=aIB?EOt4qmSE?}sJddMvq{Q)S%%tTa@j{U_Uyu&QT1a*sn~?l z>td;pgyqXm!UPt+k0H&MDTZuMcm!{sKX&W+^$tTDb}2qcbeZK-&w)7GhF)Vi28Qxw z0Hz^B%6S!|qK#4>n|r0Ue(gheh1*7GV&Wf{12n#+`rA%M%bM95%tt<`Es#T$~yP;^`4cl86cHo%fQ@@GpJh3GtyXDg)*4a0Rs}9BFNRPGk#!z#v zF(Iduoywgj#);P3>i*Ia8|a!D86vbm09L9+3DaV52%xyjsObvI-Am`n*4CbFOUd?& zG)~}KD^s3&?QIMFYVusqAz(dxeN25h+SUl!3CPMVWFA2%vC(nrS(rj?0(1?36;7Kf zSv9CVnV1JW8>ADqYoR2fhnL`BmrB;$J4>16sXZ%l6JX*B8}Q?BLofw2up_TmfHm^I z1|=DD`}koQ4xW*I>IkAUmBa}Y`|p%=Al;2%w0vQ<#8#bWlgP)||C!(^5=#_;w?Bj7(;G_~fmZ>cvs-J_ zfx`atn(+{m%!vhl{>ASU^uLuI5Cbjm)$TL9ubK(A6qx#Y08K* z*H8f$G#93?Cz5j1zc*XsTW7Ik*4$ajN`^Vg|J%<+I*whJu%sGnTR_nr9>ax&SyxLb zwgny)SgV~VJ2Z7w6-vd-h?|vp(h|vgz}LsQ$JbZCWalFwl|$+rJ~s2L_%iDqT@~pS zUA0znaylSxTcIf5BgrTPNmPEO%84tp_asF?TKmX(VXMhL{m5*jIRCfebEgX)@0W8+ z|6rE*Ba^?=5D|Jt)4iNIkY#zyO9DXTHOK3 z6qt|#wcRpxB9OY1fhtqX|K^JM-ni14M* z6=jqj{y2;r@Q;YO+o%n-4EI4r{1-!IUg0t75m@VxO)DR6&~R2*D^t4W(u1%WX^&?#JtVP;u5h^ zl4um)Fe#s45zVzxj_(XFtuA%}L62Vmpw=)%tV3#4fD8spcB)JkP#YpC{*{|S@*KS! zACtCa&$t$Azqe-KgUPB%rOCn__)lz_?A96JQc{l36;qBc%B>Mq=7b|QGQ=S03uZN_ zM+Y~Ye&J&wu0L)d5i;yiQHIx{O?(!)z(#9Oq`nW@;5sp&E!(NWS_C+&us*D6xnRYd zic@my@aH>aOaEu~!NkZABJY3rJyv?On6)Z2z2w?$vdqv!40TNSr5#Mv(HAMqtTh9e%pu!i>ZlJ=~ z(KI-BSp2mJ@xBDxR!~$7!J>or%yyX*r1<}n8dCgUsgcOR+JL2o!1z~ct8#m3oW7;b zEOHNjHLiWs_&|{pi1Ne{62NFM^6&tL$m4Ub&_ILs8H|F{fK`2^$C|CFVy4~Xyakcl znypP1H+YBME9|i&{<}349_$)^cU5|{vG!A))I7Uk9D5Exi0D?-pfzYwJOr(ql7MPO zPvE_3MK9nPGaT_4ZDXQvM7ftBwD(Lx!ef?}KzbGEV7WvntN<`oR|}>1ZwIa;As(Z+ zt(pv*v}T(#5E5jiMie{S#CLzDN7gL2wTsZK^v~;S8o%hpVl%HOVvz!xR>w(hp6o7y za2a*%Rle+*w3wOZ$4tv>!+x@yWo2qakr0IP*m`FG8@s#~(gj`?lL?aR^ML0Nm zr3Da1U4z(Zdrc*B@Z%~`P%CDS3IAqLeIQjpq#$LL2ir6@jNRHsk$~SD<}JhP<}Jb# zx)BBTj#4(NJT0bN=|uYp84D;nL7qm@;i!veYj5w22LZ_}+1UHS&BW z`F}i=fD56AP>5}XNa>KiI(XT#a-B2*^?%pR_2(pSD;RSAe93fD6<&#{yqIk9SqGO% zht|ZKTI&Xl+}5PG$VpIDz|jz(8}VTXn&{N^zhM6#NFw9Uq-`*jnXF9qpw$c6O}18+ZN#^Usn?KI=Snn>GRqEM}(6bs%tUlK)u|6QXeAXATJBI7ZD`L^IVdNX?{)bfv*GT9HQaumLGllP8$Z5g*vno+uw@Ez_%K5XjqAB!mut8^lUf1_u~fHw?P; z$OP6(OfY1;6fww6_xFLyV6Z*Es%wDC#s*ba>9cvsaGbk1nfn#A9Rk^iF-@=s@Qktd z@r`wBccSp6hzX%^^RBPY1At61hd~G=TWoVKt%E0Yas{PbZ`4l zvP@M4eHgFe-B0q7P`Awb5Q*Gcp42h|Egp%;jzEfIX|3YKf)3ZpmE58XZ-8)kHc}Zr z5W;t*3Lzfr6oKUS)5(K|FEb)Q1>ds(zeHsclazI^ht3S382l1rTOOGJiy=vYL=R`X zA2ZzYi$T<+N{|MFeG4};{(E_7R5(JqM)P|=x{zF8fJ*ozHZ<4YB*9B>bb^5I)$2#% zZqGz}NRsRkY>0!&CO;)7e<|T>qZ?z_#e*Bo7I;)hl+mKNs>myJXZ2U+bg4>80e{Th zp1FxanIeIX9uIrVbRU5mH+ue+Bf{0zj7*Xkcz(^G*6d0ZOf?B*>5=9! zI~vo)1SW#=P;$e-=9p*={`l&yUy@57;qq|lvRK!wB~qi+8SBg{bP{MxVMRluHU86} zu)%^>L2^^{K2u_23L%}ua!Pb6#grO6L8b@r2x3iGuoWeW!YG}b6L#nhq5P!hCxu5x z$+xkH@vM#RynD?s?>4?`(znzmrr_~y%=ns_)|lt-;A?73mhwp8jfh3r5-C#H8*KON zd?NkS9T!i}E=|FYgsv5G&Fks>N*cu`D~tFwTmpPQ*`h;Va^#U%P$ z(dJoGOwpC5LQtk&4w9c~`j1L$^?hIq_nD`i;`DmzLilrGE4h`~oL(X)rj@}YnM_6_ zr)5F3I6;{rhj|HiWeNpFwBqFPO8aK513Vkop+F3agNRs?(Xc07cDB(mpvCaf}f(%&cU?8`x0W;K9 z+Lf&&YSWPLB)@P@xt}sA)Z40*b3tJ|IrpSNBQ*HuBv(AgzfB8?JY=^y@E~!i2x4wl zG|0b=#A}I$<|!UWt~PM`o(`DAcdj_XdK4C7E2hu4rIyG@4nw;>U4I+K5XK9)?_agAa^j!ulQVB^_u^%2vT5ii7Xqe~SJWkC0sagzK6Y?J(JydV>^|FTI3y+S=rcy*EYj4_5`DTPQT=jl` zs<|*+r#bO0*eL59%bAujgn_C|;Im=&)CkeQ_MetDkpA8#TO&YzSU2o@eI59Hoby=i-pkmM9nw!Ry z;+mKHMEGXZY#UZHM|Bf!<>*P+9(a>+7k&p7yRnL;q6xV1vcz>b4;=7cUDiM z(^Kz!4F+>V*PqJEW^^G8&7!4*WlVupF*q8h6W+xZt|*?6-~@`ZtYW=AU*$|@iwx{& z?=kf+0be)S&$)DK8Vh!Eid3E^#+l6+j$itZT~FLMINR=jeQekUpX8HJgXDE>I5*-) z72+vx_EUbq>G)4qow8%mK|gDXM`67o`pQf^@(V9=4_+Db1Z9gpHoxt}$BDS?eJP`! z^J2b^x!o?B`H5Q#`Fa~ZtcOfTS~NeeZSD-wW~$;+S3V-n7Q6Wp_jozGST*?d4I`Al zg>)hIjv;|Wo11R!9_Jrr{ivA=k(sp}$2Fdps(8==T!{mwLgIfIgHb0KH&oR02NsG@ zYZPZ8RJQ&>&#@Vw1~T$Sl@aYu5n;JG`HD8UI4-$0=#W3icb>@&g(#F(kI@HFOsRS#IF~;nwJ#o74=zn5v#yzQp=_iu*WH{Ms`@Ukgy|q$6}XQ8roc`ZD@W4BIsW>$h4+J^aG@bNHqC+K_Af!#jYP z^5baap8Klsqdi7UdGv1wBk^UTebn$uR5nVJ_8&S|!hy`vkzsNT!N>r4crU)^r zGzBN=NHxSahncvNbVx>VyGg~6lHO4|aqo%ypK$P$sEu5|268Pdz7i+qy>q&&<$Tgz z`wV?`&crycan0d#%JgVoQpx+6q8R6&!p4uTGCwrn6EAZ|E>3lvVoeVN^Y!>bY z<^-o9lr{J@?~9n5T)0Nks>(vs1^{1Tn4WRn5;zV@b%g_&yY96He4u!OJ*a`DW?ola zW45ayCZ7wQX~_w9=bY^C=lcS>?7v14tdJmMM4niFnqhizexR^z;2hju{N zTuOxJIxzjr#eIDStH|d|duFu#5ZHpmVEI|bP>GH~)`4`TX8 z&ezH_%kTo}J>m_;aAp6@3PJ}D{ZzCZ%0wR&&ae<7SqvrwztABqQsy$LMc~%OBL?O6 zsTT=XaxA<`r{tz2q;M2KFB?lwj=jw(w^OEt5~enCQU^N7jj`yxUWm;;i&>}H8W9iUd$ANdLn6bAt<@P!WB5NJ) z1_~Nm)@YL>n3oDm>xYn5ioOqw^m02{iy5%Z(Jc-~>A(2C;(CjaeKTXBsgXJiBD-E^ zgZ;5k|C)2lj;cNZ9Ew7oLy?-YywEQdW)nnPVi_*BkqDH2_ZHiV5PL|7!Krv9$L%ad zf-i7HaD@!aXzUSr5)kusRv#F@_0RX$oVwM^4^KOy7)UO*^&R|%djmIGp&gTt*4(sF z+x9Mx(T(#2@C?1av>R@v`Z2yD4mCN|_RnrLm~>aOH5jo17@;4zOpHO2R2ZQer-NuR>Tb<&k4UpZk^?)Vp37$Z@cWnCHQXQ&1N(Y zHCbYp#tGYM4xX5M$*i6_+y0mlvb+y5ym>cMzDSC=8J2#o+5PVOsJ$P9*i3K;T`sCo zQ23icVo1LMqP2L;{GI5-dLKqH<|s7+YfX)@c+~u-VMrE;c+a`3$LDry>}XyKzpZ+Z1|+C>%wD^ZyUFT` zMT5Kqaj$P)PCxlPj=OEe?0!)0slx02!zZk+R_`=RktBWh4i-a<8s0AAi`$EE&juYh8Y&?Pipu zhof1`C&meYsopqfHW0#*HlQHctx$!IGTEZ!3 zU{Y9j=|>bUs>z?iad=l+N2-7hTJkLbUjcMEu8@2o^p+W0bis^#zCRkA8`Dw9M%}-a zUma-a`5W>>Dd$~yuo~e>gxMU}vZREPCdbo@Ey@%jFBUa&P>VB&rw_3~*a0APs^!oO z{Zeku=mZ2Tpd_~K^)SmEQ> zVSO>HVrye?cnG1tNVgJxh>efmNKA*Fy>__YreehbC3pj=85fXY-6jD&UnN80OfsQ_bgo_jT$x^AbOCZ;hL&QP& ziv2-`2w4IN#g4bGCLcRI3H^!;A_K|F(sHpC7ps+(``KOc&!C}>ZsB{Tf1>&GbVT4u z;m;w6F!W7`7=60n@!iBB1c_iX|IiZ&Z)E0#RXE_pG3Eg&|02q-1UO?3+&}g$x`QnW z9}n1~P{_*+5)aDvAoazFT@tP1_5DqN$V;@jI>F#kiC$%um1E^8+PSlwxc)ZFh}g@e zsQ9(yIg=CTSJiBypnIg$*NNpsxW;U@~Rr-Mk^QSz-RzpO#z!*T)adN{c?M6jU23qTu^SoMb$wPkEUV zE0iw{yd}iy?HL-y6gAMTA?|aFrJ7E~f&wf5&a4=<3*AwVYIv2ncP54{fZ(tkAIRhd$oyI#`j9&kSCIoD zGv$YSU2a0FUNW10c_F%-*$Jdr{XrsK=W{3heuS2-JrMlB#6Us}+VmUi=A!fOX@ zSQodlsafR$vfH-AMq2O-;-6rpCnvm2&Q2c*n1wNOoSRYwO73lMdQ$OF-cz4%H~kz! z_B4%lVbNRs{sC?^T3q+-^P69H-rjB25Y(uXxXs~WLE5dnoJ#AIs0x?|Ys!U*Heznp zUx^Ya!+UL>*I~NWNe8x5UVpV$?M}X}*ZC~lQz8SlgXAH-=TI=5Hl<_%`$f1fICw6M zSr2};H499cOO_34)J!df8{9o{0ZbFvx7itGG)*eqD)p-J)+VLNs`WPRF{z6h^{&NC zY84oA!`7p4N=mH}I^4JWDuO}l6Ac+xda)g~JlSnV?M4};8^xauT&Q=mscu=Eys9mD zpS%Ds1CErN8?EV~+v4@y#z*m^5oN zZsK@`GUuLo?s*E4INhxvw|-u}%T{DxC4KIwfHg?(S4zFI5q}ieIpF$R5U81T%dQBt zwy6BH^t|&5NR=CJ);1mENP7{Nqs}Qhlg&75-3J{%MMQlfQSnU3n|2%6HjnJm%6s>8 z3Dr7}qJL2hqbAPouoCn-c;fs0++A9|SjE#m>dEB6N)1PUCxR3onVxHAPI(~IuQCDq z4Tl{Wu8ha9VXfJ{6S8ZV&FOax?C>bytgvl9qc`tCz`4Ay4>$O|O>Tt_%Y%S_xj`Rl zlFKTM%|&5Vb3!1zTbnigK3}RUIrGzF;d<6J>%p!XBmrqDuJrY1#=z*zdX|xz&*lAu z<%+*oaIn64Ob+uB3oozZcVJvyPk}rt-QP%cp#nWt<ixr`u3@@8sB<0W3 zPpuo2@0UX@KY)i*heAVRPfq+(b_a$QTr~DhIlPq~_bV6{!lfUDC>F~oVN9XR`rK+Q zz14P&+p&}e#G{`upqugFa|2%iTP7xr`&g4fHCnwzR1dPntNsH;V)lvfwcd zMkwhG*u}(=sX$uHFJu9zq0<6}n4THMf^gG07FJ}FbB&|q2?(2|gLB)cF+$7cWbx+$ zPT30VL5AifM2QSyJOUNT6eO4=CB^{}Vs_))P9Fn*U))gxO@_Ybj*zAl1<+kd38_W~qV(xDg@9qx zKZ=F0CGYvZlp{PPS6?)pgA*}(JtdUQ6{e1Py%4M&Jlutw2U>&#dK6xTT)$q_ecFIm%XjZJ-T&c9pW z?AUknxfwc578K@saF9&kFHc}?Wj3ZPRu4{*Ne4e>hO&WO1(YaxLc%_998MAaFOi&^ zDp`cg#raPVU8PbPh*ye96aPJ`iEfTr;Xa4QF12Cca)v4H;HYMa z$3sQ#Sh#)XuNpq1VG%ae7mAftyXBE}r&48F3H%U-bZ|b{t{7`V^wD5=41FDL$k-fs zf&@$#lxF$uzlK}b^5f)1=t@$|QOyQjc&!)`T3hbVDMQz%h6aB(H2JZ4-^IrdMXE4K zK!PjeEqJ(sEJS(qThRhyrWT*4?&J3I!tjpcci#gtI}z!^T(JdJs}i(S88P1)HNv&P z|Koh|+=YrgKF#vpLIM{v6Xn*=Hw(eoYl_9~h7wi%zo#C8@_8lP*`J@i?xgeMI}3AF z!u1GsK4{5_iLSr?9*}itXuq|@7#EUAw`8@8nD|fvl2)VP*-;ZKyXpAjs>r(>Unan+ zpnyQEf|;HEbB!S_jIZPTM!EPwl@yZ-!*pnnPDKtnF-#}p{02}y@TNzLs6{u`EQTSY zo6PmXpyiu$OC$>rHp)35V|N*Vb;Cz@E9qV-Lv_PrU7%nVYBjC?(e;A)Eh{i{swd1u zxc338Mds|9E_6+h#ON2ro>)JN*b@9(Q6CoHbv?gX!s!G5#R^I%b(=r!nL9WL>FJXer^BDy3OBKQBbpI6(0|^TWV5PPc+N%sE(-@KaY2ZhRN!&-XYQ>Rh# z9*;wo34tFx*XgK{Tgxa*y)UU~2b$%0r@xY)d|APlyOv|QGvSPpFJs=nKb_swT}>w$ zmTP=Pw>aIxJ87;w2YQN+5{2oQ-tt_MJZIylv$MD25~uZjCp|K!F9kF=_bEoFUQ%d; zcYorS|B?ka@{Z#pt^;B0!g%LzUcK(Y%uv%m&JOKWV8_gTMs;2f z1hlOj1Zgk_@Wg~9-xt+ef9SBP(9w1vEZ10C6_HbL$^C884qvGQYu*Of4*V~&&&l(h7<*CWM8s=&V=vuTbV@pX65(25)NhVg%0T*f9@q( zv|u{8m;e07u6CUr)a)QHKTIC2S2L10+T8_u_~vAtK3SR3ulwA;_c~;Wdv8OW#QC$1 z(TwFqcd=)z6DYmSJ%VUbR}bwk*szqUq27|E9IlIlvIKKGUYvvibK~Iv*#n{`(vvqU+RFV>I!n$Pfc_BJR>Un+h5kw0)kdGKpO zAtyfOP?%t%TQ_-N;8Zn41RHtRqtoc7OJBET^Uab3?E0-50M@|T>GZ^*!o$bc@jctJ zIJ{LuASE)zxr|{F&z6O=gut&Ia=l`l(J5s|Qh15Sqav^rJIJS`p_EeE#hT~9=COR3Mth;haj}`RV-;*aI=RzcY zSI`-}CyA?SRXj|&mlg7#(&tEDxld*?*84LYMB+h0VMG6auBRpqPjV9dugVY*gkaVG z`}jjzCgDy=K^Ze7LC)YNxlOS`8FT$hc1*EAWr_bwLQS(mMR_ENB;q7VgGsnVFiDuS kIE|j<{u3SPe}|Gu>Y66NhK2iwg8=@~LP0=?H~!Q5A3sn>WdHyG diff --git a/policyengine_uk/tests/test_excise_quantity_projection.py b/policyengine_uk/tests/test_excise_quantity_projection.py new file mode 100644 index 0000000000..304828a361 --- /dev/null +++ b/policyengine_uk/tests/test_excise_quantity_projection.py @@ -0,0 +1,487 @@ +"""Alcohol and tobacco quantity inputs grow with the OBR's clearances per head +(#2596). + +Each input declares its product group's index under +``gov.economic_assumptions.indices.obr.per_capita``, and dataset projection +grows it by the same ``yoy_growth`` series: the growth in the OBR's clearances +of the group (October 2026 receipts Tables 4.11 and 4.12, after the March 2026 +EFO's 2024-25 outturn) over one plus ``ons.population`` growth. Dataset +projection grows household weights by ``ons.population``, so the weighted +quantities grow with the clearances, and a duty-rate reform moves receipts +through the rate alone. +""" + +from functools import lru_cache +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest +import yaml +from hypothesis import given, settings +from hypothesis import strategies as st + +from policyengine_uk import CountryTaxBenefitSystem, Simulation +from policyengine_uk.data import UKSingleYearDataset +from policyengine_uk.data import economic_assumptions +from policyengine_uk.data.economic_assumptions import ( + extend_single_year_dataset, + uprating_input_fingerprint, +) +from policyengine_uk.model_api import Scenario + +pytestmark = pytest.mark.usefixtures("cloned_uk_tax_benefit_system") + +GROWTH = "gov.economic_assumptions.yoy_growth.obr.per_capita" +INDEX = "gov.economic_assumptions.indices.obr.per_capita" +POPULATION = "gov.economic_assumptions.yoy_growth.ons.population" +INPUTS = { + "tobacco_clearances.cigarettes": ("cigarettes_per_week",), + "tobacco_clearances.hand_rolling_tobacco": ("hand_rolling_tobacco_grams_per_week",), + "tobacco_clearances.other_tobacco_products": ( + "cigars_grams_per_week", + "heated_tobacco_grams_per_week", + "other_tobacco_grams_per_week", + ), + "alcohol_clearances.beer_and_cider": ( + "beer_litres", + "cider_litres", + "sparkling_cider_litres", + ), + "alcohol_clearances.wine": ("other_fermented_litres", "wine_litres"), + "alcohol_clearances.spirits": ("spirits_litres",), +} +GROUP = {name: group for group, names in INPUTS.items() for name in names} +# OBR clearances by product group, 2024-25 to 2031-32: the March 2026 EFO's +# 2024-25 outturn (receipts Tables 3.11 and 3.12, column C), then the October +# 2026 EFO's 2025-26 outturn and forecast (Tables 4.11 and 4.12, columns C to +# I). Million sticks, thousand kg, and thousand hectolitres of alcohol. +CLEARANCES = { + "tobacco_clearances.cigarettes": (13578.199084269218, 12224.568453264392, 10962.756413840059, 9853.5452739735829, 9042.8621730867635, 8222.5673694138313, 7677.1786518547042, 7165.065963684312), + "tobacco_clearances.hand_rolling_tobacco": (4028.4786869103718, 3338.0207129187247, 3112.6781722236924, 2886.8250342433803, 2649.2381871262742, 2482.242556381048, 2384.6513795861088, 2273.4315215751089), + "tobacco_clearances.other_tobacco_products": (696.58878604045401, 601.72119516629675, 570.78371235846851, 547.48868265410397, 502.43014658034485, 478.01479115950667, 460.50663098211788, 434.81308397816122), + "alcohol_clearances.beer_and_cider": (2168.2921615793966, 2108.7303043893562, 2069.8474469367434, 2027.3458030333193, 2012.6406194934405, 2001.8700847628832, 1982.9201100206114, 1962.5485885566063), + "alcohol_clearances.wine": (1619.9398031719702, 1569.041217813324, 1567.7826624097067, 1540.2033043369458, 1530.2485351419407, 1520.7475110216285, 1505.5439058920508, 1491.4031570887553), + "alcohol_clearances.spirits": (1289.5543778979199, 1272.4562775587838, 1262.3692523269856, 1241.1318815407608, 1235.8250379612948, 1230.5012764962146, 1218.4826638265554, 1209.5688671337984), +} # fmt: skip +FORECAST_YEARS = range(2024, 2032) +YEARS = range(2024, 2033) +ALCOHOL_RATES = "gov.hmrc.alcohol_duty.rates" +TOBACCO_RATES = "gov.hmrc.tobacco_duty.rates" +DUTIES = ("alcohol_duty", "tobacco_duty") + + +@lru_cache(maxsize=None) +def current_law(): + return CountryTaxBenefitSystem().parameters + + +@pytest.fixture(scope="module") +def parameters(): + return current_law() + + +def growth(parameters, path, year): + return parameters.get_child(path)(f"{year}-01-01") + + +def index(parameters, group, year): + return parameters.get_child(f"{INDEX}.{group}")(f"{year}-01-01") + + +def household_frame(year, households): + """One adult in each household, with the given household inputs. Each + household is a dict of inputs; ``household_weight`` defaults to 1.""" + count = len(households) + columns = sorted({name for household in households for name in household}) + return UKSingleYearDataset( + person=pd.DataFrame( + { + "person_id": range(1, count + 1), + "person_benunit_id": range(1, count + 1), + "person_household_id": range(1, count + 1), + "age": [40] * count, + } + ), + benunit=pd.DataFrame({"benunit_id": range(1, count + 1)}), + household=pd.DataFrame( + { + "household_id": range(1, count + 1), + "region": ["LONDON"] * count, + "tenure_type": ["OWNED_OUTRIGHT"] * count, + "council_tax": [0.0] * count, + "rent": [0.0] * count, + "household_weight": [1.0] * count, + **{ + name: [float(household.get(name, 0.0)) for household in households] + for name in columns + }, + } + ), + fiscal_year=year, + ) + + +def situation(inputs, year=2024): + return { + "people": {"person": {"age": {year: 40}}}, + "benunits": {"benunit": {"members": ["person"]}}, + "households": { + "household": { + "members": ["person"], + **{name: {year: value} for name, value in inputs.items()}, + } + }, + } + + +# A household that buys every product: 5% beer, 4.5% cider, 7% sparkling +# cider, 12% wine, 40% spirits and 5% other fermented drinks, and pays £16 a +# pack for its cigarettes. +SMOKER_AND_DRINKER = { + "beer_litres": 300.0, + "beer_abv": 0.05, + "cider_litres": 60.0, + "cider_abv": 0.045, + "sparkling_cider_litres": 20.0, + "sparkling_cider_abv": 0.07, + "wine_litres": 80.0, + "wine_abv": 0.12, + "spirits_litres": 10.0, + "spirits_abv": 0.40, + "other_fermented_litres": 15.0, + "other_fermented_abv": 0.05, + "cigarettes_per_week": 70.0, + "cigarette_price_per_pack": 16.0, + "hand_rolling_tobacco_grams_per_week": 25.0, + "cigars_grams_per_week": 5.0, + "other_tobacco_grams_per_week": 3.0, + "heated_tobacco_grams_per_week": 4.0, +} +QUANTITIES = {name: SMOKER_AND_DRINKER[name] for name in GROUP} + + +def test_every_quantity_input_declares_its_product_group_index(): + """Every alcohol and tobacco quantity input declares its product group's + index, and dataset projection grows it by the same group's series.""" + from policyengine_uk.system import system + + folder = Path(__file__).parents[1] / "variables/input/consumption" + quantities = { + path.stem + for path in [*folder.glob("alcohol/*.py"), *folder.glob("tobacco/*.py")] + if path.stem.endswith(("_litres", "_per_week")) + } + assert quantities == set(GROUP) + for name, group in GROUP.items(): + assert system.variables[name].uprating == f"{INDEX}.{group}", name + mapping = yaml.safe_load( + (Path(__file__).parents[1] / "data/uprating_indices.yaml").read_text() + ) + clearances = {key: value for key, value in mapping.items() if "clearances" in key} + assert clearances == { + f"{GROWTH}.{group}": sorted(names) for group, names in INPUTS.items() + } + + +def test_strengths_shares_and_the_cigarette_price_stay_as_they_are(): + """Strengths and draught shares are ratios, and the cigarette price is a + price, not a quantity (#2596 leaves its forward path out): none of them + declares an index or appears in the uprating map.""" + from policyengine_uk.system import system + + folder = Path(__file__).parents[1] / "variables/input/consumption" + others = { + path.stem + for path in [*folder.glob("alcohol/*.py"), *folder.glob("tobacco/*.py")] + } - set(GROUP) + assert others >= {"beer_abv", "wine_draught_share", "cigarette_price_per_pack"} + mapped = { + name + for names in economic_assumptions._uprating_index_map().values() + for name in names + } + for name in others: + assert system.variables[name].uprating is None, name + assert name not in mapped, name + + +def test_indices_are_positive_from_the_2024_base_year(parameters): + for group in INPUTS: + assert index(parameters, group, 2024) == 1 + for year in range(2024, 2040): + assert index(parameters, group, year) > 0, (group, year) + + +def test_each_series_is_its_clearances_per_head(parameters): + """Each year's growth, to six decimals, is the clearances' growth over + one plus that year's population growth, the growth of household + weights.""" + for group, levels in CLEARANCES.items(): + assert growth(parameters, f"{GROWTH}.{group}", 2024) == 0 + for year in range(2025, 2032): + published = levels[year - 2024] / levels[year - 2025] + population = growth(parameters, POPULATION, year) + assert growth(parameters, f"{GROWTH}.{group}", year) == round( + published / (1 + population) - 1, 6 + ), (group, year) + # After the forecast the clearances hold flat. + for year in range(2032, 2074): + population = growth(parameters, POPULATION, year) + assert growth(parameters, f"{GROWTH}.{group}", year) == round( + 1 / (1 + population) - 1, 6 + ), (group, year) + + +def test_cigarettes_per_head_halve_by_2031(parameters): + """The October 2026 forecast puts 2031-32 cigarette clearances 47.2% + below 2024-25. Population grows 1.5% over those years, so clearances per + head fall 48.0%, and the index stands at 0.520.""" + levels = CLEARANCES["tobacco_clearances.cigarettes"] + assert levels[-1] / levels[0] - 1 == pytest.approx(-0.4723, abs=1e-4) + population = np.prod( + [1 + growth(parameters, POPULATION, year) for year in range(2025, 2032)] + ) + assert population - 1 == pytest.approx(0.0154, abs=1e-4) + assert index(parameters, "tobacco_clearances.cigarettes", 2031) == pytest.approx( + levels[-1] / levels[0] / population, abs=1e-5 + ) + assert index(parameters, "tobacco_clearances.cigarettes", 2031) == pytest.approx( + 0.520, abs=5e-4 + ) + + +def test_weighted_quantities_follow_the_clearances(parameters): + """Two households with different weights: dataset projection grows each + quantity per head and each weight by population growth, so every + weighted total grows with the OBR's clearances, and in 2032-33, after + the forecast, holds at 2031-32's. Growing quantities by the clearances + themselves would count population growth twice: 1.5% too much by + 2031-32.""" + households = [QUANTITIES, {name: 2 * value for name, value in QUANTITIES.items()}] + frame = household_frame(2024, households) + frame.household["household_weight"] = [1_000.0, 250.0] + projected = extend_single_year_dataset(frame, parameters, end_year=2032) + base = projected[2024].household + weights = {year: projected[year].household["household_weight"] for year in YEARS} + population = 1.0 + for year in YEARS: + if year > 2024: + population *= 1 + growth(parameters, POPULATION, year) + assert weights[year].sum() == pytest.approx(1_250 * population, rel=1e-12) + for name, group in GROUP.items(): + levels = CLEARANCES[group] + published = levels[min(year, 2031) - 2024] / levels[0] + total = (weights[year] * projected[year].household[name]).sum() + base_total = (weights[2024] * base[name]).sum() + assert total / base_total == pytest.approx(published, rel=1e-5), ( + name, + year, + ) + assert population - 1 > 0.015 + + +QUANTITY = st.floats(min_value=0, max_value=1e4, allow_nan=False) + + +# Each example projects a dataset through 2032. +@settings(max_examples=8, deadline=None, derandomize=True) +@given( + quantities=st.fixed_dictionaries({name: QUANTITY for name in GROUP}), + weight=st.floats(min_value=0.1, max_value=1e5, allow_nan=False), +) +def test_projected_quantities_are_the_base_times_the_index(quantities, weight): + """For any quantities and weight: dataset projection gives each year's + quantity as the 2024 quantity times its series' growth compounded, which + is its index to the index's five decimals; Core's uprating of a 2024 + situation input (the other route to a later year) agrees; the weight + grows by population alone.""" + parameters = current_law() + frame = household_frame(2024, [quantities]) + frame.household["household_weight"] = [weight] + projected = extend_single_year_dataset(frame, parameters, end_year=2032) + simulation = Simulation(situation=situation(quantities)) + for name, group in GROUP.items(): + compounded = 1.0 + for year in YEARS: + if year > 2024: + compounded *= 1 + growth(parameters, f"{GROWTH}.{group}", year) + stored = projected[year].household[name].iloc[0] + assert stored == pytest.approx(quantities[name] * compounded, rel=1e-12) + ratio = index(parameters, group, year) / index(parameters, group, 2024) + assert compounded == pytest.approx(ratio, rel=1e-4), (name, year) + uprated = simulation.calculate(name, year)[0] + assert uprated == pytest.approx(quantities[name] * ratio, rel=1e-6) + population = 1.0 + for year in YEARS: + if year > 2024: + population *= 1 + growth(parameters, POPULATION, year) + stored = projected[year].household["household_weight"].iloc[0] + assert stored == pytest.approx(weight * population, rel=1e-12) + + +def test_a_later_year_charges_duty_on_the_uprated_quantity(parameters): + """A projected 2027 household pays the duty a household holding the 2027 + quantities pays, less than on its 2024 quantities: 2027 rates on fewer + cigarettes and drinks.""" + projected = Simulation(dataset=household_frame(2024, [SMOKER_AND_DRINKER])) + uprated = { + name: SMOKER_AND_DRINKER[name] + * index(parameters, group, 2027) + / index(parameters, group, 2024) + for name, group in GROUP.items() + } + held = {**SMOKER_AND_DRINKER, **uprated} + at_2027 = Simulation(situation=situation(held, year=2027)) + flat = Simulation(situation=situation(SMOKER_AND_DRINKER, year=2027)) + for duty in DUTIES: + duty_2027 = projected.calculate(duty, 2027)[0] + assert duty_2027 == pytest.approx(at_2027.calculate(duty, 2027)[0], rel=1e-4) + assert duty_2027 < flat.calculate(duty, 2027)[0] + # Cigarettes alone: 70 a week is 3,640 a year. Per head, clearances fall + # 10.3%, 10.5% and 10.2% in 2025-26, 2026-27 and 2027-28, an index of + # 0.721 in 2027, which leaves 2,624. + assert index(parameters, "tobacco_clearances.cigarettes", 2027) == 0.72092 + cigarettes = projected.calculate("cigarettes_per_week", 2027)[0] * 52 + assert cigarettes == pytest.approx(3_640 * 0.72092, rel=1e-5) + + +def rate_leaves(parameters): + """Every alcohol and tobacco duty rate the duty formulas read, without + the wine strength easement's switch, which is true or false.""" + leaves = [ + leaf + for node in (ALCOHOL_RATES, TOBACCO_RATES) + for leaf in parameters.get_child(node).get_descendants() + if not hasattr(leaf, "children") + ] + switches = [leaf.name for leaf in leaves if leaf.metadata["unit"] == "bool"] + assert switches == [f"{ALCOHOL_RATES}.wine_easement"] + return [leaf.name for leaf in leaves if leaf.name not in switches] + + +# Each example builds a reform, which reprocesses the parameter tree. +@settings(max_examples=4, deadline=None, derandomize=True) +@given( + factor=st.floats(min_value=0.1, max_value=3, allow_nan=False), + year=st.integers(min_value=2027, max_value=2031), + scale=st.floats(min_value=0.1, max_value=10, allow_nan=False), +) +def test_a_duty_rate_reform_moves_receipts_by_the_rate_alone(factor, year, scale): + """Scaling every alcohol and tobacco duty rate in a year from 2027-28, + when the current-law rates hold flat, leaves every projected quantity as + it was and multiplies each duty by the factor: the duties are linear in + their rates, and the rates are not among the parameters the dataset + projection reads.""" + parameters = current_law() + inputs = { + name: value * scale if name in GROUP else value + for name, value in SMOKER_AND_DRINKER.items() + } + changes = { + name: {str(year): parameters.get_child(name)(f"{year}-06-01") * factor} + for name in rate_leaves(parameters) + } + scenario = Scenario(parameter_changes=changes) + baseline = Simulation(dataset=household_frame(2024, [inputs])) + reform = Simulation(dataset=household_frame(2024, [inputs]), scenario=scenario) + assert uprating_input_fingerprint( + reform.tax_benefit_system.parameters + ) == uprating_input_fingerprint(baseline.tax_benefit_system.parameters) + for name in GROUP: + for period in (2024, year): + np.testing.assert_array_equal( + reform.calculate(name, period), baseline.calculate(name, period) + ) + for duty in DUTIES: + before = baseline.calculate(duty, year)[0] + assert before > 0 + assert reform.calculate(duty, year)[0] == pytest.approx( + before * factor, rel=1e-5 + ) + + +# Outputs that read neither duty, directly or through a total. +UNTOUCHED = ( + "income_tax", + "national_insurance", + "household_benefits", + "household_market_income", + "hbai_household_net_income", + "vat", +) + + +def test_the_change_touches_only_the_quantity_inputs(parameters): + """Differential against the projection without this change, which + leaves the quantity inputs out of the uprating map and out of Core's + fallback: every other stored input is the same in every year, the + quantity inputs differ by their index alone, and outputs differ only + through the two duties.""" + households = [ + {**SMOKER_AND_DRINKER, "remote_gaming_net_losses": 500.0}, + {"beer_litres": 50.0, "cigarettes_per_week": 0.0}, + ] + frame = household_frame(2024, households) + frame.person["employment_income"] = [30_000.0, 55_000.0] + frame.household["household_weight"] = [700.0, 300.0] + mapping = economic_assumptions._uprating_index_map() + unmapped = economic_assumptions._unmapped_uprating_indices + without_map = { + key: names for key, names in mapping.items() if "clearances" not in key + } + assert sum(len(names) for names in mapping.values()) - sum( + len(names) for names in without_map.values() + ) == len(GROUP) + with pytest.MonkeyPatch.context() as patch: + patch.setattr(economic_assumptions, "_uprating_index_map", lambda: without_map) + patch.setattr( + economic_assumptions, + "_unmapped_uprating_indices", + lambda uprating: { + name: index + for name, index in unmapped(uprating).items() + if name not in GROUP + }, + ) + before = extend_single_year_dataset(frame, parameters, end_year=2032) + after = extend_single_year_dataset(frame, parameters, end_year=2032) + for year in YEARS: + for table in ("person", "benunit", "household"): + old, new = getattr(before[year], table), getattr(after[year], table) + assert list(old.columns) == list(new.columns) + for column in old.columns: + if column in GROUP: + group = GROUP[column] + ratio = index(parameters, group, year) / index( + parameters, group, 2024 + ) + np.testing.assert_allclose( + new[column], old[column] * ratio, rtol=1e-4 + ) + np.testing.assert_array_equal(old[column], frame.household[column]) + else: + pd.testing.assert_series_equal(old[column], new[column]) + old_simulation = Simulation(dataset=before) + new_simulation = Simulation(dataset=after) + for year in YEARS: + changed = sum( + new_simulation.calculate(duty, year) - old_simulation.calculate(duty, year) + for duty in DUTIES + ) + if year > 2024: + assert changed[0] < 0 + for total in ("household_tax", "gov_tax"): + difference = new_simulation.calculate( + total, year + ) - old_simulation.calculate(total, year) + np.testing.assert_allclose(difference, changed, atol=0.01) + net = new_simulation.calculate( + "household_net_income", year + ) - old_simulation.calculate("household_net_income", year) + np.testing.assert_allclose(net, -changed, atol=0.01) + for name in UNTOUCHED: + np.testing.assert_array_equal( + new_simulation.calculate(name, year), + old_simulation.calculate(name, year), + ) diff --git a/policyengine_uk/tests/test_import_obr_forecasts.py b/policyengine_uk/tests/test_import_obr_forecasts.py index e778774825..010ea59454 100644 --- a/policyengine_uk/tests/test_import_obr_forecasts.py +++ b/policyengine_uk/tests/test_import_obr_forecasts.py @@ -14,10 +14,12 @@ from policyengine_uk.utils import import_obr_forecasts from policyengine_uk.utils.import_obr_forecasts import ( + CLEARANCE_SERIES_SPECS, RECEIPTS_SERIES_SPECS, SERIES_SPECS, STATUTORY_GAP_SPECS, _sheet_paths, + ClearanceSeriesSpec, ForecastGap, MACHINE_GAMES_HIGHER_STAKE_SHARE, ReceiptsSeriesSpec, @@ -30,6 +32,8 @@ get_yoy_growth_path, infer_workbook_release, main, + per_head_growth, + population_growth, read_calendar_values, read_workbook_sheets, render_forecast_gap_yaml, @@ -498,6 +502,133 @@ def receipts_table( "bingo": "Bingo duty", "pool_betting": "Pool betting duty", } +# Synthetic clearances from each EFO's first (outturn) year, in each line's +# own unit: million sticks, thousand kg or thousand hectolitres of alcohol. +CLEARANCE_LEVELS = { + "cigarettes": (12_000.0, 10_800.0, 9_720.0), + "hand_rolling_tobacco": (3_300.0, 3_135.0, 2_978.25), + "other_tobacco_products": (600.0, 588.0, 576.24), + "beer_and_cider": (2_100.0, 2_079.0, 2_058.21), + "wine": (1_570.0, 1_585.7, 1_569.843), + "spirits": (1_270.0, 1_257.3, 1_244.727), +} +PREVIOUS_CLEARANCE_LEVELS = { + "cigarettes": (13_500.0, 12_150.0, 11_000.0), + "hand_rolling_tobacco": (4_000.0, 3_600.0, 3_400.0), + "other_tobacco_products": (700.0, 650.0, 640.0), + "beer_and_cider": (2_150.0, 2_120.0, 2_100.0), + "wine": (1_600.0, 1_590.0, 1_580.0), + "spirits": (1_300.0, 1_280.0, 1_270.0), +} + + +def clearance_table( + title: str, + first_year: int, + rows: list[tuple[str, tuple]], + notes: tuple[str, ...] = (), + outturn_years: int = 1, + start: int = 2, + unit: str | None = None, +) -> dict[int, list[str]]: + """A table laid out as the OBR's Tables 4.11 and 4.12: title in column B, + the "Outturn" and "Forecast" markers on the next row, fiscal years on the + row after, then one row per line and any notes. A clearance table has no + unit cell, since a label or a note states the unit; ``unit`` adds one on + a row of its own below the title, as a receipts table has.""" + years = max(len(levels) for _, levels in rows) + columns = [column_letter(3 + i) for i in range(years)] + sheet = {start: [make_inline_cell(f"B{start}", title)]} + if unit is not None: + start += 1 + sheet[start] = [make_inline_cell(f"C{start}", unit)] + sheet[start + 1] = [ + *([make_inline_cell(f"C{start + 1}", "Outturn")] if outturn_years else []), + *( + [make_inline_cell(f"{columns[outturn_years]}{start + 1}", "Forecast")] + if outturn_years < years + else [] + ), + ] + sheet[start + 2] = [ + make_inline_cell( + f"{column}{start + 2}", f"{first_year + i}-{str(first_year + i + 1)[2:]}" + ) + for i, column in enumerate(columns) + ] + row = start + 3 + for label, levels in rows: + sheet[row] = [make_inline_cell(f"B{row}", label)] + [ + make_number_cell(f"{column}{row}", level) + for column, level in zip(columns, levels) + ] + row += 1 + for note in notes: + sheet[row] = [make_inline_cell(f"B{row}", note)] + row += 1 + return sheet + + +def tobacco_table(chapter, first_year, levels, outturn_years=1): + """Table 4.11 (3.11 to March 2026): cigarettes in million sticks, then + non-cigarette clearances in thousand kg and their "of which" split.""" + non_cigarette = tuple( + float(hand_rolled) + float(other) + for hand_rolled, other in zip( + levels["hand_rolling_tobacco"], levels["other_tobacco_products"] + ) + ) + return make_sheet( + { + 1: [make_inline_cell("A1", "Back to contents")], + **clearance_table( + f" {chapter}.11 Tobacco clearances", + first_year, + [ + ("Cigarette clearances (million sticks)", levels["cigarettes"]), + ("Non-cigarette clearances ('000s kg)", non_cigarette), + ("of which:", ()), + ("Hand rolled tobacco clearances", levels["hand_rolling_tobacco"]), + ("Other tobacco products", levels["other_tobacco_products"]), + ], + outturn_years=outturn_years, + ), + } + ) + + +def alcohol_table(chapter, first_year, levels, outturn_years=1, note=None): + """Table 4.12 (3.12 to March 2026): clearances in thousand hectolitres of + alcohol, which a note states, then the duties' receipts in £ billion + under the same labels, which the clearance lines must not read.""" + labels = { + "beer_and_cider": " Beer and cider", + "wine": " Wine1", + "spirits": " Spirits2", + } + clearances = clearance_table( + f"{chapter}.12 Alcohol clearance forecasts", + first_year, + [(labels[key], levels[key]) for key in labels], + notes=( + note or "Note: All clearances are in thousands of hectolitres of alcohol.", + "1 Includes other fermented products such as fruit ciders.", + "2 Includes spirit-based ‘ready-to-drinks’ such as alcopops.", + ), + outturn_years=outturn_years, + ) + receipts = clearance_table( + f"{chapter}.12 Alcohol receipts forecasts", + first_year, + [(labels[key], (4.0,) * len(levels[key])) for key in labels], + notes=("Note: All taxes are on an accrued basis.",), + outturn_years=outturn_years, + start=max(clearances) + 2, + unit="£ billion", + ) + return make_sheet( + {1: [make_inline_cell("A1", "Back to contents")], **clearances, **receipts} + ) def make_receipts_xlsx( @@ -510,11 +641,14 @@ def make_receipts_xlsx( sdlt_rows: list[tuple[str, tuple]] | None = None, chapter: int = 3, gambling_levels: dict[str, tuple] | None = None, + clearance_levels: dict[str, tuple] | None = None, + alcohol_note: str | None = None, ) -> bytes: """Receipts tables numbered in ``chapter``: 3 to March 2026, 4 from October 2026.""" levels = levels or RECEIPTS_LEVELS gambling_levels = gambling_levels or GAMBLING_LEVELS + clearance_levels = clearance_levels or CLEARANCE_LEVELS sheet_318 = make_sheet({2: [make_inline_cell("B2", "Other table")]}) sheet_319 = make_sheet( { @@ -617,6 +751,12 @@ def make_receipts_xlsx( ], outturn_years=0, ) + sheets[f"{chapter}.11"] = tobacco_table( + chapter, first_year, clearance_levels, outturn_years + ) + sheets[f"{chapter}.12"] = alcohol_table( + chapter, first_year, clearance_levels, outturn_years, alcohol_note + ) sheets[f"{chapter}.18"] = sheet_318 sheets[f"{chapter}.19"] = sheet_319 return make_workbook(sheets) @@ -632,6 +772,7 @@ def make_previous_receipts_xlsx(**overrides) -> bytes: "levels": PREVIOUS_RECEIPTS_LEVELS, "cash_table": "3.9", "northern_ireland_label": "Northern Ireland Business Rates", + "clearance_levels": PREVIOUS_CLEARANCE_LEVELS, **overrides, } ) @@ -812,16 +953,50 @@ def test_small_gaps_are_written_in_fixed_point(value, text): ) + "\n" ) +CLEARANCE_SERIES_FIXTURE = """ {key}: + description: {key} clearances growth per person. + values: + # Base year, which every run keeps + 2023-01-01: 0.0000 + # Older forecast + 2024-01-01: 0.0100 + 2031-01-01: 0.0200 + metadata: + unit: /1 + label: {key} clearances growth per person + reference: + - title: Old + href: https://example.com/old +""" +CLEARANCE_FIXTURE = ( + " per_capita:\n" + + "".join( + f" {group}:\n" + + "".join( + CLEARANCE_SERIES_FIXTURE.format(key=spec.path[-1]) + for spec in CLEARANCE_SERIES_SPECS + if spec.path[1] == group + ) + for group in ("tobacco_clearances", "alcohol_clearances") + ) + + "\n" +) +# Population growth changes in 2026 and 2030, so each year's clearance +# growth must take that year's population growth out. +POPULATION_FIXTURE = """ons: + population: + values: + 2024-01-01: 0.0100 + 2026-01-01: 0.0200 + 2030-01-01: -0.0050 +""" YOY_GROWTH_FIXTURE = ( "obr:\n" + "".join(FLAT_SERIES_FIXTURE.format(key=key) for key in SERIES_KEYS[:2]) + RECEIPTS_FIXTURE + + CLEARANCE_FIXTURE + "".join(FLAT_SERIES_FIXTURE.format(key=key) for key in SERIES_KEYS[2:]) - + """ons: - population: - values: - 2024-01-01: 0.0100 -""" + + POPULATION_FIXTURE ) GAP_FIXTURE = """description: Gap. values: @@ -1011,17 +1186,23 @@ def failing_replace(source, target): # Receipts growth series: stamp_duty_land_tax and business_rates under obr, -# which link their first year to the previous EFO's outturn, and the -# gambling_duty_bases, which grow a duty's base from HMRC's outturn. +# which link their first year to the previous EFO's outturn, the +# gambling_duty_bases, which grow a duty's base from HMRC's outturn, and the +# per_capita clearance series, which link their first year as receipts do. -RECEIPTS_SPECS = tuple(spec for spec in RECEIPTS_SERIES_SPECS if spec.outturn is None) +RECEIPTS_SPECS = tuple( + spec + for spec in RECEIPTS_SERIES_SPECS + if spec.outturn is None and not isinstance(spec, ClearanceSeriesSpec) +) RECEIPTS_PATHS = {spec.path[-1]: spec.path for spec in RECEIPTS_SPECS} GAMBLING_PATHS = { spec.path[-1]: spec.path for spec in RECEIPTS_SERIES_SPECS if spec.outturn is not None } -ALL_RECEIPTS_PATHS = {**RECEIPTS_PATHS, **GAMBLING_PATHS} +CLEARANCE_PATHS = {spec.path[-1]: spec.path for spec in CLEARANCE_SERIES_SPECS} +ALL_RECEIPTS_PATHS = {**RECEIPTS_PATHS, **GAMBLING_PATHS, **CLEARANCE_PATHS} def _series(tree, path): @@ -1511,13 +1692,32 @@ def test_rendered_growth_compounds_the_published_levels(years, data): "great_britain": ("32.376814327646052", "34.051963932489258", "37.413586572144581", "38.240930947824609", "39.111850954748434", "42.419067764546902", "42.463193622519263"), "northern_ireland": ("0.38233350730688931", "0.39471664406441903", "0.40263364127921164", "0.41045270712298504", "0.41810894612429189", "0.42590403232705243", "0.43449009714292341"), } # fmt: skip +# Published clearances as the workbooks hold them, from each EFO's outturn +# year: October 2026 Tables 4.11 (rows 5, 8 and 9) and 4.12 (rows 5 to 7) +# from 2025-26, and March 2026 Tables 3.11 and 3.12 from 2024-25. +OCTOBER_2026_CLEARANCES = { + "cigarettes": ("12224.568453264392", "10962.756413840059", "9853.5452739735829", "9042.8621730867635", "8222.5673694138313", "7677.1786518547042", "7165.065963684312"), + "hand_rolling_tobacco": ("3338.0207129187247", "3112.6781722236924", "2886.8250342433803", "2649.2381871262742", "2482.242556381048", "2384.6513795861088", "2273.4315215751089"), + "other_tobacco_products": ("601.72119516629675", "570.78371235846851", "547.48868265410397", "502.43014658034485", "478.01479115950667", "460.50663098211788", "434.81308397816122"), + "beer_and_cider": ("2108.7303043893562", "2069.8474469367434", "2027.3458030333193", "2012.6406194934405", "2001.8700847628832", "1982.9201100206114", "1962.5485885566063"), + "wine": ("1569.041217813324", "1567.7826624097067", "1540.2033043369458", "1530.2485351419407", "1520.7475110216285", "1505.5439058920508", "1491.4031570887553"), + "spirits": ("1272.4562775587838", "1262.3692523269856", "1241.1318815407608", "1235.8250379612948", "1230.5012764962146", "1218.4826638265554", "1209.5688671337984"), +} # fmt: skip +MARCH_2026_CLEARANCES = { + "cigarettes": ("13578.199084269218", "12254.443899171421", "10935.71615536125", "9796.6640404179016", "8866.6130178015883", "7995.729597841897", "7564.6566739628925"), + "hand_rolling_tobacco": ("4028.4786869103718", "3526.3857151341385", "3293.6576388022659", "2892.1697163272493", "2753.8285838812085", "2585.2813720012768", "2487.166773432516"), + "other_tobacco_products": ("696.58878604045401", "669.27631225428354", "677.85087256392069", "583.39376884631531", "632.00622388116062", "635.62283867654082", "648.80354856597478"), + "beer_and_cider": ("2168.2921615793966", "2083.3332688432952", "2047.9991564911497", "2055.0663286925724", "2069.6074004097741", "2085.2376704507528", "2101.0702511760765"), + "wine": ("1619.9398031719702", "1534.8113403538073", "1521.0993234521127", "1529.4264001780152", "1542.2932290667245", "1555.708517982323", "1569.3328887285095"), + "spirits": ("1289.5543778979199", "1239.4917691306655", "1236.8062704924209", "1240.0908662025893", "1247.2437066467319", "1254.8362045261438", "1262.8339376703782"), +} # fmt: skip def test_october_2026_tables_reproduce_the_committed_series(): """yoy_growth.yaml holds exactly what the importer writes from the - October 2026 receipts tables (Tables 4.7, 4.8, 4.2 and 4.10) and the - March 2026 ones (3.7, 3.8 and 3.2): rerunning it on the committed file - changes nothing.""" + October 2026 receipts tables (Tables 4.7, 4.8, 4.2, 4.10, 4.11 and 4.12) + and the March 2026 ones (3.7, 3.8, 3.2, 3.11 and 3.12): rerunning it on + the committed file changes nothing.""" committed = get_yoy_growth_path().read_text() rendered = render( committed, @@ -1528,6 +1728,7 @@ def test_october_2026_tables_reproduce_the_committed_series(): cash_table="4.8", chapter=4, gambling_levels=OCTOBER_2026_GAMBLING, + clearance_levels=OCTOBER_2026_CLEARANCES, ), previous=make_previous_receipts_xlsx( release="March 2026", @@ -1535,6 +1736,7 @@ def test_october_2026_tables_reproduce_the_committed_series(): levels=MARCH_2026_RECEIPTS, cash_table="3.8", northern_ireland_label="Northern Ireland business rates", + clearance_levels=MARCH_2026_CLEARANCES, ), release=("October", 2026), previous_release=("March", 2026), @@ -2504,3 +2706,273 @@ def test_a_written_series_keeps_every_index_positive(levels, schedules): for year in sorted(series.values): index = round(index * (1 + series.values[year]), 5) assert index > 0 + + +# Alcohol and tobacco clearances per head (#2596): Tables 4.11 and 4.12 (3.11 +# and 3.12 to March 2026), whose growth over ons.population growth projects +# the household quantity inputs. + +CLEARANCE_SPECS = {spec.path[-1]: spec for spec in CLEARANCE_SERIES_SPECS} +# ons.population as POPULATION_FIXTURE holds it, in effect in each year. +FIXTURE_POPULATION = {2024: 0.01, 2025: 0.01, 2026: 0.02, 2027: 0.02, 2030: -0.005} + + +def test_clearance_lines_are_read_from_their_own_tables(): + """Each line comes from its clearance table, in its unit, not from the + receipts table below Table 3.12 under the same labels.""" + current = read_workbook_sheets(make_receipts_xlsx()) + previous = read_workbook_sheets(make_previous_receipts_xlsx()) + for key, spec in CLEARANCE_SPECS.items(): + line = find_receipts_line(current, spec) + before = find_receipts_line(previous, spec) + table = "3.11" if "tobacco" in spec.path[1] else "3.12" + assert (line.table, before.table) == (table, table) + assert line.levels == dict(zip((2024, 2025, 2026), CLEARANCE_LEVELS[key])) + assert before.levels[2023] == PREVIOUS_CLEARANCE_LEVELS[key][0] + assert (line.outturn_years, before.outturn_years) == ({2024}, {2023}) + assert find_receipts_line(current, CLEARANCE_SPECS["wine"]).label == "Wine" + + +def test_clearance_growth_is_per_head_of_the_population_the_file_holds(): + """Each year's growth is the clearances' growth over one plus that year's + ons.population growth in the file being written; after the forecast the + clearances hold flat, so the series is 1 / (1 + population) - 1.""" + rendered = render() + for key, path in CLEARANCE_PATHS.items(): + published = { + 2023: PREVIOUS_CLEARANCE_LEVELS[key][0], + **dict(zip((2024, 2025, 2026), CLEARANCE_LEVELS[key])), + } + expected = { + year: round( + published[year] + / published[year - 1] + / (1 + _in_effect(FIXTURE_POPULATION, year)) + - 1, + 6, + ) + for year in (2024, 2025, 2026) + } + assert _values(rendered, path) == { + 2023: 0.0, + **expected, + 2027: round(1 / 1.02 - 1, 6), + 2030: round(1 / 0.995 - 1, 6), + }, key + # Cigarettes fall 10% a year, 10.9% per head once population grows 1%. + assert _values(rendered, CLEARANCE_PATHS["cigarettes"])[2025] == -0.108911 + + +def test_clearance_comments_and_references_name_both_efos_and_the_population(): + rendered = render() + block = rendered[rendered.index(" cigarettes:") :] + block = block[: block.index(" hand_rolling_tobacco:")] + assert ( + " # Base year, which every run keeps\n" + " 2023-01-01: 0.0000\n" + " # Outturn (2024-25 over 2023-24) from OBR EFO March 2026 over " + 'March 2025, receipts Table 3.11 "Cigarette clearances (million ' + 'sticks)"; per head: clearance growth over one plus ons.population ' + "growth\n" + ) in block + assert ( + " # Beyond the OBR forecast: clearances held flat, so per head " + "they fall with ons.population growth\n" + " 2027-01-01: -0.019608\n" + " 2030-01-01: 0.005025\n" + ) in block + references = _series(yaml.safe_load(rendered), CLEARANCE_PATHS["wine"])["metadata"][ + "reference" + ] + assert [reference["title"] for reference in references] == [ + "OBR EFO March 2026 (detailed forecast tables, receipts, Table 3.12)", + "OBR EFO March 2025 (detailed forecast tables, receipts, Table 3.12; " + "2023-24 outturn)", + ] + + +TOBACCO_TITLE = {"B": "4.11 Tobacco clearances"} +TOBACCO_HEADER = [ + {"C": "Outturn", "D": "Forecast"}, + {"C": "2025-26", "D": "2026-27"}, +] + + +@pytest.mark.parametrize( + "rows, message", + [ + # The cigarette label states the unit; hand-rolled tobacco takes it + # from the non-cigarette row. + ( + [{"B": "Hand rolled tobacco clearances", "C": "3338", "D": "3113"}], + r"must state the unit \"\('000s kg\)\" .* found \[\]", + ), + ( + [ + {"B": "Non-cigarette clearances (tonnes)", "C": "3.9", "D": "3.7"}, + {"B": "Hand rolled tobacco clearances", "C": "3338", "D": "3113"}, + ], + "must state the unit", + ), + ( + [ + {"B": "Non-cigarette clearances ('000s kg)", "C": "3940", "D": "3683"}, + {"B": "Cigars ('000s kg)", "C": "50", "D": "45"}, + {"B": "Hand rolled tobacco clearances", "C": "3338", "D": "3113"}, + ], + "in exactly one cell", + ), + ], +) +def test_a_clearance_line_needs_its_unit_stated_once(rows, message): + sheet = {"4.11": [TOBACCO_TITLE, *TOBACCO_HEADER, *rows]} + with pytest.raises(ValueError, match=message): + find_receipts_line(sheet, CLEARANCE_SPECS["hand_rolling_tobacco"]) + + +def test_a_cigarette_line_in_another_unit_is_not_read(): + # The unit sits in the cigarette label, so a new unit renames the row. + rows = [TOBACCO_TITLE, *TOBACCO_HEADER] + rows.append({"B": "Cigarette clearances (billion sticks)", "C": "12", "D": "11"}) + with pytest.raises(ValueError, match="found none"): + find_receipts_line({"4.11": rows}, CLEARANCE_SPECS["cigarettes"]) + + +def test_an_alcohol_table_in_another_unit_is_refused_before_any_write(tmp_path): + yoy, gap_dir = write_tree(tmp_path) + before = {path: path.read_bytes() for path in [yoy, *gap_dir.iterdir()]} + receipts = make_receipts_xlsx( + alcohol_note="Note: All clearances are in millions of litres of alcohol." + ) + with pytest.raises(ValueError, match="thousands of hectolitres of alcohol"): + run_main(tmp_path, make_test_xlsx(), "--yaml-path", str(yoy), receipts=receipts) + assert {path: path.read_bytes() for path in before} == before + + +def test_a_clearance_spec_needs_a_unit(): + with pytest.raises(ValueError, match="needs a unit"): + ClearanceSeriesSpec(path=("per_capita", "x"), title="t", row="r") + + +def test_real_october_2026_clearance_series(): + """The committed series, from the real October 2026 Tables 4.11 and 4.12 + over the real March 2026 Tables 3.11 and 3.12, per head of the committed + ons.population, by hand.""" + population = population_growth(get_yoy_growth_path().read_text()) + assert (population[2025], population[2026]) == (0.0033, 0.002) + growth = compute_receipts_growth( + REAL_OCTOBER_2026_RECEIPTS, + REAL_RECEIPTS[2026], + specs=CLEARANCE_SERIES_SPECS, + ) + per_head = { + path[-1]: per_head_growth(series, population).values + for path, series in growth.items() + } + # Cigarettes: 2025-26 outturn (October 2026) over 2024-25 outturn (March + # 2026), 10.0% fewer sticks, 10.3% fewer per head. + assert per_head["cigarettes"][2025] == round( + 12224.568453264392 / 13578.199084269218 / 1.0033 - 1, 6 + ) + assert per_head["cigarettes"][2025] == -0.102653 + assert per_head["hand_rolling_tobacco"][2025] == -0.17412 + # Beer and cider: 2026-27 over 2025-26, both from October 2026. + assert per_head["beer_and_cider"][2026] == round( + 2069.8474469367434 / 2108.7303043893562 / 1.002 - 1, 6 + ) + committed = get_yoy_growth_path().read_text() + for key, values in per_head.items(): + assert sorted(values) == list(range(2025, 2032)), key + stored = _values(committed, CLEARANCE_PATHS[key]) + assert stored[2024] == 0.0 + assert {year: stored[year] for year in values} == values, key + + +def test_a_population_change_moves_the_clearance_series_alone(): + """Editing ons.population without rerunning the importer leaves the + committed file different from what the importer writes, so the + byte-for-byte test fails until someone reruns it. Only the clearance + series move.""" + committed = get_yoy_growth_path().read_text() + edited = committed.replace( + " 2026-01-01: 0.0020\n 2027-01-01: 0.0013\n", + " 2026-01-01: 0.0030\n 2027-01-01: 0.0013\n", + ) + assert population_growth(edited)[2026] == 0.003 + rendered = render( + edited, + current=REAL_OCTOBER_2026_RECEIPTS, + previous=REAL_RECEIPTS[2026], + **COMMITTED_RELEASES, + ) + for path in ALL_RECEIPTS_PATHS.values(): + before, after = _values(committed, path), _values(rendered, path) + if path in CLEARANCE_PATHS.values(): + levels = [float(level) for level in OCTOBER_2026_CLEARANCES[path[-1]]] + assert after[2026] == round(levels[1] / levels[0] / 1.003 - 1, 6) + assert after[2026] != before[2026] + assert {y: v for y, v in after.items() if y != 2026} == { + y: v for y, v in before.items() if y != 2026 + } + else: + assert after == before, path + + +CLEARANCE_LEVEL = st.floats(min_value=1, max_value=20_000, allow_nan=False) +# A year's clearances from half to twice the year before's, so the index stays +# well above the five decimals the model rounds it to. +CLEARANCE_GROWTH = st.floats(min_value=0.5, max_value=2, allow_nan=False) +POPULATION_GROWTH = st.floats(min_value=-0.02, max_value=0.03, allow_nan=False).map( + lambda value: round(value, 4) +) + + +@settings(max_examples=40, deadline=None, derandomize=True) +@given(years=st.integers(2, 7), data=st.data()) +def test_rendered_clearances_grow_the_weighted_total_with_the_clearances(years, data): + """For any clearances and population growth: in every year the household + quantity's growth times the household weight's growth is the clearances' + growth, to the series' six decimals, and after the forecast it is 1; the + series stays above -100%, so its index stays positive; a second run + changes nothing.""" + base, levels = {}, {} + for key in CLEARANCE_LEVELS: + level = data.draw(CLEARANCE_LEVEL) + base[key] = (level,) + series = [] + for _ in range(years): + level *= data.draw(CLEARANCE_GROWTH) + series.append(level) + levels[key] = tuple(series) + population = { + year: data.draw(POPULATION_GROWTH, label=f"population {year}") + for year in range(2024, 2036) + } + content = YOY_GROWTH_FIXTURE.replace( + POPULATION_FIXTURE, + "ons:\n population:\n values:\n" + + "".join( + f" {year}-01-01: {format_growth(value)}\n" + for year, value in population.items() + ), + ) + current = make_receipts_xlsx(clearance_levels=levels) + previous = make_previous_receipts_xlsx(clearance_levels=base) + rendered = render(content, current=current, previous=previous) + last = 2024 + years - 1 + for key, path in CLEARANCE_PATHS.items(): + values = _values(rendered, path) + published = { + 2023: base[key][0], + **dict(zip(range(2024, last + 1), levels[key])), + } + for year in range(2024, 2036): + growth = _in_effect(values, year) + assert growth > -1 + weighted = (1 + growth) * (1 + population[year]) + target = published[year] / published[year - 1] if year <= last else 1 + # Six-decimal rounding moves the product by at most 5e-7 times + # one plus the population growth. + assert abs(weighted - target) <= 5e-7 * (1 + population[year]) + 1e-12 + assert values[2023] == 0.0 + assert render(rendered, current=current, previous=previous) == rendered diff --git a/policyengine_uk/utils/import_obr_forecasts.py b/policyengine_uk/utils/import_obr_forecasts.py index a22a95f0cb..b28524bf8c 100644 --- a/policyengine_uk/utils/import_obr_forecasts.py +++ b/policyengine_uk/utils/import_obr_forecasts.py @@ -5,7 +5,7 @@ import os import re import shutil -from dataclasses import dataclass +from dataclasses import dataclass, replace from datetime import date from decimal import Decimal from functools import lru_cache @@ -668,6 +668,80 @@ class ReceiptsSeriesSpec: references: tuple[tuple[str, str], ...] = () +@dataclass(frozen=True) +class ClearanceSeriesSpec(ReceiptsSeriesSpec): + """A line of the OBR's tobacco or alcohol clearance tables, which grows + household quantity inputs per head (#2596). + + The line counts quantities, not pounds, so its table has no "£ billion" + cell. ``unit`` is the unit as the table states it, in a row label or a + note, and the importer requires it in exactly one cell of the table in + both EFOs. Dataset projection grows household weights by + ``ons.population``, so ``render_receipts_growth_yaml`` writes the + clearances' growth over one plus that growth (``per_head_growth``) and + the weighted total grows with the clearances. After the OBR's forecast + the series holds the clearances flat, as road fuel volume does. + """ + + unit: str = "" + + def __post_init__(self): + if not self.unit: + raise ValueError(f"{'.'.join(self.path)}: a clearance line needs a unit") + + +TOBACCO_CLEARANCES = "tobacco clearances" +ALCOHOL_CLEARANCES = "alcohol clearance forecasts" +# Table 4.11 (3.11 in March 2026) states each unit in a row label: the +# hand-rolled and other tobacco lines split "Non-cigarette clearances ('000s +# kg)". Table 4.12 (3.12) states its unit in a note. +TOBACCO_STICKS = "(million sticks)" +TOBACCO_KILOGRAMS = "('000s kg)" +ALCOHOL_HECTOLITRES = "thousands of hectolitres of alcohol" + +# The product groups the OBR forecasts. Table 4.12's wine line includes other +# fermented products and its spirits line spirit-based ready-to-drinks (its +# footnotes 1 and 2). +CLEARANCE_SERIES_SPECS = ( + ClearanceSeriesSpec( + path=("per_capita", "tobacco_clearances", "cigarettes"), + title=TOBACCO_CLEARANCES, + row="cigarette clearances (million sticks)", + unit=TOBACCO_STICKS, + ), + ClearanceSeriesSpec( + path=("per_capita", "tobacco_clearances", "hand_rolling_tobacco"), + title=TOBACCO_CLEARANCES, + row="hand rolled tobacco clearances", + unit=TOBACCO_KILOGRAMS, + ), + ClearanceSeriesSpec( + path=("per_capita", "tobacco_clearances", "other_tobacco_products"), + title=TOBACCO_CLEARANCES, + row="other tobacco products", + unit=TOBACCO_KILOGRAMS, + ), + ClearanceSeriesSpec( + path=("per_capita", "alcohol_clearances", "beer_and_cider"), + title=ALCOHOL_CLEARANCES, + row="beer and cider", + unit=ALCOHOL_HECTOLITRES, + ), + ClearanceSeriesSpec( + path=("per_capita", "alcohol_clearances", "wine"), + title=ALCOHOL_CLEARANCES, + row="wine", + unit=ALCOHOL_HECTOLITRES, + ), + ClearanceSeriesSpec( + path=("per_capita", "alcohol_clearances", "spirits"), + title=ALCOHOL_CLEARANCES, + row="spirits", + unit=ALCOHOL_HECTOLITRES, + ), +) + + GAMBLING_RATES = "gov.hmrc.gambling_duty" MACHINE_GAMES = f"{GAMBLING_RATES}.machine_games" @@ -848,6 +922,8 @@ def machine_games_spec(higher_stake_share: float) -> ReceiptsSeriesSpec: (RateComponent(f"{GAMBLING_RATES}.pool_betting.rate"),), ((2024, 8_125), (2025, 3_970)), ), + # The alcohol and tobacco quantity inputs (#2596). + *CLEARANCE_SERIES_SPECS, ) @@ -920,8 +996,9 @@ def line_rate(spec: ReceiptsSeriesSpec, year: int) -> float: @dataclass(frozen=True) class ReceiptsLine: - """A receipts line as one EFO publishes it: £bn by the calendar year in - which each fiscal year starts, and which of those years are outturn.""" + """A receipts line as one EFO publishes it: £bn (a clearance line's own + unit) by the calendar year in which each fiscal year starts, and which of + those years are outturn.""" table: str label: str @@ -937,7 +1014,9 @@ class ReceiptsGrowth: the year before in the previous EFO, where it is outturn. Each later year is the ratio of this EFO's consecutive levels. A line with its own outturn starts at that outturn's second year and has no previous line; - one with rates grows its base instead (see ``ReceiptsSeriesSpec``). + one with rates grows its base instead (see ``ReceiptsSeriesSpec``). A + clearance line holds the clearances' growth, which + ``render_receipts_growth_yaml`` writes per head (``per_head_growth``). """ values: dict[int, float] @@ -1053,13 +1132,38 @@ def validate_receipts_unit( ) +def validate_clearance_unit( + table_rows: list[dict[str, str | None]], spec: ClearanceSeriesSpec, where: str +) -> None: + """Require a clearance line's unit in exactly one cell of its table. + + Table 4.11 states the unit in a row label ("Cigarette clearances + (million sticks)") and Table 4.12 in a note below the rows ("All + clearances are in thousands of hectolitres of alcohol."). The first + year's growth divides one EFO's level by another's, so both must state + the same unit; a new unit needs deliberate importer support. + """ + found = [ + normalise_label(value) + for row in table_rows + for value in row.values() + if spec.unit in normalise_label(value) + ] + if len(found) != 1: + raise ValueError( + f'{where} must state the unit "{spec.unit}" of ' + f"{'.'.join(spec.path)} in exactly one cell; found {found}" + ) + + def find_receipts_line( sheets: dict[str, list[dict[str, str | None]]], spec: ReceiptsSeriesSpec ) -> ReceiptsLine: """The one row labelled ``spec.row`` in a table titled ``spec.title``. - Only that table's rows are read: its cash unit, year headers and its - "Outturn" and "Forecast" markers, which may sit on one row or several. + Only that table's rows are read: its cash unit (a clearance line's unit), + year headers and its "Outturn" and "Forecast" markers, which may sit on + one row or several. """ matches = [] for sheet, rows in sheets.items(): @@ -1103,7 +1207,16 @@ def find_receipts_line( ] if labels and receipts_label(labels[0]) == spec.row: matches.append( - (table, sheet, labels[0], row, years, markers, header_rows) + ( + table, + sheet, + labels[0], + row, + years, + markers, + header_rows, + table_rows, + ) ) name = ".".join(spec.path) @@ -1113,7 +1226,7 @@ def find_receipts_line( f'Expected one "{spec.row}" row in the "{spec.title}" table for ' f"{name}; found {found}" ) - table, sheet, label, row, years, markers, header_rows = matches[0] + table, sheet, label, row, years, markers, header_rows, table_rows = matches[0] where = f"Table {table} (sheet {sheet})" repeated = sorted({y for y in years.values() if list(years.values()).count(y) > 1}) if repeated: @@ -1121,7 +1234,10 @@ def find_receipts_line( f"{where} heads more than one column " + ", ".join(fiscal_year_label(y) for y in repeated) ) - validate_receipts_unit(header_rows, where) + if isinstance(spec, ClearanceSeriesSpec): + validate_clearance_unit(table_rows, spec, where) + else: + validate_receipts_unit(header_rows, where) levels = {} for column, year in years.items(): text = row.get(column) @@ -1223,23 +1339,74 @@ def compute_receipts_growth( levels[year - 1] / rates[year - 1] ) values[year] = round(base_growth - 1, 6) + 0.0 - # The model compounds the growth into an index rounded to five - # decimals (create_economic_assumption_indices), from 1 in a line's - # first year when the line has its own outturn. A positive base that - # rounds the index to zero would zero it in every later year. - index = 1.0 - for year, value in values.items(): - index = round(index * (1 + value), 5) - if index <= 0: - raise ValueError( - f"{name}: {fiscal_year_label(year)} growth of {value} takes " - f"the index to {index} at five decimals, so the base is " - "too small to grow from" - ) + check_index(name, values) result[spec.path] = ReceiptsGrowth(values, line, previous, spec) return result +def check_index(name: str, values: dict[int, float]) -> None: + """Refuse growth that zeroes the index the model builds from it. + + The model compounds the growth into an index rounded to five decimals + (create_economic_assumption_indices), from 1 in a line's first year when + the line has its own outturn. A positive base that rounds the index to + zero would zero it in every later year. + """ + index = 1.0 + for year, value in values.items(): + index = round(index * (1 + value), 5) + if index <= 0: + raise ValueError( + f"{name}: {fiscal_year_label(year)} growth of {value} takes " + f"the index to {index} at five decimals, so the base is " + "too small to grow from" + ) + + +def population_growth(content: str) -> dict[int, float]: + """``ons.population`` growth by year, as a yoy_growth.yaml holds it.""" + values = yaml.safe_load(content)["ons"]["population"]["values"] + return dict( + sorted( + (int(str(instant)[:4]), float(value)) for instant, value in values.items() + ) + ) + + +def growth_in_effect(series: dict[int, float], year: int) -> float: + """A growth series' value in ``year`` as the model reads it: its latest + entry at or before the year, or its first entry before that, since the + model backdates each parameter's first value.""" + earlier = [entry for entry in series if entry <= year] + return series[max(earlier) if earlier else min(series)] + + +def per_head_growth( + growth: ReceiptsGrowth, population: dict[int, float] +) -> ReceiptsGrowth: + """A clearance line's growth per head, to six decimals: each year's ratio + of consecutive levels over one plus that year's ``ons.population`` + growth, minus one. + + Dataset projection grows each household's quantity by this and its + weight by the population growth, so the weighted quantity grows by + their product, the clearances' growth. + """ + first = min(growth.values) + levels = {first - 1: growth.previous_line.levels[first - 1], **growth.line.levels} + values = { + year: round( + levels[year] / levels[year - 1] / (1 + growth_in_effect(population, year)) + - 1, + 6, + ) + + 0.0 + for year in growth.values + } + check_index(".".join(growth.spec.path), values) + return replace(growth, values=values) + + def format_growth(value: float) -> str: """Fixed-point text with at least four decimals that reads back as the same float: 0.038 is written 0.0380, 1e-05 as 0.00001.""" @@ -1387,7 +1554,9 @@ def render_receipts_values( def receipts_basis(spec: ReceiptsSeriesSpec | None) -> str: """How a line's growth is measured, for its comments: nothing for - receipts, the rates taken out for a base.""" + receipts, the rates taken out for a base, population for clearances.""" + if isinstance(spec, ClearanceSeriesSpec): + return "; per head: clearance growth over one plus ons.population growth" if spec is None or not spec.rates: return "" return "; base growth: receipts over the rate the model charges" @@ -1398,8 +1567,10 @@ def receipts_tail(content: str, growth: ReceiptsGrowth) -> tuple[dict[int, float average_earnings path, held at zero in years a base's rate is zero in or comes from.""" last = max(growth.values) - path = average_earnings_path(content, last) spec = growth.spec + if isinstance(spec, ClearanceSeriesSpec): + return clearance_tail(content, last) + path = average_earnings_path(content, last) if spec is None or not spec.rates: return path, "the average_earnings long-run path above" # Rates hold their last value after their last change, as the path does. @@ -1430,6 +1601,26 @@ def receipts_tail(content: str, growth: ReceiptsGrowth) -> tuple[dict[int, float return tail, "the average_earnings long-run path above" +def clearance_tail(content: str, last: int) -> tuple[dict[int, float], str]: + """A clearance series after the OBR's forecast, and its comment. The + clearances hold flat, so per head they grow by 1 / (1 + population) - 1, + to six decimals, in each year to ``ons.population``'s last, written where + the value changes.""" + population = population_growth(content) + yearly = { + year: round(1 / (1 + growth_in_effect(population, year)) - 1, 6) + 0.0 + for year in range(last + 1, max(max(population), last + 1) + 1) + } + tail = { + year: value + for year, value in yearly.items() + if year == last + 1 or value != yearly[year - 1] + } + return tail, ( + "clearances held flat, so per head they fall with ons.population growth" + ) + + def receipts_references( growth: ReceiptsGrowth, release: tuple[str, int], @@ -1474,13 +1665,19 @@ def render_receipts_growth_yaml( Years before this EFO's first year are kept, so a series' index keeps its base year. Years after the OBR's receipts forecast follow - average_earnings as ``content`` holds it. The result is checked to - differ from ``content`` only in those values and references. + average_earnings as ``content`` holds it. A clearance series is written + per head of ``ons.population`` as ``content`` holds it, and holds the + clearances flat after the forecast. The result is checked to differ from + ``content`` only in those values and references. """ before = yaml.safe_load(content) expected = copy.deepcopy(before) lines = content.split("\n") + population = None for path, series in growth.items(): + if isinstance(series.spec, ClearanceSeriesSpec): + population = population or population_growth(content) + series = per_head_growth(series, population) tail, tail_comment = receipts_tail(content, series) references = receipts_references(series, release, previous_release) @@ -1742,14 +1939,26 @@ def print_summary( print(f"- {spec.key}: {', '.join(window)}") -def print_receipts_summary(growth: dict[tuple[str, ...], ReceiptsGrowth]) -> None: +def print_receipts_summary( + growth: dict[tuple[str, ...], ReceiptsGrowth], + population: dict[int, float] | None = None, +) -> None: + """Print each series' growth: a clearance series per head when + ``population`` is given, as it is written.""" print(f"Updating {len(growth)} OBR receipts growth series") for path, series in growth.items(): + basis = "" + if isinstance(series.spec, ClearanceSeriesSpec): + if population is None: + basis = ", clearances" + else: + series = per_head_growth(series, population) + basis = ", per head" window = ", ".join( f"{year}: {format_growth(value)}" for year, value in sorted(series.values.items()) ) - print(f"- {'.'.join(path)} (Table {series.line.table}): {window}") + print(f"- {'.'.join(path)} (Table {series.line.table}{basis}): {window}") def build_arg_parser() -> argparse.ArgumentParser: @@ -1815,8 +2024,9 @@ def build_arg_parser() -> argparse.ArgumentParser: help=( "Do not update the receipts growth series (stamp_duty_land_tax, " "business_rates) that project corporate SDLT and business rates, " - "or the gambling_duty_bases series that project the gambling " - "duty inputs" + "the gambling_duty_bases series that project the gambling duty " + "inputs, or the per_capita clearance series that project the " + "alcohol and tobacco quantity inputs" ), ) parser.add_argument( @@ -2005,7 +2215,9 @@ def main(argv: list[str] | None = None) -> int: f"an EFO before {month} {year}" ) receipts_growth = compute_receipts_growth(receipts_bytes, previous_bytes) - print_receipts_summary(receipts_growth) + print_receipts_summary( + receipts_growth, population_growth(source_yaml_path.read_text()) + ) calendar_values = ( read_calendar_values(source_yaml_path) if args.gaps_only else series_values ) diff --git a/policyengine_uk/variables/input/consumption/alcohol/beer_litres.py b/policyengine_uk/variables/input/consumption/alcohol/beer_litres.py index f1f41ec55c..0b2957ed15 100644 --- a/policyengine_uk/variables/input/consumption/alcohol/beer_litres.py +++ b/policyengine_uk/variables/input/consumption/alcohol/beer_litres.py @@ -7,3 +7,4 @@ class beer_litres(Variable): definition_period = YEAR label = "Annual litres of beer" unit = "litre" + uprating = "gov.economic_assumptions.indices.obr.per_capita.alcohol_clearances.beer_and_cider" diff --git a/policyengine_uk/variables/input/consumption/alcohol/cider_litres.py b/policyengine_uk/variables/input/consumption/alcohol/cider_litres.py index c977131f07..dbd8b0e28f 100644 --- a/policyengine_uk/variables/input/consumption/alcohol/cider_litres.py +++ b/policyengine_uk/variables/input/consumption/alcohol/cider_litres.py @@ -7,3 +7,4 @@ class cider_litres(Variable): definition_period = YEAR label = "Annual litres of cider" unit = "litre" + uprating = "gov.economic_assumptions.indices.obr.per_capita.alcohol_clearances.beer_and_cider" diff --git a/policyengine_uk/variables/input/consumption/alcohol/other_fermented_litres.py b/policyengine_uk/variables/input/consumption/alcohol/other_fermented_litres.py index 4f6e8fc9f6..14f8058254 100644 --- a/policyengine_uk/variables/input/consumption/alcohol/other_fermented_litres.py +++ b/policyengine_uk/variables/input/consumption/alcohol/other_fermented_litres.py @@ -7,3 +7,4 @@ class other_fermented_litres(Variable): definition_period = YEAR label = "Annual litres of other fermented" unit = "litre" + uprating = "gov.economic_assumptions.indices.obr.per_capita.alcohol_clearances.wine" diff --git a/policyengine_uk/variables/input/consumption/alcohol/sparkling_cider_litres.py b/policyengine_uk/variables/input/consumption/alcohol/sparkling_cider_litres.py index 4d634f136b..d7d0b38b67 100644 --- a/policyengine_uk/variables/input/consumption/alcohol/sparkling_cider_litres.py +++ b/policyengine_uk/variables/input/consumption/alcohol/sparkling_cider_litres.py @@ -7,3 +7,4 @@ class sparkling_cider_litres(Variable): definition_period = YEAR label = "Annual litres of sparkling cider" unit = "litre" + uprating = "gov.economic_assumptions.indices.obr.per_capita.alcohol_clearances.beer_and_cider" diff --git a/policyengine_uk/variables/input/consumption/alcohol/spirits_litres.py b/policyengine_uk/variables/input/consumption/alcohol/spirits_litres.py index 707e4b55dc..7509803e42 100644 --- a/policyengine_uk/variables/input/consumption/alcohol/spirits_litres.py +++ b/policyengine_uk/variables/input/consumption/alcohol/spirits_litres.py @@ -7,3 +7,6 @@ class spirits_litres(Variable): definition_period = YEAR label = "Annual litres of spirits" unit = "litre" + uprating = ( + "gov.economic_assumptions.indices.obr.per_capita.alcohol_clearances.spirits" + ) diff --git a/policyengine_uk/variables/input/consumption/alcohol/wine_litres.py b/policyengine_uk/variables/input/consumption/alcohol/wine_litres.py index f47ee347c2..8afabf63fb 100644 --- a/policyengine_uk/variables/input/consumption/alcohol/wine_litres.py +++ b/policyengine_uk/variables/input/consumption/alcohol/wine_litres.py @@ -7,3 +7,4 @@ class wine_litres(Variable): definition_period = YEAR label = "Annual litres of wine" unit = "litre" + uprating = "gov.economic_assumptions.indices.obr.per_capita.alcohol_clearances.wine" diff --git a/policyengine_uk/variables/input/consumption/tobacco/cigarettes_per_week.py b/policyengine_uk/variables/input/consumption/tobacco/cigarettes_per_week.py index 1e5e12513d..680bf799cf 100644 --- a/policyengine_uk/variables/input/consumption/tobacco/cigarettes_per_week.py +++ b/policyengine_uk/variables/input/consumption/tobacco/cigarettes_per_week.py @@ -7,4 +7,7 @@ class cigarettes_per_week(Variable): definition_period = YEAR label = "Cigarettes bought per week" unit = "cigarette" + uprating = ( + "gov.economic_assumptions.indices.obr.per_capita.tobacco_clearances.cigarettes" + ) quantity_type = STOCK diff --git a/policyengine_uk/variables/input/consumption/tobacco/cigars_grams_per_week.py b/policyengine_uk/variables/input/consumption/tobacco/cigars_grams_per_week.py index 4fbe27611d..2cf1b98fb8 100644 --- a/policyengine_uk/variables/input/consumption/tobacco/cigars_grams_per_week.py +++ b/policyengine_uk/variables/input/consumption/tobacco/cigars_grams_per_week.py @@ -7,4 +7,5 @@ class cigars_grams_per_week(Variable): definition_period = YEAR label = "Grams of cigars bought per week" unit = "gram" + uprating = "gov.economic_assumptions.indices.obr.per_capita.tobacco_clearances.other_tobacco_products" quantity_type = STOCK diff --git a/policyengine_uk/variables/input/consumption/tobacco/hand_rolling_tobacco_grams_per_week.py b/policyengine_uk/variables/input/consumption/tobacco/hand_rolling_tobacco_grams_per_week.py index 05e3678f02..56829d8abb 100644 --- a/policyengine_uk/variables/input/consumption/tobacco/hand_rolling_tobacco_grams_per_week.py +++ b/policyengine_uk/variables/input/consumption/tobacco/hand_rolling_tobacco_grams_per_week.py @@ -7,4 +7,5 @@ class hand_rolling_tobacco_grams_per_week(Variable): definition_period = YEAR label = "Grams of hand rolling tobacco bought per week" unit = "gram" + uprating = "gov.economic_assumptions.indices.obr.per_capita.tobacco_clearances.hand_rolling_tobacco" quantity_type = STOCK diff --git a/policyengine_uk/variables/input/consumption/tobacco/heated_tobacco_grams_per_week.py b/policyengine_uk/variables/input/consumption/tobacco/heated_tobacco_grams_per_week.py index e4415f473a..33ec50223a 100644 --- a/policyengine_uk/variables/input/consumption/tobacco/heated_tobacco_grams_per_week.py +++ b/policyengine_uk/variables/input/consumption/tobacco/heated_tobacco_grams_per_week.py @@ -7,4 +7,5 @@ class heated_tobacco_grams_per_week(Variable): definition_period = YEAR label = "Grams of heated tobacco bought per week" unit = "gram" + uprating = "gov.economic_assumptions.indices.obr.per_capita.tobacco_clearances.other_tobacco_products" quantity_type = STOCK diff --git a/policyengine_uk/variables/input/consumption/tobacco/other_tobacco_grams_per_week.py b/policyengine_uk/variables/input/consumption/tobacco/other_tobacco_grams_per_week.py index 213b11742e..8824f55835 100644 --- a/policyengine_uk/variables/input/consumption/tobacco/other_tobacco_grams_per_week.py +++ b/policyengine_uk/variables/input/consumption/tobacco/other_tobacco_grams_per_week.py @@ -7,4 +7,5 @@ class other_tobacco_grams_per_week(Variable): definition_period = YEAR label = "Grams of other tobacco bought per week" unit = "gram" + uprating = "gov.economic_assumptions.indices.obr.per_capita.tobacco_clearances.other_tobacco_products" quantity_type = STOCK From 22fbd0cc1b91cf45854cff3774d281f135b87aae Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 31 Oct 2026 06:24:58 -0400 Subject: [PATCH 02/27] Document how the clearance indices project alcohol and tobacco quantities The excise duties page now maps each quantity input to its OBR line, shows from the projection code why the series divide clearance growth by population growth, lists the Budget 2026 measures that reach the two duties, and compares the model's duty on the OBR's volumes with its receipts forecast. It records each source file, sheet, row and hash. Co-Authored-By: Claude Opus 5.5 --- changelog.d/2596.fixed.md | 1 + docs/engineering/excise-duties.md | 147 ++++++++++++++++++ .../gov/economic_assumptions/README.md | 2 +- 3 files changed, 149 insertions(+), 1 deletion(-) create mode 100644 changelog.d/2596.fixed.md diff --git a/changelog.d/2596.fixed.md b/changelog.d/2596.fixed.md new file mode 100644 index 0000000000..ffbdac2355 --- /dev/null +++ b/changelog.d/2596.fixed.md @@ -0,0 +1 @@ +Dataset projection now grows the alcohol and tobacco quantity inputs by the OBR's clearance forecasts per person, one index for each product group, so later years no longer charge duty on base-year quantities. diff --git a/docs/engineering/excise-duties.md b/docs/engineering/excise-duties.md index 2e353c04a9..4a52bd9191 100644 --- a/docs/engineering/excise-duties.md +++ b/docs/engineering/excise-duties.md @@ -193,6 +193,153 @@ rate: every later year stays one 1 April step behind. Reform classes and legacy reform dictionaries change the annual values after uprating and are not affected. +## Projecting alcohol and tobacco quantities + +Each alcohol and tobacco quantity input declares its product group's index, +and dataset projection grows it by the same group's series in +`gov.economic_assumptions.yoy_growth.obr.per_capita` +(`policyengine_uk/data/uprating_indices.yaml`). Each series is the growth in +the OBR's clearances of the group per person (#2596): + +| Inputs | Series | OBR line | +| --- | --- | --- | +| `cigarettes_per_week` | `tobacco_clearances.cigarettes` | Table 4.11, "Cigarette clearances (million sticks)" | +| `hand_rolling_tobacco_grams_per_week` | `tobacco_clearances.hand_rolling_tobacco` | Table 4.11, "Hand rolled tobacco clearances" | +| `cigars_grams_per_week`, `other_tobacco_grams_per_week`, `heated_tobacco_grams_per_week` | `tobacco_clearances.other_tobacco_products` | Table 4.11, "Other tobacco products" | +| `beer_litres`, `cider_litres`, `sparkling_cider_litres` | `alcohol_clearances.beer_and_cider` | Table 4.12, "Beer and cider" | +| `wine_litres`, `other_fermented_litres` | `alcohol_clearances.wine` | Table 4.12, "Wine", which includes other fermented products | +| `spirits_litres` | `alcohol_clearances.spirits` | Table 4.12, "Spirits", which includes spirit-based ready-to-drinks | + +Table 4.11 splits non-cigarette clearances into hand-rolled tobacco and other +tobacco products only, so the cigar, other tobacco and heated tobacco inputs +share one series. The strengths (`_abv`) and draught shares are +ratios, and `cigarette_price_per_pack` is a price, so they keep their values; +a forward path for the cigarette price is a separate change. + +### Population + +`extend_single_year_dataset` copies the base year's dataset into each later +year, and `apply_single_year_uprating` multiplies each column that +`uprating_indices.yaml` lists by one plus its series' growth in that year. +The file lists `household_weight` under +`gov.economic_assumptions.yoy_growth.ons.population`. So, for household i in +year t, with population growth p(t) and a quantity series g(t): + +- the weight is w(i, t) = w(i, t - 1) × (1 + p(t)); +- the quantity is q(i, t) = q(i, t - 1) × (1 + g(t)); +- the weighted total is Q(t) = Σ w(i, t) × q(i, t) = (1 + p(t)) × (1 + g(t)) × Q(t - 1). + +With g(t) = (1 + c(t)) / (1 + p(t)) - 1, where c(t) is the growth in +clearances, Q(t) = (1 + c(t)) × Q(t - 1): the weighted total follows the +clearances. A series of c(t) itself would give +Q(t) = (1 + p(t)) × (1 + c(t)) × Q(t - 1) and count population growth twice. +`ons.population` grows 1.54% from 2024-25 to 2031-32, so totals grown that +way would end 1.54% above the clearances. + +For cigarettes in 2025-26, the October 2026 EFO's outturn of 12,224.6 million +sticks against the March 2026 EFO's 2024-25 outturn of 13,578.2 million is a +fall of 9.97%. Population grows 0.33%, so the series holds +12,224.6 / 13,578.2 / 1.0033 - 1 = -10.27% (-0.102653). + +### Years + +| Model year | Series | Source | +| --- | --- | --- | +| 2024 (2024-25) | Base year: 0, index 1 | The alcohol and tobacco duties start in model year 2024 | +| 2025 | Outturn | October 2026 Tables 4.11 and 4.12 (2025-26) over March 2026 Tables 3.11 and 3.12 (2024-25) | +| 2026 to 2031 | Forecast | October 2026 Tables 4.11 and 4.12 | +| 2032 on | Clearances held flat: 1 / (1 + p(t)) - 1 | As `road_fuel_volume` after the OBR's forecast | + +The OBR's years run from 1 April and the model's from 6 April. As for road +fuel, each series stores the fiscal year that starts in year t at t-01-01. +The indices, `gov.economic_assumptions.indices.obr.per_capita.*`, start at 1 +in 2024 and stand at: + +| Group | 2025 | 2026 | 2027 | 2028 | 2029 | 2030 | 2031 | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| Cigarettes | 0.897 | 0.803 | 0.721 | 0.660 | 0.599 | 0.558 | 0.520 | +| Hand-rolled tobacco | 0.826 | 0.769 | 0.712 | 0.652 | 0.610 | 0.584 | 0.556 | +| Other tobacco products | 0.861 | 0.815 | 0.781 | 0.715 | 0.679 | 0.653 | 0.615 | +| Beer and cider | 0.969 | 0.950 | 0.929 | 0.921 | 0.914 | 0.903 | 0.891 | +| Wine | 0.965 | 0.963 | 0.945 | 0.937 | 0.929 | 0.917 | 0.907 | +| Spirits | 0.984 | 0.974 | 0.956 | 0.950 | 0.944 | 0.933 | 0.924 | + +Past `PROJECTION_END_YEAR` (2032), policyengine-core scales each input by +the index it declares. + +`policyengine_uk/utils/import_obr_forecasts.py` writes the six series from +`CLEARANCE_SERIES_SPECS` in the same run as the other receipts series (see +the steps for a new EFO in the economic assumptions README). It requires each +table to state its unit, in a row label or a note, and reads `ons.population` +from the `yoy_growth.yaml` it rewrites. + +### Budget measures and reforms + +The October 2026 forecast incorporates the measures announced at Budget 2026 +and since the March forecast (EFO paragraph 3.2). The OBR's policy Table 1.1 +holds no change to an alcohol or tobacco duty rate. Three measures reach the +two duties, and none moves either by more than £13 million a year: + +- measure 19, more generous Small Producer Relief from 1 February 2027: + alcohol duty £1.5 million lower in 2026-27 and £9.2 million to £9.9 + million lower a year after (the model leaves out Small Producer Relief); +- measure 45, HMRC's measures to close the tax gap: tobacco duty £0.7 + million higher in 2026-27, rising to £12.7 million in 2031-32, and alcohol + duty £0.3 million to £4.8 million higher; +- measure 95, the removal of the £135 customs duty relief on low-value + imports from October 2028: tobacco duty up to £0.8 million and alcohol + duty up to £0.4 million lower a year. + +The forecast still assumes rising duty rates. The OBR expects alcohol +receipts to grow, "driven by increases to the duty rate that offset an +assumed average 1 per cent decline in alcohol consumption in each year" +(paragraph 4.43), and tobacco receipts to fall "as higher duty rates only +partly offset falling consumption" (paragraph 4.44). The clearances carry +that consumption trend under the OBR's own rate path, not a Budget change. + +The dataset projection reads no duty rate (`uprating_input_paths` in +`policyengine_uk/data/economic_assumptions.py`), so the indices grow +quantities whatever rates the model charges. A reform to alcohol or tobacco +duty rates therefore moves receipts through the rates alone: the model +applies no consumption response to it. Current law in the model holds the +alcohol rates at their 1 February 2026 values, the tobacco rates at their +1 October 2026 values and the default cigarette price at its 2024 average, +while the OBR's volumes come with the rates it expects. + +### Receipts against the OBR's forecast + +On quantities that follow the clearances, the model's duty moves with its +own rates. A constructed dataset of one household at weight 1 that holds the +UK's 2024-25 clearances shows the effect. It treats beer and cider as beer +at 4% with no draught relief, wine and other fermented products as wine at +12%, spirits at 40%, other tobacco products as other smoking tobacco, and +cigarettes at the default price. On that mix the model's duty per unit +cleared rises 5.7% for alcohol by 2026-27 and 19.0% for tobacco by 2027-28, +with the rate rises of February 2025 and February 2026 for alcohol and of +November 2025 and October 2026 for tobacco, and then holds. The OBR's +receipts per unit cleared keep rising, by 22.1% for alcohol and 48.2% for +tobacco by 2031-32. From 2027-28 the model's duty therefore falls short of +the OBR's forecast by more each year: on this construct, alcohol duty goes +from 7.6% above the OBR's 2024-25 receipts to 6.8% below its 2031-32 +forecast, and tobacco duty from 1.9% above to 18.0% below. The construct's +product mix sets the 2024-25 gaps: charging all beer and cider at the beer +rate with no draught relief puts that line 22.4% above its receipts. A +dataset replaces the mix with its own. The later fall comes from the rates. + +### Sources + +The series and the figures above come from these files. The parser fixtures +in `policyengine_uk/tests/fixtures/obr_receipts/` keep the receipts sheets +of the two receipts workbooks unchanged. + +| File | SHA-256 | Sheet and cells | +| --- | --- | --- | +| OBR October 2026 EFO, detailed forecast tables zip (`efo-october-2026-detailed-forecast-tables-zipfile.zip`) | `06ac908c69fce7d69dbfda4e16fc3b091a83bd8140ecd2f24990acd13d945fc2` | Holds the two workbooks below | +| October 2026 receipts tables (`efo-october-2026-detailed-forecast-tables-receipts.xlsx`) | `d115705716d77b11c190385d41cb1924f30080c9331a8a386c04d4e65a368e95` | 4.11: rows 5 (cigarettes, unit in B5), 6 (non-cigarette, unit in B6), 8 (hand-rolled) and 9 (other tobacco products), columns C (2025-26 outturn) to I (2031-32); 4.12: rows 5 to 7 (beer and cider, wine, spirits), columns C to I, unit note B8, footnotes B9 and B10; 4.8: row 27 (tobacco duties, cash) and 4.12 rows 16 to 18 (alcohol receipts, accrued), for the comparison above | +| October 2026 policy tables (`efo-october-2026-detailed-forecast-tables-policy.xlsx`) | `cdcc2a06b100241bf3010bd159a4e5d43daef2348a3850797b1e88a308fd45fd` | 1.1: rows 95 (measure 19), 203 and 204 (measure 45), 446 and 447 (measure 95), 494 and 515 (totals for alcohol and tobacco duty) | +| October 2026 EFO (`economic-and-fiscal-outlook-october-2026-web-accessible.pdf`) | `9e0e9c340e35e603066f3124a614fcd68feadb3d5d10d1f76fb536e150e6225e` | Paragraphs 3.2, 4.43 and 4.44 | +| March 2026 receipts tables (`obr_efo_march_2026_receipts.xlsx`) | `f65b6cb7f96931d4abd85940794eb60f6f65afca8bbf16c74a4548bf92766a28` | 3.11: rows 5, 8 and 9, column C (2024-25 outturn); 3.12: rows 5 to 7, column C, and rows 16 to 18, column C (2024-25 receipts); 3.8: row 27, column D | + ## Population data follow-ups These formulas do not impute quantities or vehicle characteristics from the diff --git a/policyengine_uk/parameters/gov/economic_assumptions/README.md b/policyengine_uk/parameters/gov/economic_assumptions/README.md index 0efafedc02..ba8b8e2f58 100644 --- a/policyengine_uk/parameters/gov/economic_assumptions/README.md +++ b/policyengine_uk/parameters/gov/economic_assumptions/README.md @@ -72,7 +72,7 @@ household weights by `ons.population`. They link their outturn year to the previous EFO's outturn, as the four receipts series do, and hold the clearances flat after the forecast. The importer reads `ons.population` from the `yoy_growth.yaml` it rewrites. See -[household excise duties](../../../../docs/engineering/excise-duties.md#projection) +[household excise duties](../../../../docs/engineering/excise-duties.md#projecting-alcohol-and-tobacco-quantities) for the mapping of inputs to groups, the arithmetic and the sources. `national_living_wage` projects the minimum wage rates From 648d59c4c4cdcd4dd1d9db8fe7d2e5843d96f0b8 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 31 Oct 2026 07:01:06 -0400 Subject: [PATCH 03/27] Say plainly which unit a receipts line holds Co-Authored-By: Claude Opus 5.5 --- policyengine_uk/utils/import_obr_forecasts.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/policyengine_uk/utils/import_obr_forecasts.py b/policyengine_uk/utils/import_obr_forecasts.py index 4466852e9a..c91dea9283 100644 --- a/policyengine_uk/utils/import_obr_forecasts.py +++ b/policyengine_uk/utils/import_obr_forecasts.py @@ -1000,8 +1000,8 @@ def line_rate(spec: ReceiptsSeriesSpec, year: int) -> float: @dataclass(frozen=True) class ReceiptsLine: - """A receipts line as one EFO publishes it: £bn (a clearance line's own - unit) by the calendar year in which each fiscal year starts, and which of + """A receipts line as one EFO publishes it: £bn, or a clearance line's own + unit, by the calendar year in which each fiscal year starts, and which of those years are outturn.""" table: str @@ -1165,9 +1165,9 @@ def find_receipts_line( ) -> ReceiptsLine: """The one row labelled ``spec.row`` in a table titled ``spec.title``. - Only that table's rows are read: its cash unit (a clearance line's unit), - year headers and its "Outturn" and "Forecast" markers, which may sit on - one row or several. + Only that table's rows are read: its cash unit, or a clearance line's + unit, its year headers and its "Outturn" and "Forecast" markers, which + may sit on one row or several. """ matches = [] for sheet, rows in sheets.items(): From 764117740fb1af469e9d1d881c37831ea4cab2fb Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 31 Oct 2026 07:38:14 -0400 Subject: [PATCH 04/27] Tighten the clearance series after review A clearance spec now refuses rates and an outturn of its own, which per_head_growth cannot use. The tail comment states the formula, which holds once population growth turns negative in 2055. The differential test also checks gov_balance. The README's unit rule names receipts tables only, and its refresh step lists every pin and the wine line's footnote change. The docs name the October 2024 tobacco rise and say which hashes are of the original downloads. Co-Authored-By: Claude Opus 5.5 --- docs/engineering/excise-duties.md | 7 ++++--- .../parameters/gov/economic_assumptions/README.md | 15 +++++++++++---- .../gov/economic_assumptions/yoy_growth.yaml | 12 ++++++------ .../tests/test_excise_quantity_projection.py | 9 +++++---- .../tests/test_import_obr_forecasts.py | 13 ++++++++++--- policyengine_uk/utils/import_obr_forecasts.py | 13 +++++++++---- 6 files changed, 45 insertions(+), 24 deletions(-) diff --git a/docs/engineering/excise-duties.md b/docs/engineering/excise-duties.md index 4a52bd9191..1e903a2943 100644 --- a/docs/engineering/excise-duties.md +++ b/docs/engineering/excise-duties.md @@ -316,7 +316,8 @@ at 4% with no draught relief, wine and other fermented products as wine at cigarettes at the default price. On that mix the model's duty per unit cleared rises 5.7% for alcohol by 2026-27 and 19.0% for tobacco by 2027-28, with the rate rises of February 2025 and February 2026 for alcohol and of -November 2025 and October 2026 for tobacco, and then holds. The OBR's +October 2024, November 2025 and October 2026 for tobacco, each of which +reaches its first full model year after 2024-25, and then holds. The OBR's receipts per unit cleared keep rising, by 22.1% for alcohol and 48.2% for tobacco by 2031-32. From 2027-28 the model's duty therefore falls short of the OBR's forecast by more each year: on this construct, alcohol duty goes @@ -335,10 +336,10 @@ of the two receipts workbooks unchanged. | File | SHA-256 | Sheet and cells | | --- | --- | --- | | OBR October 2026 EFO, detailed forecast tables zip (`efo-october-2026-detailed-forecast-tables-zipfile.zip`) | `06ac908c69fce7d69dbfda4e16fc3b091a83bd8140ecd2f24990acd13d945fc2` | Holds the two workbooks below | -| October 2026 receipts tables (`efo-october-2026-detailed-forecast-tables-receipts.xlsx`) | `d115705716d77b11c190385d41cb1924f30080c9331a8a386c04d4e65a368e95` | 4.11: rows 5 (cigarettes, unit in B5), 6 (non-cigarette, unit in B6), 8 (hand-rolled) and 9 (other tobacco products), columns C (2025-26 outturn) to I (2031-32); 4.12: rows 5 to 7 (beer and cider, wine, spirits), columns C to I, unit note B8, footnotes B9 and B10; 4.8: row 27 (tobacco duties, cash) and 4.12 rows 16 to 18 (alcohol receipts, accrued), for the comparison above | +| October 2026 receipts tables, as published in the zip (`efo-october-2026-detailed-forecast-tables-receipts.xlsx`) | `d115705716d77b11c190385d41cb1924f30080c9331a8a386c04d4e65a368e95` | 4.11: rows 5 (cigarettes, unit in B5), 6 (non-cigarette, unit in B6), 8 (hand-rolled) and 9 (other tobacco products), columns C (2025-26 outturn) to I (2031-32); 4.12: rows 5 to 7 (beer and cider, wine, spirits), columns C to I, unit note B8, footnotes B9 and B10; 4.8: row 27 (tobacco duties, cash) and 4.12 rows 16 to 18 (alcohol receipts, accrued), for the comparison above | | October 2026 policy tables (`efo-october-2026-detailed-forecast-tables-policy.xlsx`) | `cdcc2a06b100241bf3010bd159a4e5d43daef2348a3850797b1e88a308fd45fd` | 1.1: rows 95 (measure 19), 203 and 204 (measure 45), 446 and 447 (measure 95), 494 and 515 (totals for alcohol and tobacco duty) | | October 2026 EFO (`economic-and-fiscal-outlook-october-2026-web-accessible.pdf`) | `9e0e9c340e35e603066f3124a614fcd68feadb3d5d10d1f76fb536e150e6225e` | Paragraphs 3.2, 4.43 and 4.44 | -| March 2026 receipts tables (`obr_efo_march_2026_receipts.xlsx`) | `f65b6cb7f96931d4abd85940794eb60f6f65afca8bbf16c74a4548bf92766a28` | 3.11: rows 5, 8 and 9, column C (2024-25 outturn); 3.12: rows 5 to 7, column C, and rows 16 to 18, column C (2024-25 receipts); 3.8: row 27, column D | +| March 2026 receipts tables, as downloaded (`obr_efo_march_2026_receipts.xlsx`; the parser fixture of that name keeps only some of its sheets) | `f65b6cb7f96931d4abd85940794eb60f6f65afca8bbf16c74a4548bf92766a28` | 3.11: rows 5, 8 and 9, column C (2024-25 outturn); 3.12: rows 5 to 7, column C, and rows 16 to 18, column C (2024-25 receipts); 3.8: row 27, column D | ## Population data follow-ups diff --git a/policyengine_uk/parameters/gov/economic_assumptions/README.md b/policyengine_uk/parameters/gov/economic_assumptions/README.md index 46a3ec16ed..a4dcbc6fcd 100644 --- a/policyengine_uk/parameters/gov/economic_assumptions/README.md +++ b/policyengine_uk/parameters/gov/economic_assumptions/README.md @@ -479,8 +479,9 @@ the forecast levels, though not their growth. renumbering does not matter, and reads only that table's rows. It takes each workbook's release from its contents sheet, or else its file name or URL, and refuses receipts tables from another EFO or of unknown release. - Every selected table in both releases must declare `£ billion` in its - header (`£bn` and `£ bn` are also accepted, ignoring case and whitespace). + Every selected receipts table in both releases must declare `£ billion` + in its header (`£bn` and `£ bn` are also accepted, ignoring case and + whitespace); the clearance tables state their own units (below). Other units, including `£m`, and missing or ambiguous declarations stop the import before any file is written, so different cash magnitudes cannot be combined across releases. @@ -503,11 +504,17 @@ the forecast levels, though not their growth. Each line's table must state its unit, in a row label or a note, in both EFOs: million sticks for cigarettes, thousand kilograms for the other tobacco lines, and thousand hectolitres of alcohol for alcohol. The - importer stops on any other. It divides each year's growth by + importer stops on any other. It does not read the lines' footnotes, so + check that each alcohol line covers the same products in both EFOs: + March 2026 added "Includes other fermented products" to the wine line, + which March 2025's lacked. It divides each year's growth by `ons.population` as `yoy_growth.yaml` holds it, so rerun it whenever that series changes. Then update the published levels in `policyengine_uk/tests/test_excise_quantity_projection.py` - (`CLEARANCES`) and the real-table fixtures in + (`CLEARANCES`) and `policyengine_uk/tests/test_import_obr_forecasts.py` + (`OCTOBER_2026_CLEARANCES` and `MARCH_2026_CLEARANCES`, renamed for the + new EFOs), the values those tests check by hand, the index table in + `docs/engineering/excise-duties.md`, and the real-table fixtures in `policyengine_uk/tests/fixtures/obr_receipts/` (see its README). 7. If the new EFO runs past `PROJECTION_END_YEAR`, move it on (see [Dataset projection horizon](#dataset-projection-horizon)). diff --git a/policyengine_uk/parameters/gov/economic_assumptions/yoy_growth.yaml b/policyengine_uk/parameters/gov/economic_assumptions/yoy_growth.yaml index d09bedb97d..f3da9e7981 100644 --- a/policyengine_uk/parameters/gov/economic_assumptions/yoy_growth.yaml +++ b/policyengine_uk/parameters/gov/economic_assumptions/yoy_growth.yaml @@ -1326,7 +1326,7 @@ obr: 2029-01-01: -0.092617 2030-01-01: -0.068657 2031-01-01: -0.06894 - # Beyond the OBR forecast: clearances held flat, so per head they fall with ons.population growth + # Beyond the OBR forecast: clearances held flat: 1 / (1 + ons.population growth) - 1 2032-01-01: -0.002195 2033-01-01: -0.002096 2034-01-01: -0.001896 @@ -1377,7 +1377,7 @@ obr: 2029-01-01: -0.064999 2030-01-01: -0.041711 2031-01-01: -0.048922 - # Beyond the OBR forecast: clearances held flat, so per head they fall with ons.population growth + # Beyond the OBR forecast: clearances held flat: 1 / (1 + ons.population growth) - 1 2032-01-01: -0.002195 2033-01-01: -0.002096 2034-01-01: -0.001896 @@ -1428,7 +1428,7 @@ obr: 2029-01-01: -0.050588 2030-01-01: -0.039029 2031-01-01: -0.058055 - # Beyond the OBR forecast: clearances held flat, so per head they fall with ons.population growth + # Beyond the OBR forecast: clearances held flat: 1 / (1 + ons.population growth) - 1 2032-01-01: -0.002195 2033-01-01: -0.002096 2034-01-01: -0.001896 @@ -1480,7 +1480,7 @@ obr: 2029-01-01: -0.007436 2030-01-01: -0.011936 2031-01-01: -0.012643 - # Beyond the OBR forecast: clearances held flat, so per head they fall with ons.population growth + # Beyond the OBR forecast: clearances held flat: 1 / (1 + ons.population growth) - 1 2032-01-01: -0.002195 2033-01-01: -0.002096 2034-01-01: -0.001896 @@ -1531,7 +1531,7 @@ obr: 2029-01-01: -0.008291 2030-01-01: -0.012466 2031-01-01: -0.011764 - # Beyond the OBR forecast: clearances held flat, so per head they fall with ons.population growth + # Beyond the OBR forecast: clearances held flat: 1 / (1 + ons.population growth) - 1 2032-01-01: -0.002195 2033-01-01: -0.002096 2034-01-01: -0.001896 @@ -1582,7 +1582,7 @@ obr: 2029-01-01: -0.006394 2030-01-01: -0.012237 2031-01-01: -0.009692 - # Beyond the OBR forecast: clearances held flat, so per head they fall with ons.population growth + # Beyond the OBR forecast: clearances held flat: 1 / (1 + ons.population growth) - 1 2032-01-01: -0.002195 2033-01-01: -0.002096 2034-01-01: -0.001896 diff --git a/policyengine_uk/tests/test_excise_quantity_projection.py b/policyengine_uk/tests/test_excise_quantity_projection.py index 304828a361..9985d04396 100644 --- a/policyengine_uk/tests/test_excise_quantity_projection.py +++ b/policyengine_uk/tests/test_excise_quantity_projection.py @@ -234,9 +234,10 @@ def test_each_series_is_its_clearances_per_head(parameters): def test_cigarettes_per_head_halve_by_2031(parameters): - """The October 2026 forecast puts 2031-32 cigarette clearances 47.2% - below 2024-25. Population grows 1.5% over those years, so clearances per - head fall 48.0%, and the index stands at 0.520.""" + """The October 2026 EFO forecasts 2031-32 cigarette clearances 47.2% + below the March 2026 EFO's 2024-25 outturn. Population grows 1.5% over + those years, so clearances per head fall 48.0%, and the index stands at + 0.520.""" levels = CLEARANCES["tobacco_clearances.cigarettes"] assert levels[-1] / levels[0] - 1 == pytest.approx(-0.4723, abs=1e-4) population = np.prod( @@ -471,7 +472,7 @@ def test_the_change_touches_only_the_quantity_inputs(parameters): ) if year > 2024: assert changed[0] < 0 - for total in ("household_tax", "gov_tax"): + for total in ("household_tax", "gov_tax", "gov_balance"): difference = new_simulation.calculate( total, year ) - old_simulation.calculate(total, year) diff --git a/policyengine_uk/tests/test_import_obr_forecasts.py b/policyengine_uk/tests/test_import_obr_forecasts.py index 38515eeb24..fa3c2843e4 100644 --- a/policyengine_uk/tests/test_import_obr_forecasts.py +++ b/policyengine_uk/tests/test_import_obr_forecasts.py @@ -2824,8 +2824,10 @@ def test_clearance_comments_and_references_name_both_efos_and_the_population(): "growth\n" ) in block assert ( - " # Beyond the OBR forecast: clearances held flat, so per head " - "they fall with ons.population growth\n" + " # Beyond the OBR forecast: clearances held flat: " + "1 / (1 + ons.population growth) - 1\n" + # Per head the series falls while population grows and rises once it + # falls (2030 in the fixture). " 2027-01-01: -0.019608\n" " 2030-01-01: 0.005025\n" ) in block @@ -2897,9 +2899,14 @@ def test_an_alcohol_table_in_another_unit_is_refused_before_any_write(tmp_path): assert {path: path.read_bytes() for path in before} == before -def test_a_clearance_spec_needs_a_unit(): +def test_a_clearance_spec_needs_a_unit_and_takes_no_rates_or_outturn(): with pytest.raises(ValueError, match="needs a unit"): ClearanceSeriesSpec(path=("per_capita", "x"), title="t", row="r") + spec = CLEARANCE_SPECS["wine"] + with pytest.raises(ValueError, match="takes no rates"): + replace(spec, rates=GAMBLING_SPECS[0].rates) + with pytest.raises(ValueError, match="takes no rates"): + replace(spec, outturn=GAMBLING_SPECS[0].outturn) def test_real_october_2026_clearance_series(): diff --git a/policyengine_uk/utils/import_obr_forecasts.py b/policyengine_uk/utils/import_obr_forecasts.py index c91dea9283..b8e3b929ed 100644 --- a/policyengine_uk/utils/import_obr_forecasts.py +++ b/policyengine_uk/utils/import_obr_forecasts.py @@ -690,8 +690,15 @@ class ClearanceSeriesSpec(ReceiptsSeriesSpec): unit: str = "" def __post_init__(self): + name = ".".join(self.path) if not self.unit: - raise ValueError(f"{'.'.join(self.path)}: a clearance line needs a unit") + raise ValueError(f"{name}: a clearance line needs a unit") + # per_head_growth reads the previous EFO's line and the raw levels. + if self.rates or self.outturn is not None: + raise ValueError( + f"{name}: a clearance line counts quantities, so it takes no " + "rates and links to the previous EFO, not its own outturn" + ) TOBACCO_CLEARANCES = "tobacco clearances" @@ -1620,9 +1627,7 @@ def clearance_tail(content: str, last: int) -> tuple[dict[int, float], str]: for year, value in yearly.items() if year == last + 1 or value != yearly[year - 1] } - return tail, ( - "clearances held flat, so per head they fall with ons.population growth" - ) + return tail, "clearances held flat: 1 / (1 + ons.population growth) - 1" def receipts_references( From 7cbc36cf9a824d80a12fc2fce8561ccfc61647c1 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 31 Oct 2026 08:06:13 -0400 Subject: [PATCH 05/27] State the tax-gap measure's two end years, not a trend Co-Authored-By: Claude Opus 5.5 --- docs/engineering/excise-duties.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/engineering/excise-duties.md b/docs/engineering/excise-duties.md index 1e903a2943..89866a7e54 100644 --- a/docs/engineering/excise-duties.md +++ b/docs/engineering/excise-duties.md @@ -284,7 +284,7 @@ two duties, and none moves either by more than £13 million a year: alcohol duty £1.5 million lower in 2026-27 and £9.2 million to £9.9 million lower a year after (the model leaves out Small Producer Relief); - measure 45, HMRC's measures to close the tax gap: tobacco duty £0.7 - million higher in 2026-27, rising to £12.7 million in 2031-32, and alcohol + million higher in 2026-27 and £12.7 million higher in 2031-32, and alcohol duty £0.3 million to £4.8 million higher; - measure 95, the removal of the £135 customs duty relief on low-value imports from October 2028: tobacco duty up to £0.8 million and alcohol From b9494753d402caa732bac9c5483857e5372a22aa Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 31 Oct 2026 16:30:38 -0400 Subject: [PATCH 06/27] Refuse a unit beside a receipts line; divide a series' own growth per head A line's row may hold only its label and a value under each year. The importer used to skip any other cell, so a "kg" cell beside "Other tobacco products" let the line borrow the thousand kg its non-cigarette total states, and growth read 999.0. A clearance table's year row holds only years, and the one cell that states a clearance line's unit must describe the line: its own label, the label of the total it breaks down (the row above "of which:"), or a note. ReceiptsGrowth carries each year's growth before rounding, and per_head_growth divides that, so a gambling base with its own outturn (previous_line None) grows per head by its base growth, not by its receipts or a growth rebuilt from levels and rates. The committed yoy_growth.yaml is unchanged byte for byte. The population test now covers all 13 per-head series: after the merge with #2761 the seven gambling bases move with population too. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_import_obr_forecasts.py | 535 +++++++++++++++++- policyengine_uk/utils/import_obr_forecasts.py | 190 +++++-- 2 files changed, 666 insertions(+), 59 deletions(-) diff --git a/policyengine_uk/tests/test_import_obr_forecasts.py b/policyengine_uk/tests/test_import_obr_forecasts.py index 5c5e403056..6b4bb59291 100644 --- a/policyengine_uk/tests/test_import_obr_forecasts.py +++ b/policyengine_uk/tests/test_import_obr_forecasts.py @@ -1,4 +1,5 @@ import os +import re from dataclasses import replace from datetime import date from pathlib import Path @@ -23,19 +24,24 @@ ClearanceSeriesSpec, ForecastGap, MACHINE_GAMES_HIGHER_STAKE_SHARE, + ReceiptsGrowth, + ReceiptsLine, ReceiptsSeriesSpec, machine_games_spec, average_earnings_path, + check_index, compute_receipts_growth, find_receipts_line, format_gap, format_growth, get_yoy_growth_path, + growth_in_effect, infer_workbook_release, main, per_head_growth, population_growth, read_calendar_values, + receipts_label, read_workbook_sheets, render_forecast_gap_yaml, render_receipts_growth_yaml, @@ -3135,34 +3141,48 @@ def test_real_october_2026_clearance_series(): assert {year: stored[year] for year in values} == values, key -def test_a_population_change_moves_the_clearance_series_alone(): - """Editing ons.population without rerunning the importer leaves the - committed file different from what the importer writes, so the - byte-for-byte test fails until someone reruns it. Only the clearance - series move.""" +def test_a_population_change_moves_the_per_head_series_once_the_importer_reruns(): + """Editing ons.population leaves every per-head series as the importer + wrote it for the old population, so from 2026 the weighted cigarette + total runs 1.003 / 1.002 = 1.000998 times the clearances' path, and the + byte-for-byte test fails until someone reruns the importer. Rerunning + rewrites each per-head series' 2026 value, the six clearance series and + the seven gambling bases alike, and nothing else.""" committed = get_yoy_growth_path().read_text() edited = committed.replace( " 2026-01-01: 0.0020\n 2027-01-01: 0.0013\n", " 2026-01-01: 0.0030\n 2027-01-01: 0.0013\n", ) assert population_growth(edited)[2026] == 0.003 + for path in ALL_RECEIPTS_PATHS.values(): + assert _values(edited, path) == _values(committed, path), path + levels = [float(level) for level in OCTOBER_2026_CLEARANCES["cigarettes"]] + cigarettes = _values(edited, CLEARANCE_PATHS["cigarettes"])[2026] + weighted = (1 + cigarettes) * 1.003 / (levels[1] / levels[0]) + assert weighted == pytest.approx(1.003 / 1.002, abs=1e-6) + assert round(1.003 / 1.002, 6) == 1.000998 + rendered = render( edited, current=REAL_OCTOBER_2026_RECEIPTS, previous=REAL_RECEIPTS[2026], **COMMITTED_RELEASES, ) + growth = compute_receipts_growth(REAL_OCTOBER_2026_RECEIPTS, REAL_RECEIPTS[2026]) + moved = set() for path in ALL_RECEIPTS_PATHS.values(): before, after = _values(committed, path), _values(rendered, path) - if path in CLEARANCE_PATHS.values(): - levels = [float(level) for level in OCTOBER_2026_CLEARANCES[path[-1]]] - assert after[2026] == round(levels[1] / levels[0] / 1.003 - 1, 6) + if growth[path].spec.per_head: + assert after[2026] == round(growth[path].ratios[2026] / 1.003 - 1, 6) assert after[2026] != before[2026] assert {y: v for y, v in after.items() if y != 2026} == { y: v for y, v in before.items() if y != 2026 } + moved.add(path) else: assert after == before, path + assert moved == set(CLEARANCE_PATHS.values()) | set(GAMBLING_PATHS.values()) + assert len(moved) == 13 CLEARANCE_LEVEL = st.floats(min_value=1, max_value=20_000, allow_nan=False) @@ -3223,3 +3243,502 @@ def test_rendered_clearances_grow_the_weighted_total_with_the_clearances(years, assert abs(weighted - target) <= 5e-7 * (1 + population[year]) + 1e-12 assert values[2023] == 0.0 assert render(rendered, current=current, previous=previous) == rendered + + +# A unit stated beside a line, or anywhere that does not describe it, must +# stop the run: the importer would otherwise read the line's values in the +# unit some other cell states. + +TOBACCO_UNITS = ( + "kg", + "'000s kg", + "('000s kg)", + "tonnes", + "(million sticks)", + "£ billion", + "thousands of hectolitres of alcohol", +) + + +def mixed_unit_workbook(rows): + """A one-sheet workbook of inline strings, as Sol's review built it.""" + table = rows[0]["B"].split()[0] + return make_workbook( + { + table: make_sheet( + { + number: [ + make_inline_cell(f"{column}{number}", str(value)) + for column, value in row.items() + ] + for number, row in enumerate(rows, start=1) + } + ) + } + ) + + +def test_a_unit_beside_a_clearance_line_is_refused_not_borrowed(): + """Sol's minimal case: the previous release holds other tobacco in + thousand kg, the current one in kg in a cell beside the line, while its + non-cigarette total still states thousand kg. Read as one unit, the line + grows 999-fold; the importer refuses it.""" + previous = mixed_unit_workbook( + [ + {"B": "3.11 Tobacco clearances"}, + {"C": "Outturn"}, + {"C": "2024-25"}, + {"B": "Non-cigarette clearances ('000s kg)", "C": 4000}, + {"B": "of which:"}, + {"B": "Other tobacco products", "C": 600}, + ] + ) + current = mixed_unit_workbook( + [ + {"B": "4.11 Tobacco clearances"}, + {"D": "Outturn", "E": "Forecast"}, + {"D": "2025-26", "E": "2026-27"}, + {"B": "Non-cigarette clearances ('000s kg)", "D": 4000, "E": 3900}, + {"B": "of which:"}, + {"B": "Other tobacco products", "C": "kg", "D": 600000, "E": 570000}, + ] + ) + spec = CLEARANCE_SPECS["other_tobacco_products"] + with pytest.raises(ValueError, match=r"holds \{'C': 'kg'\} beside"): + compute_receipts_growth(current, previous, specs=(spec,)) + # Without the kg cell the same layout reads. + reconciled = mixed_unit_workbook( + [ + {"B": "4.11 Tobacco clearances"}, + {"D": "Outturn", "E": "Forecast"}, + {"D": "2025-26", "E": "2026-27"}, + {"B": "Non-cigarette clearances ('000s kg)", "D": 4000, "E": 3900}, + {"B": "of which:"}, + {"B": "Other tobacco products", "D": 600, "E": 570}, + ] + ) + growth = compute_receipts_growth(reconciled, previous, specs=(spec,))[spec.path] + assert growth.values == {2025: 0.0, 2026: -0.05} + assert per_head_growth(growth, {2025: 0.01, 2026: 0.01}).values[2025] == round( + 1 / 1.01 - 1, 6 + ) + + +@pytest.mark.parametrize( + "rows, message", + [ + # A unit column: its heading sits in the fiscal-year row. + ( + [ + {"C": "Unit", "D": "2025-26", "E": "2026-27"}, + {"B": "Non-cigarette clearances ('000s kg)", "D": "3940", "E": "3683"}, + {"B": "of which:"}, + {"B": "Hand rolled tobacco clearances", "D": "3338", "E": "3113"}, + ], + "in its fiscal-year row", + ), + # The total states the unit, but the line is not one of its parts. + ( + [ + {"C": "2025-26", "D": "2026-27"}, + {"B": "Non-cigarette clearances ('000s kg)", "C": "3940", "D": "3683"}, + {"B": "Hand rolled tobacco clearances", "C": "3338", "D": "3113"}, + ], + "not the line's label, the label of the total it breaks down", + ), + # A total below the line describes other lines. + ( + [ + {"C": "2025-26", "D": "2026-27"}, + {"B": "Hand rolled tobacco clearances", "C": "3338", "D": "3113"}, + {"B": "Non-cigarette clearances ('000s kg)", "C": "3940", "D": "3683"}, + {"B": "of which:"}, + {"B": "Other tobacco products", "C": "602", "D": "571"}, + ], + "not the line's label", + ), + # A row between the "of which:" row and the line that is not a part. + ( + [ + {"C": "2025-26", "D": "2026-27"}, + {"B": "Non-cigarette clearances ('000s kg)", "C": "3940", "D": "3683"}, + {"B": "of which:"}, + {"B": "Heated tobacco is counted elsewhere"}, + {"B": "Hand rolled tobacco clearances", "C": "3338", "D": "3113"}, + ], + "not the line's label", + ), + # Text that states the unit but is not a note. + ( + [ + {"C": "2025-26", "D": "2026-27"}, + {"B": "Hand rolled tobacco clearances", "C": "3338", "D": "3113"}, + {"B": "Source: HMRC, cigars in ('000s kg)"}, + ], + "not the line's label", + ), + # A unit after the line's last year. + ( + [ + {"C": "2025-26", "D": "2026-27"}, + {"B": "Non-cigarette clearances ('000s kg)", "C": "3940", "D": "3683"}, + {"B": "of which:"}, + { + "B": "Hand rolled tobacco clearances", + "C": "3338", + "D": "3113", + "E": "tonnes", + }, + ], + r"holds \{'E': 'tonnes'\} beside", + ), + ], +) +def test_a_clearance_unit_must_describe_its_line(rows, message): + sheet = {"4.11": [TOBACCO_TITLE, {"C": "Outturn", "D": "Forecast"}, *rows]} + with pytest.raises(ValueError, match=message): + find_receipts_line(sheet, CLEARANCE_SPECS["hand_rolling_tobacco"]) + + +def test_a_unit_note_beside_a_receipts_line_is_refused(): + """A pound line's row holds its label and values alone too: a "£ million" + cell beside it would put its values in another magnitude.""" + rows = [ + {"B": "4.7 Property transactions taxes: receipts by sector"}, + {"C": "£ billion"}, + {"C": "Outturn", "D": "Forecast"}, + {"C": "2025-26", "D": "2026-27"}, + { + "B": "SDLT from residential property", + "C": "11.5", + "D": "12.0", + "E": "£ million", + }, + ] + spec = next(spec for spec in RECEIPTS_SPECS if spec.path[-1] == "residential") + with pytest.raises(ValueError, match=r"holds \{'E': '£ million'\} beside"): + find_receipts_line({"4.7": rows}, spec) + + +def put_unit_beside_real_line(workbook, table, label, unit, scale=1000): + """Sol's change to a real table: shift the sheet's cells from column C, + row 3, one column right, put ``unit`` in C beside the line labelled + ``label`` and scale that line's values. Every other archive entry, + including the shared strings, stays.""" + output = BytesIO() + with ZipFile(BytesIO(workbook)) as source, ZipFile(output, "w") as dest: + path = _sheet_paths(source)[table] + strings = [ + "".join(node.itertext()) + for node in ET.fromstring(source.read("xl/sharedStrings.xml")) + ] + sheet = ET.fromstring(source.read(path)) + lines = 0 + for row in sheet.findall("m:sheetData/m:row", XLSX_NS): + number = int(row.get("r")) + text = row.find("m:c[@t='s']/m:v", XLSX_NS) + line = text is not None and strings[int(text.text)] == label + lines += line + if number < 3: + continue + for cell in row.findall("m:c", XLSX_NS): + column = cell.get("r").rstrip("0123456789") + if len(column) == 1 and column >= "C": + cell.set("r", f"{chr(ord(column) + 1)}{number}") + value = cell.find("m:v", XLSX_NS) + if line and value is not None and cell.get("t") != "s": + value.text = repr(float(value.text) * scale) + if line: + cell = ET.SubElement( + row, f"{{{XLSX_MAIN}}}c", {"r": f"C{number}", "t": "inlineStr"} + ) + ET.SubElement( + ET.SubElement(cell, f"{{{XLSX_MAIN}}}is"), f"{{{XLSX_MAIN}}}t" + ).text = unit + assert lines == 1, (table, label, lines) + for entry in source.infolist(): + dest.writestr( + entry, + ET.tostring(sheet) if entry.filename == path else source.read(entry), + ) + return output.getvalue() + + +def test_real_tables_with_a_kg_cell_beside_a_line_are_refused(tmp_path): + """The real October 2026 Table 4.11 changed as Sol's review changed it: + other tobacco in kg, stated beside the line, with its values 1,000 + times larger. Read in the total's thousand kg, the line would grow + 862-fold. The renderer refuses it, and a run on the real March 2026 + tables changed the same way stops before it writes anything.""" + changed = put_unit_beside_real_line( + REAL_OCTOBER_2026_RECEIPTS, "4.11", "Other tobacco products", "kg" + ) + (line,) = [ + row + for row in read_workbook_sheets(changed)["4.11"] + if receipts_label(row.get("B")) == "other tobacco products" + ] + assert line["C"] == "kg" + assert line["D"] == repr( + float(OCTOBER_2026_CLEARANCES["other_tobacco_products"][0]) * 1000 + ) + with pytest.raises(ValueError, match=r"holds \{'C': 'kg'\} beside"): + render( + get_yoy_growth_path().read_text(), + current=changed, + previous=REAL_RECEIPTS[2026], + **COMMITTED_RELEASES, + ) + yoy, gap_dir = write_tree(tmp_path) + before = {path: path.read_bytes() for path in [yoy, *gap_dir.iterdir()]} + with pytest.raises(ValueError, match=r"holds \{'C': 'kg'\} beside"): + run_main( + tmp_path, + make_test_xlsx(), + "--yaml-path", + str(yoy), + receipts=put_unit_beside_real_line( + REAL_RECEIPTS[2026], "3.11", "Other tobacco products", "kg" + ), + previous_receipts=REAL_RECEIPTS[2025], + ) + assert {path: path.read_bytes() for path in before} == before + assert not list(yoy.parent.rglob("*.staged")) + # Shifted without the kg cell, the real table reads as before. + shifted = put_unit_beside_real_line( + REAL_OCTOBER_2026_RECEIPTS, "4.11", "Other tobacco products", "", scale=1 + ) + path = CLEARANCE_SPECS["other_tobacco_products"].path + assert ( + compute_receipts_growth( + shifted, REAL_RECEIPTS[2026], specs=CLEARANCE_SERIES_SPECS + )[path].values + == compute_receipts_growth( + REAL_OCTOBER_2026_RECEIPTS, + REAL_RECEIPTS[2026], + specs=CLEARANCE_SERIES_SPECS, + )[path].values + ) + + +REAL_OCTOBER_2026_SHEETS = read_workbook_sheets(REAL_OCTOBER_2026_RECEIPTS) + + +@settings(max_examples=60, deadline=None, derandomize=True) +@given( + key=st.sampled_from(sorted(CLEARANCE_LEVELS)), + unit=st.one_of( + st.sampled_from(TOBACCO_UNITS), + # Any other text, but not a fiscal year or a marker, which would make + # the row a heading rather than a line. + st.text(min_size=1, max_size=12).filter( + lambda text: ( + text.strip() + and not re.fullmatch(r"\s*\d{4}-\d{2}\s*", text) + and receipts_label(text) not in ("outturn", "forecast") + ) + ), + ), + beside=st.sampled_from(["none", "before", "after"]), + total=st.sampled_from(["('000s kg)", "(tonnes)", "(kg)", "('000s tonnes)"]), +) +def test_a_clearance_line_reads_only_in_the_unit_its_table_states( + key, unit, beside, total +): + """For any unit text beside any real October 2026 clearance line, before + its years or after them, the importer refuses the line. With nothing + beside it, a tobacco part reads only under a non-cigarette total that + states thousand kg, and then reads the real values.""" + spec = CLEARANCE_SPECS[key] + table = "4.11" if "tobacco" in spec.path[1] else "4.12" + rows = [dict(row) for row in REAL_OCTOBER_2026_SHEETS[table]] + for row in rows: + label = receipts_label(row.get("B")) + if label.startswith("non-cigarette clearances"): + row["B"] = f"Non-cigarette clearances {total}" + line = next( + row + for row in rows + if receipts_label(row.get("B")) == spec.row + and any(column != "B" for column in row) + ) + if beside == "before": + for row in rows[1:]: + for column in sorted( + (column for column in row if len(column) == 1 and column >= "C"), + reverse=True, + ): + row[chr(ord(column) + 1)] = row.pop(column) + line["C"] = unit + elif beside == "after": + line["J"] = unit + sheets = {table: rows} + if beside != "none": + event("refused: beside") + with pytest.raises(ValueError, match="beside"): + find_receipts_line(sheets, spec) + return + parts = ("hand_rolling_tobacco", "other_tobacco_products") + if key in parts and total != "('000s kg)": + event("refused: total in another unit") + with pytest.raises(ValueError, match="must state the unit"): + find_receipts_line(sheets, spec) + return + event("read") + assert find_receipts_line(sheets, spec).levels == dict( + zip( + range(2025, 2032), + (float(level) for level in OCTOBER_2026_CLEARANCES[key]), + ) + ) + + +# per_head_growth divides the growth a series holds, a duty's base for a line +# with rates, whether the line links to the previous EFO or has its own +# outturn. + +REMOTE_GAMING = next( + spec for spec in RECEIPTS_SERIES_SPECS if spec.path[-1] == "remote_gaming" +) + + +def test_per_head_growth_divides_the_base_growth_the_series_holds(): + """Sol's minimal case: Remote Gaming Duty receipts double while the base, + receipts over the rate, grows 10%, and population grows 1%. Per head + the base grows 1.10 / 1.01 - 1, not 2 / 1.01 - 1 (receipts) or the + growth the levels and the model's rates give, with or without a previous + line.""" + line = ReceiptsLine( + table="4.10", + label="Remote Gaming Duty", + levels={2025: 100.0, 2026: 200.0}, + outturn_years=frozenset(), + ) + series = ReceiptsGrowth( + values={2025: 0.0, 2026: 0.10}, + line=line, + previous_line=None, + spec=REMOTE_GAMING, + ) + population = {2025: 0.01, 2026: 0.01} + assert per_head_growth(series, population).values == { + 2025: round(1 / 1.01 - 1, 6), + 2026: 0.089109, + } + previous = ReceiptsLine( + table="3.10", + label="Remote Gaming Duty", + levels={2024: 100.0}, + outturn_years=frozenset({2024}), + ) + assert ( + per_head_growth(replace(series, previous_line=previous), population).values[ + 2026 + ] + == 0.089109 + ) + + +def test_real_remote_gaming_base_per_head_rounds_once(): + """The real line has its own outturn and no previous line. Per head, each + year divides the base's growth before rounding by one plus population + growth and rounds once, as the committed series holds it.""" + growth = compute_receipts_growth( + REAL_OCTOBER_2026_RECEIPTS, REAL_RECEIPTS[2026], specs=(REMOTE_GAMING,) + )[REMOTE_GAMING.path] + assert growth.previous_line is None + population = population_growth(get_yoy_growth_path().read_text()) + per_head = per_head_growth(growth, population) + for year, ratio in growth.ratios.items(): + assert per_head.values[year] == round( + ratio / (1 + growth_in_effect(population, year)) - 1, 6 + ) + assert per_head.ratios[year] == ratio / (1 + growth_in_effect(population, year)) + stored = _values(get_yoy_growth_path().read_text(), REMOTE_GAMING.path) + assert {year: stored[year] for year in per_head.values} == per_head.values + + +@settings(max_examples=60, deadline=None, derandomize=True) +@given( + ratios=st.lists( + st.floats(min_value=0.5, max_value=2, allow_nan=False), min_size=1, max_size=8 + ), + population=st.lists(POPULATION_GROWTH, min_size=8, max_size=8), +) +def test_per_head_growth_times_population_growth_is_the_series_growth( + ratios, population +): + """For any growth before rounding and any population growth: per head, + one plus the series times one plus population growth is the growth to + within the six-decimal rounding, 5e-7 times one plus population growth; + the per-head series rounds its own ratios; and a series without its + ratios divides one plus its six-decimal values instead.""" + years = range(2025, 2025 + len(ratios)) + by_year = dict(zip(years, ratios)) + population_by_year = dict(zip(range(2025, 2033), population)) + values = {year: round(ratio - 1, 6) + 0.0 for year, ratio in by_year.items()} + line = ReceiptsLine("4.12", "Wine", {}, frozenset()) + spec = CLEARANCE_SPECS["wine"] + series = ReceiptsGrowth(values, line, None, spec, by_year) + per_head = per_head_growth(series, population_by_year) + for year, ratio in by_year.items(): + growth = 1 + population_by_year[year] + assert ( + abs((1 + per_head.values[year]) * growth - ratio) <= 5e-7 * growth + 1e-12 + ) + assert per_head.values[year] == round(per_head.ratios[year] - 1, 6) + 0.0 + rounded = per_head_growth(replace(series, ratios=None), population_by_year) + assert rounded.values == { + year: round((1 + value) / (1 + population_by_year[year]) - 1, 6) + 0.0 + for year, value in values.items() + } + + +def test_a_series_growth_before_rounding_must_round_to_its_values(): + line = ReceiptsLine("4.12", "Wine", {}, frozenset()) + with pytest.raises(ValueError, match="does not round to the growth"): + ReceiptsGrowth({2025: 0.01}, line, None, CLEARANCE_SPECS["wine"], {2025: 1.02}) + with pytest.raises(ValueError, match="does not round to the growth"): + ReceiptsGrowth( + {2025: 0.01}, + line, + None, + CLEARANCE_SPECS["wine"], + {2025: 1.01, 2026: 1.0}, + ) + + +def test_population_growth_before_its_first_entry_is_that_entry(): + """The model backdates a parameter's first value, so a year before + ons.population's first entry takes that entry, not zero: wine flat in + 2025 falls 1 / 1.02 - 1 per head when population's first entry, 2026, + is 2%.""" + assert growth_in_effect({2026: 0.02, 2030: -0.005}, 2025) == 0.02 + assert growth_in_effect({2026: 0.02, 2030: -0.005}, 2029) == 0.02 + assert growth_in_effect({2026: 0.02, 2030: -0.005}, 2031) == -0.005 + before = ReceiptsLine("3.12", "Wine", {2024: 100.0}, frozenset({2024})) + current = ReceiptsLine("4.12", "Wine", {2025: 100.0}, frozenset({2025})) + series = ReceiptsGrowth( + {2025: 0.0}, current, before, CLEARANCE_SPECS["wine"], {2025: 1.0} + ) + assert per_head_growth(series, {2026: 0.02}).values == {2025: -0.019608} + + +def test_per_head_growth_refuses_an_index_that_rounds_to_zero(): + """Clearances that fall from 1,000,000 to 4.55 leave a raw index of + 0.00001 at five decimals; divided by 1.03 for population, the per-head + index rounds to zero, so the importer refuses the series.""" + before = ReceiptsLine("3.12", "Wine", {2024: 1e6}, frozenset({2024})) + current = ReceiptsLine("4.12", "Wine", {2025: 4.55}, frozenset({2025})) + ratio = 4.55 / 1e6 + series = ReceiptsGrowth( + {2025: round(ratio - 1, 6) + 0.0}, + current, + before, + CLEARANCE_SPECS["wine"], + {2025: ratio}, + ) + check_index("raw", series.values) + assert round(1 + series.values[2025], 5) == 0.00001 + with pytest.raises(ValueError, match="too small to grow from"): + per_head_growth(series, {2025: 0.03}) diff --git a/policyengine_uk/utils/import_obr_forecasts.py b/policyengine_uk/utils/import_obr_forecasts.py index d16144ecf8..7e4538f178 100644 --- a/policyengine_uk/utils/import_obr_forecasts.py +++ b/policyengine_uk/utils/import_obr_forecasts.py @@ -1054,12 +1054,29 @@ class ReceiptsGrowth: one with rates grows its base instead (see ``ReceiptsSeriesSpec``). A clearance line holds the clearances' growth, which ``render_receipts_growth_yaml`` writes per head (``per_head_growth``). + + ``ratios`` holds one plus each year's growth before rounding, as + ``compute_receipts_growth`` finds it, so ``per_head_growth`` rounds the + per-head series once. Each ratio must round to its year's value. """ values: dict[int, float] line: ReceiptsLine previous_line: ReceiptsLine | None spec: ReceiptsSeriesSpec | None = None + ratios: dict[int, float] | None = None + + def __post_init__(self): + if self.ratios is None: + return + if sorted(self.ratios) != sorted(self.values) or any( + round(ratio - 1, 6) + 0.0 != self.values[year] + for year, ratio in self.ratios.items() + ): + raise ValueError( + f"{self.line.label}: the growth before rounding {self.ratios} " + f"does not round to the growth {self.values}" + ) def fiscal_year_label(year: int) -> str: @@ -1169,28 +1186,71 @@ def validate_receipts_unit( ) +def row_cells(row: dict[str, str | None]) -> list[tuple[str, str]]: + """A row's non-empty cells, left to right, as (column, normalised text).""" + return [ + (column, normalise_label(value)) + for column, value in sorted(row.items(), key=lambda item: column_index(item[0])) + if normalise_label(value) + ] + + def validate_clearance_unit( - table_rows: list[dict[str, str | None]], spec: ClearanceSeriesSpec, where: str + table_rows: list[dict[str, str | None]], + line_index: int, + spec: ClearanceSeriesSpec, + where: str, ) -> None: - """Require a clearance line's unit in exactly one cell of its table. - - Table 4.11 states the unit in a row label ("Cigarette clearances - (million sticks)") and Table 4.12 in a note below the rows ("All - clearances are in thousands of hectolitres of alcohol."). The first - year's growth divides one EFO's level by another's, so both must state - the same unit; a new unit needs deliberate importer support. + """Require a clearance line's unit in exactly one cell of its table, and + that cell to describe the line. + + The OBR states the unit in one of three places, and only these describe + the line: + + - the line's own label, as Table 4.11's "Cigarette clearances (million + sticks)"; + - the label of the total the line breaks down: the row above an "of + which:" row, with only the total's other parts between that and the + line, as "Non-cigarette clearances ('000s kg)" for hand-rolled and + other tobacco; + - a note on a row of its own, as Table 4.12's "Note: All clearances are + in thousands of hectolitres of alcohol." + + The first year's growth divides one EFO's level by another's, so both + must state the same unit there. A unit anywhere else, such as another + line's label or a unit column, describes something else; a new unit or + layout needs deliberate importer support. """ + name = ".".join(spec.path) found = [ - normalise_label(value) - for row in table_rows - for value in row.values() - if spec.unit in normalise_label(value) + (index, text) + for index, row in enumerate(table_rows) + for _, text in row_cells(row) + if spec.unit in text ] if len(found) != 1: raise ValueError( - f'{where} must state the unit "{spec.unit}" of ' - f"{'.'.join(spec.path)} in exactly one cell; found {found}" + f'{where} must state the unit "{spec.unit}" of {name} in exactly ' + f"one cell; found {[text for _, text in found]}" ) + index, text = found[0] + cells = row_cells(table_rows[index]) + if index == line_index: + return + if len(cells) == 1 and text.startswith("note"): + return + between = table_rows[index + 2 : line_index] + if ( + index + 1 < line_index + and cells[0][1] == text + and [cell for _, cell in row_cells(table_rows[index + 1])] == ["of which:"] + and all(len(row_cells(row)) > 1 for row in between) + ): + return + raise ValueError( + f'{where} states the unit "{spec.unit}" of {name} in {text!r}, which is ' + "not the line's label, the label of the total it breaks down, or a note" + ) def find_receipts_line( @@ -1209,12 +1269,14 @@ def find_receipts_line( continue where = f"Table {table} (sheet {sheet})" years: dict[str, int] = {} + year_row: dict[str, str | None] = {} markers: dict[str, int] = {} header_rows = [] - for row in table_rows: + for index, row in enumerate(table_rows): row_years = _fiscal_year_columns(row) if row_years: years = row_years + year_row = row continue if not years: header_rows.append(row) @@ -1235,14 +1297,14 @@ def find_receipts_line( continue first_year_column = min(column_index(column) for column in years) labels = [ - value + (column, value) for column, value in sorted( row.items(), key=lambda item: column_index(item[0]) ) if column_index(column) < first_year_column and normalise_label(value) ] - if labels and receipts_label(labels[0]) == spec.row: + if labels and receipts_label(labels[0][1]) == spec.row: matches.append( ( table, @@ -1253,6 +1315,8 @@ def find_receipts_line( markers, header_rows, table_rows, + index, + year_row, ) ) @@ -1263,7 +1327,18 @@ def find_receipts_line( f'Expected one "{spec.row}" row in the "{spec.title}" table for ' f"{name}; found {found}" ) - table, sheet, label, row, years, markers, header_rows, table_rows = matches[0] + ( + table, + sheet, + (label_column, label), + row, + years, + markers, + header_rows, + table_rows, + line_index, + year_row, + ) = matches[0] where = f"Table {table} (sheet {sheet})" repeated = sorted({y for y in years.values() if list(years.values()).count(y) > 1}) if repeated: @@ -1271,8 +1346,34 @@ def find_receipts_line( f"{where} heads more than one column " + ", ".join(fiscal_year_label(y) for y in repeated) ) + # The line's row holds its label and a value under each year, nothing + # else: a unit or note beside it would describe its values, and the + # importer would read them as if in the table's unit. + beside = { + column: value + for column, value in row.items() + if normalise_label(value) and column != label_column and column not in years + } + if beside: + raise ValueError( + f"{where} holds {beside} beside the {name} row's label and values; " + "a unit or note on the row needs deliberate importer support" + ) if isinstance(spec, ClearanceSeriesSpec): - validate_clearance_unit(table_rows, spec, where) + # A clearance table states each unit in a label or a note, so a cell + # beside its years, such as a unit column's heading, is a layout the + # importer does not read. + heading = { + column: value + for column, value in year_row.items() + if normalise_label(value) and column not in years + } + if heading: + raise ValueError( + f"{where} holds {heading} in its fiscal-year row; a clearance " + "table states each unit in a label or a note" + ) + validate_clearance_unit(table_rows, line_index, spec, where) else: validate_receipts_unit(header_rows, where) levels = {} @@ -1365,12 +1466,10 @@ def compute_receipts_growth( f"{name}: {fiscal_year_label(year)} receipts of {level} are " "not positive, so there is no growth to or from them" ) - values = { - year: round(growth_ratio(spec, levels, year) - 1, 6) + 0.0 - for year in sorted(levels)[1:] - } + ratios = {year: growth_ratio(spec, levels, year) for year in sorted(levels)[1:]} + values = {year: round(ratio - 1, 6) + 0.0 for year, ratio in ratios.items()} check_index(name, values) - result[spec.path] = ReceiptsGrowth(values, line, previous, spec) + result[spec.path] = ReceiptsGrowth(values, line, previous, spec, ratios) return result @@ -1414,36 +1513,25 @@ def growth_in_effect(series: dict[int, float], year: int) -> float: def per_head_growth( growth: ReceiptsGrowth, population: dict[int, float] ) -> ReceiptsGrowth: - """A series' growth per head, to six decimals: each year's growth before - rounding (``growth_ratio``), over one plus that year's ``ons.population`` - growth, minus one. - - Dataset projection grows each household's input by this and its weight - by the population growth, so the weighted input grows by their product, - the line's growth: a duty's base, for a line with rates. + """A series' growth per head, to six decimals: one plus each year's + growth, over one plus that year's ``ons.population`` growth, minus one. + + The growth is the series' own, before rounding where ``growth.ratios`` + holds it: a duty's base for a line with rates, whether it links to the + previous EFO or has its own outturn. Dataset projection grows each + household's input by this and its weight by the population growth, so + the weighted input grows by their product, the series' growth. """ - spec = growth.spec - first = min(growth.line.levels) - if growth.previous_line is None: - earlier = { - year: value / 1e6 - for year, value in spec.outturn.receipts_thousands - if year < first - } - else: - earlier = {first - 1: growth.previous_line.levels[first - 1]} - levels = {**earlier, **growth.line.levels} - values = { - year: round( - growth_ratio(spec, levels, year) / (1 + growth_in_effect(population, year)) - - 1, - 6, - ) - + 0.0 - for year in growth.values + ratios = growth.ratios + if ratios is None: + ratios = {year: 1 + value for year, value in growth.values.items()} + per_head = { + year: ratio / (1 + growth_in_effect(population, year)) + for year, ratio in ratios.items() } + values = {year: round(ratio - 1, 6) + 0.0 for year, ratio in per_head.items()} check_index(".".join(growth.spec.path), values) - return replace(growth, values=values) + return replace(growth, values=values, ratios=per_head) def format_growth(value: float) -> str: From ed791d69a4f1f443231640e9a577701529d13fff Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 31 Oct 2026 16:41:26 -0400 Subject: [PATCH 07/27] Give the OBR its verbs; state when alcohol duty falls below receipts The OBR states each clearance unit, splits Table 4.11's non-cigarette clearances, incorporates the Budget's measures in its forecast and lists them in its policy Table 1.1; tables, notes and forecasts no longer act. On the national construct the model's alcohol duty exceeds the OBR's receipts by 3.5% in 2027-28 and 0.3% in 2028-29 and first falls short in 2029-30, by 2.5%; tobacco duty falls short from 2025-26. The excise page said both fell short from 2027-28. The economic assumptions README says the per-head series keep the old population's values until the importer reruns. Co-Authored-By: Claude Opus 5.5 --- docs/engineering/excise-duties.md | 91 +++++++++++-------- .../gov/economic_assumptions/README.md | 49 +++++----- .../tests/fixtures/obr_receipts/README.md | 4 +- .../tests/test_excise_quantity_projection.py | 15 ++- .../tests/test_import_obr_forecasts.py | 47 +++++----- policyengine_uk/utils/import_obr_forecasts.py | 79 ++++++++-------- 6 files changed, 155 insertions(+), 130 deletions(-) diff --git a/docs/engineering/excise-duties.md b/docs/engineering/excise-duties.md index 89866a7e54..f4ac6ad3de 100644 --- a/docs/engineering/excise-duties.md +++ b/docs/engineering/excise-duties.md @@ -198,8 +198,9 @@ affected. Each alcohol and tobacco quantity input declares its product group's index, and dataset projection grows it by the same group's series in `gov.economic_assumptions.yoy_growth.obr.per_capita` -(`policyengine_uk/data/uprating_indices.yaml`). Each series is the growth in -the OBR's clearances of the group per person (#2596): +(`policyengine_uk/data/uprating_indices.yaml`). The importer writes each +series as the growth in the OBR's clearances of the group per person +(#2596): | Inputs | Series | OBR line | | --- | --- | --- | @@ -210,11 +211,12 @@ the OBR's clearances of the group per person (#2596): | `wine_litres`, `other_fermented_litres` | `alcohol_clearances.wine` | Table 4.12, "Wine", which includes other fermented products | | `spirits_litres` | `alcohol_clearances.spirits` | Table 4.12, "Spirits", which includes spirit-based ready-to-drinks | -Table 4.11 splits non-cigarette clearances into hand-rolled tobacco and other -tobacco products only, so the cigar, other tobacco and heated tobacco inputs -share one series. The strengths (`_abv`) and draught shares are -ratios, and `cigarette_price_per_pack` is a price, so they keep their values; -a forward path for the cigarette price is a separate change. +The OBR splits Table 4.11's non-cigarette clearances into hand-rolled tobacco +and other tobacco products only, so the cigar, other tobacco and heated +tobacco inputs share one series. Dataset projection keeps the strengths +(`_abv`), the draught shares and `cigarette_price_per_pack` at their +base-year values, since the first two measure ratios and the third a price. +A forward path for the cigarette price needs its own change. ### Population @@ -236,10 +238,10 @@ Q(t) = (1 + p(t)) × (1 + c(t)) × Q(t - 1) and count population growth twice. `ons.population` grows 1.54% from 2024-25 to 2031-32, so totals grown that way would end 1.54% above the clearances. -For cigarettes in 2025-26, the October 2026 EFO's outturn of 12,224.6 million -sticks against the March 2026 EFO's 2024-25 outturn of 13,578.2 million is a -fall of 9.97%. Population grows 0.33%, so the series holds -12,224.6 / 13,578.2 / 1.0033 - 1 = -10.27% (-0.102653). +In the OBR's outturn, cigarette clearances fell 9.97%, from 13,578.2 million +sticks in 2024-25 (March 2026 EFO) to 12,224.6 million in 2025-26 (October +2026 EFO). Population grows 0.33% in 2025, so the importer writes +12,224.6 / 13,578.2 / 1.0033 - 1 = -10.27% (-0.102653) for that year. ### Years @@ -251,7 +253,8 @@ fall of 9.97%. Population grows 0.33%, so the series holds | 2032 on | Clearances held flat: 1 / (1 + p(t)) - 1 | As `road_fuel_volume` after the OBR's forecast | The OBR's years run from 1 April and the model's from 6 April. As for road -fuel, each series stores the fiscal year that starts in year t at t-01-01. +fuel, the importer stores the fiscal year that starts in year t at +t-01-01. The indices, `gov.economic_assumptions.indices.obr.per_capita.*`, start at 1 in 2024 and stand at: @@ -269,16 +272,20 @@ the index it declares. `policyengine_uk/utils/import_obr_forecasts.py` writes the six series from `CLEARANCE_SERIES_SPECS` in the same run as the other receipts series (see -the steps for a new EFO in the economic assumptions README). It requires each -table to state its unit, in a row label or a note, and reads `ons.population` -from the `yoy_growth.yaml` it rewrites. +the steps for a new EFO in the economic assumptions README), and reads +`ons.population` from the `yoy_growth.yaml` it rewrites. It finds each line's +unit in exactly one cell, where the OBR states it: the line's own label, the +label of the total the line breaks down, or a note. It stops before writing +anything if a line's row holds a cell beside its label and values, such as a +unit, so it never reads a line's values in a unit that another row states. ### Budget measures and reforms -The October 2026 forecast incorporates the measures announced at Budget 2026 -and since the March forecast (EFO paragraph 3.2). The OBR's policy Table 1.1 -holds no change to an alcohol or tobacco duty rate. Three measures reach the -two duties, and none moves either by more than £13 million a year: +The OBR incorporates in its October 2026 forecast the measures the +government announced at Budget 2026 and since the March forecast (EFO +paragraph 3.2). It lists no change to an alcohol or tobacco duty rate in its +policy Table 1.1, and three measures that change the two duties' receipts, +none by more than £13 million a year: - measure 19, more generous Small Producer Relief from 1 February 2027: alcohol duty £1.5 million lower in 2026-27 and £9.2 million to £9.9 @@ -290,21 +297,22 @@ two duties, and none moves either by more than £13 million a year: imports from October 2028: tobacco duty up to £0.8 million and alcohol duty up to £0.4 million lower a year. -The forecast still assumes rising duty rates. The OBR expects alcohol -receipts to grow, "driven by increases to the duty rate that offset an -assumed average 1 per cent decline in alcohol consumption in each year" -(paragraph 4.43), and tobacco receipts to fall "as higher duty rates only -partly offset falling consumption" (paragraph 4.44). The clearances carry -that consumption trend under the OBR's own rate path, not a Budget change. +The OBR still assumes rising duty rates. It expects alcohol receipts to +grow, "driven by increases to the duty rate that offset an assumed average 1 +per cent decline in alcohol consumption in each year" (paragraph 4.43), and +tobacco receipts to fall "as higher duty rates only partly offset falling +consumption" (paragraph 4.44). So the OBR forecasts the clearances along +that consumption trend, under rate rises it assumes rather than ones the +government announced at the Budget. The dataset projection reads no duty rate (`uprating_input_paths` in -`policyengine_uk/data/economic_assumptions.py`), so the indices grow -quantities whatever rates the model charges. A reform to alcohol or tobacco -duty rates therefore moves receipts through the rates alone: the model -applies no consumption response to it. Current law in the model holds the -alcohol rates at their 1 February 2026 values, the tobacco rates at their -1 October 2026 values and the default cigarette price at its 2024 average, -while the OBR's volumes come with the rates it expects. +`policyengine_uk/data/economic_assumptions.py`), so it grows quantities by +the indices whatever rates the model charges. When a reform changes alcohol +or tobacco duty rates, the model therefore moves receipts through the rates +alone and applies no consumption response. Under current law the model holds +the alcohol rates at their 1 February 2026 values, the tobacco rates at +their 1 October 2026 values and the default cigarette price at its 2024 +average, while the OBR forecasts its volumes with the rates it expects. ### Receipts against the OBR's forecast @@ -319,13 +327,18 @@ with the rate rises of February 2025 and February 2026 for alcohol and of October 2024, November 2025 and October 2026 for tobacco, each of which reaches its first full model year after 2024-25, and then holds. The OBR's receipts per unit cleared keep rising, by 22.1% for alcohol and 48.2% for -tobacco by 2031-32. From 2027-28 the model's duty therefore falls short of -the OBR's forecast by more each year: on this construct, alcohol duty goes -from 7.6% above the OBR's 2024-25 receipts to 6.8% below its 2031-32 -forecast, and tobacco duty from 1.9% above to 18.0% below. The construct's -product mix sets the 2024-25 gaps: charging all beer and cider at the beer -rate with no draught relief puts that line 22.4% above its receipts. A -dataset replaces the mix with its own. The later fall comes from the rates. +tobacco by 2031-32. + +On this construct, the model's alcohol duty exceeds the OBR's receipts by +7.6% in 2024-25 and 7.9% in 2026-27, then by less each year: by 3.5% in +2027-28 and 0.3% in 2028-29. It first falls short in 2029-30, by 2.5%, and +by 6.8% in 2031-32. The model's tobacco duty exceeds the OBR's 2024-25 +receipts by 1.9%, falls short by 0.8% in 2025-26 and 0.4% in 2026-27, and +then by more each year, to 18.0% in 2031-32. Charging all beer and cider at +the beer rate with no draught relief lifts that line 22.4% above its +2024-25 receipts, and a dataset brings its own product mix. From 2027-28 +the model holds its rates while the OBR assumes rising ones, and each gap +moves with that difference. ### Sources diff --git a/policyengine_uk/parameters/gov/economic_assumptions/README.md b/policyengine_uk/parameters/gov/economic_assumptions/README.md index 162c0ef0ed..58438e4b41 100644 --- a/policyengine_uk/parameters/gov/economic_assumptions/README.md +++ b/policyengine_uk/parameters/gov/economic_assumptions/README.md @@ -67,16 +67,17 @@ base. See for each duty's rates, the higher-stake share Machine Games Duty's rate assumes, and the approximations. -The six clearance series under `per_capita` grow the alcohol and tobacco -quantity inputs (#2596): `tobacco_clearances.cigarettes`, -`hand_rolling_tobacco` and `other_tobacco_products`, and -`alcohol_clearances.beer_and_cider`, `wine` and `spirits`. Each is the -growth in the OBR's clearances of a product group over one plus -`ons.population` growth, to six decimals, because dataset projection grows -household weights by `ons.population`. They link their outturn year to the -previous EFO's outturn, as the four receipts series do, and hold the -clearances flat after the forecast. The importer reads `ons.population` from -the `yoy_growth.yaml` it rewrites. See +Dataset projection grows the alcohol and tobacco quantity inputs (#2596) +by the six clearance series under `per_capita`: +`tobacco_clearances.cigarettes`, `hand_rolling_tobacco` and +`other_tobacco_products`, and `alcohol_clearances.beer_and_cider`, `wine` +and `spirits`. The importer writes each as the growth in the OBR's +clearances of a product group over one plus `ons.population` growth, to six +decimals, because dataset projection grows household weights by +`ons.population`. It links each series' outturn year to the previous EFO's +outturn, as it does for the four receipts series, and holds the clearances +flat after the OBR's forecast. It reads `ons.population` from the +`yoy_growth.yaml` it rewrites. See [household excise duties](../../../../docs/engineering/excise-duties.md#projecting-alcohol-and-tobacco-quantities) for the mapping of inputs to groups, the arithmetic and the sources. @@ -485,8 +486,9 @@ the forecast levels, though not their growth. each workbook's release from its contents sheet, or else its file name or URL, and refuses receipts tables from another EFO or of unknown release. Every selected receipts table in both releases must declare `£ billion` - in its header (`£bn` and `£ bn` are also accepted, ignoring case and - whitespace); the clearance tables state their own units (below). + in its header (the importer also accepts `£bn` and `£ bn`, ignoring case + and whitespace); the OBR states each clearance line's unit in a label or + a note (below). Other units, including `£m`, and missing or ambiguous declarations stop the import before any file is written, so different cash magnitudes cannot be combined across releases. @@ -507,15 +509,20 @@ the forecast levels, though not their growth. The same run also refreshes the six alcohol and tobacco clearance series under `per_capita` from the new EFO's tobacco clearances and alcohol clearance forecasts, and the previous EFO's for the outturn year's base. - Each line's table must state its unit, in a row label or a note, in both - EFOs: million sticks for cigarettes, thousand kilograms for the other - tobacco lines, and thousand hectolitres of alcohol for alcohol. The - importer stops on any other. It does not read the lines' footnotes, so - check that each alcohol line covers the same products in both EFOs: - March 2026 added "Includes other fermented products" to the wine line, - which March 2025's lacked. It divides each year's growth by - `ons.population` as `yoy_growth.yaml` holds it, so rerun it whenever - that series changes. Then update the published levels in + The importer finds each line's unit in both EFOs where the OBR states + it: in the line's own label, in the label of the total the line breaks + down, or in a note. It takes million sticks for cigarettes, thousand + kilograms for the other tobacco lines, and thousand hectolitres of + alcohol for alcohol. It stops before writing on any other unit, and on + any cell beside a line's label and values, such as a unit column. It + does not read the lines' footnotes, so check that each alcohol line + covers the same products in both EFOs: in March 2026 the OBR added + "Includes other fermented products" to the wine line. The importer + divides each year's growth by `ons.population` as `yoy_growth.yaml` + holds it, so rerun it whenever that series changes. Until then the six + clearance series and the seven gambling bases keep the values written + for the old population, and the byte-for-byte test fails. Then update + the published levels in `policyengine_uk/tests/test_excise_quantity_projection.py` (`CLEARANCES`) and `policyengine_uk/tests/test_import_obr_forecasts.py` (`OCTOBER_2026_CLEARANCES` and `MARCH_2026_CLEARANCES`, renamed for the diff --git a/policyengine_uk/tests/fixtures/obr_receipts/README.md b/policyengine_uk/tests/fixtures/obr_receipts/README.md index 8322203625..cc5562b490 100644 --- a/policyengine_uk/tests/fixtures/obr_receipts/README.md +++ b/policyengine_uk/tests/fixtures/obr_receipts/README.md @@ -7,8 +7,8 @@ sector (3.7, or 4.7), Current receipts (on a cash basis: 3.9 in March 2025, Alcohol clearance forecasts (3.12, or 4.12, a sheet that also holds the alcohol receipts forecasts) and, from March 2026, Betting and gaming duties forecast breakdown (3.10, or 4.10), plus the original shared strings. The -March 2025 tables have no betting and gaming breakdown. Only the workbook's -sheet list and relationships were reduced to those entries. The published unit cells, fiscal-year headers, markers, row labels and +March 2025 tables have no betting and gaming breakdown. We reduced only the +workbook's sheet list and relationships to those entries. The published unit cells, fiscal-year headers, markers, row labels and numeric cells are unchanged. These are minimal archives for the importer's XLSX reader, rather than complete Excel documents with styles and charts. The October 2026 workbook titles sheet 4.8 "43.8 Current receipts (on a cash diff --git a/policyengine_uk/tests/test_excise_quantity_projection.py b/policyengine_uk/tests/test_excise_quantity_projection.py index 9985d04396..b408bd0e16 100644 --- a/policyengine_uk/tests/test_excise_quantity_projection.py +++ b/policyengine_uk/tests/test_excise_quantity_projection.py @@ -185,7 +185,7 @@ def test_every_quantity_input_declares_its_product_group_index(): def test_strengths_shares_and_the_cigarette_price_stay_as_they_are(): - """Strengths and draught shares are ratios, and the cigarette price is a + """Strengths and draught shares measure ratios, and the cigarette price a price, not a quantity (#2596 leaves its forward path out): none of them declares an index or appears in the uprating map.""" from policyengine_uk.system import system @@ -234,8 +234,8 @@ def test_each_series_is_its_clearances_per_head(parameters): def test_cigarettes_per_head_halve_by_2031(parameters): - """The October 2026 EFO forecasts 2031-32 cigarette clearances 47.2% - below the March 2026 EFO's 2024-25 outturn. Population grows 1.5% over + """In its October 2026 EFO the OBR forecasts 2031-32 cigarette clearances + 47.2% below its March 2026 EFO's 2024-25 outturn. Population grows 1.5% over those years, so clearances per head fall 48.0%, and the index stands at 0.520.""" levels = CLEARANCES["tobacco_clearances.cigarettes"] @@ -349,7 +349,7 @@ def test_a_later_year_charges_duty_on_the_uprated_quantity(parameters): def rate_leaves(parameters): """Every alcohol and tobacco duty rate the duty formulas read, without - the wine strength easement's switch, which is true or false.""" + the wine strength easement's true-or-false switch.""" leaves = [ leaf for node in (ALCOHOL_RATES, TOBACCO_RATES) @@ -371,9 +371,8 @@ def rate_leaves(parameters): def test_a_duty_rate_reform_moves_receipts_by_the_rate_alone(factor, year, scale): """Scaling every alcohol and tobacco duty rate in a year from 2027-28, when the current-law rates hold flat, leaves every projected quantity as - it was and multiplies each duty by the factor: the duties are linear in - their rates, and the rates are not among the parameters the dataset - projection reads.""" + it was and multiplies each duty by the factor: each duty scales linearly + with its rates, and the dataset projection reads no rate.""" parameters = current_law() inputs = { name: value * scale if name in GROUP else value @@ -416,7 +415,7 @@ def test_a_duty_rate_reform_moves_receipts_by_the_rate_alone(factor, year, scale def test_the_change_touches_only_the_quantity_inputs(parameters): """Differential against the projection without this change, which leaves the quantity inputs out of the uprating map and out of Core's - fallback: every other stored input is the same in every year, the + fallback: every other stored input keeps its value in every year, the quantity inputs differ by their index alone, and outputs differ only through the two duties.""" households = [ diff --git a/policyengine_uk/tests/test_import_obr_forecasts.py b/policyengine_uk/tests/test_import_obr_forecasts.py index 6b4bb59291..4ba3ee4cf8 100644 --- a/policyengine_uk/tests/test_import_obr_forecasts.py +++ b/policyengine_uk/tests/test_import_obr_forecasts.py @@ -541,8 +541,8 @@ def clearance_table( """A table laid out as the OBR's Tables 4.11 and 4.12: title in column B, the "Outturn" and "Forecast" markers on the next row, fiscal years on the row after, then one row per line and any notes. A clearance table has no - unit cell, since a label or a note states the unit; ``unit`` adds one on - a row of its own below the title, as a receipts table has.""" + unit cell, since the OBR states the unit in a label or a note; ``unit`` + adds one on a row of its own below the title, as a receipts table has.""" years = max(len(levels) for _, levels in rows) columns = [column_letter(3 + i) for i in range(years)] sheet = {start: [make_inline_cell(f"B{start}", title)]} @@ -606,8 +606,9 @@ def tobacco_table(chapter, first_year, levels, outturn_years=1): def alcohol_table(chapter, first_year, levels, outturn_years=1, note=None): """Table 4.12 (3.12 to March 2026): clearances in thousand hectolitres of - alcohol, which a note states, then the duties' receipts in £ billion - under the same labels, which the clearance lines must not read.""" + alcohol, which the OBR states in a note, then the duties' receipts in + £ billion under the same labels, which the importer must not read for + the clearance lines.""" labels = { "beer_and_cider": " Beer and cider", "wine": " Wine1", @@ -1754,7 +1755,7 @@ def test_rendered_growth_compounds_the_published_levels(years, data): "great_britain": ("32.376814327646052", "34.051963932489258", "37.413586572144581", "38.240930947824609", "39.111850954748434", "42.419067764546902", "42.463193622519263"), "northern_ireland": ("0.38233350730688931", "0.39471664406441903", "0.40263364127921164", "0.41045270712298504", "0.41810894612429189", "0.42590403232705243", "0.43449009714292341"), } # fmt: skip -# Published clearances as the workbooks hold them, from each EFO's outturn +# Clearances as the OBR publishes them in the workbooks, from each EFO's outturn # year: October 2026 Tables 4.11 (rows 5, 8 and 9) and 4.12 (rows 5 to 7) # from 2025-26, and March 2026 Tables 3.11 and 3.12 from 2024-25. OCTOBER_2026_CLEARANCES = { @@ -3049,8 +3050,8 @@ def test_clearance_comments_and_references_name_both_efos_and_the_population(): @pytest.mark.parametrize( "rows, message", [ - # The cigarette label states the unit; hand-rolled tobacco takes it - # from the non-cigarette row. + # The OBR states the unit in the cigarette label, and in the + # non-cigarette total for hand-rolled tobacco. ( [{"B": "Hand rolled tobacco clearances", "C": "3338", "D": "3113"}], r"must state the unit \"\('000s kg\)\" .* found \[\]", @@ -3079,7 +3080,8 @@ def test_a_clearance_line_needs_its_unit_stated_once(rows, message): def test_a_cigarette_line_in_another_unit_is_not_read(): - # The unit sits in the cigarette label, so a new unit renames the row. + # The OBR states the unit in the cigarette label, so a new unit renames + # the row. rows = [TOBACCO_TITLE, *TOBACCO_HEADER] rows.append({"B": "Cigarette clearances (billion sticks)", "C": "12", "D": "11"}) with pytest.raises(ValueError, match="found none"): @@ -3337,14 +3339,15 @@ def test_a_unit_beside_a_clearance_line_is_refused_not_borrowed(): ], "in its fiscal-year row", ), - # The total states the unit, but the line is not one of its parts. + # The OBR states the unit in the non-cigarette total, but no "of + # which:" row puts the line among its parts. ( [ {"C": "2025-26", "D": "2026-27"}, {"B": "Non-cigarette clearances ('000s kg)", "C": "3940", "D": "3683"}, {"B": "Hand rolled tobacco clearances", "C": "3338", "D": "3113"}, ], - "not the line's label, the label of the total it breaks down", + "outside the line's label, the label of the total it breaks down", ), # A total below the line describes other lines. ( @@ -3355,9 +3358,9 @@ def test_a_unit_beside_a_clearance_line_is_refused_not_borrowed(): {"B": "of which:"}, {"B": "Other tobacco products", "C": "602", "D": "571"}, ], - "not the line's label", + "outside the line's label", ), - # A row between the "of which:" row and the line that is not a part. + # A row without values between the "of which:" row and the line. ( [ {"C": "2025-26", "D": "2026-27"}, @@ -3366,16 +3369,16 @@ def test_a_unit_beside_a_clearance_line_is_refused_not_borrowed(): {"B": "Heated tobacco is counted elsewhere"}, {"B": "Hand rolled tobacco clearances", "C": "3338", "D": "3113"}, ], - "not the line's label", + "outside the line's label", ), - # Text that states the unit but is not a note. + # Text that states the unit without opening "Note:". ( [ {"C": "2025-26", "D": "2026-27"}, {"B": "Hand rolled tobacco clearances", "C": "3338", "D": "3113"}, {"B": "Source: HMRC, cigars in ('000s kg)"}, ], - "not the line's label", + "outside the line's label", ), # A unit after the line's last year. ( @@ -3401,8 +3404,8 @@ def test_a_clearance_unit_must_describe_its_line(rows, message): def test_a_unit_note_beside_a_receipts_line_is_refused(): - """A pound line's row holds its label and values alone too: a "£ million" - cell beside it would put its values in another magnitude.""" + """The importer refuses a cell beside a pound line's label and values too: + a "£ million" cell there would put its values in another magnitude.""" rows = [ {"B": "4.7 Property transactions taxes: receipts by sector"}, {"C": "£ billion"}, @@ -3593,9 +3596,9 @@ def test_a_clearance_line_reads_only_in_the_unit_its_table_states( ) -# per_head_growth divides the growth a series holds, a duty's base for a line -# with rates, whether the line links to the previous EFO or has its own -# outturn. +# per_head_growth divides each series' own growth, a duty's base for a line +# with rates, whether the importer links the line to the previous EFO or to +# its own outturn. REMOTE_GAMING = next( spec for spec in RECEIPTS_SERIES_SPECS if spec.path[-1] == "remote_gaming" @@ -3711,8 +3714,8 @@ def test_a_series_growth_before_rounding_must_round_to_its_values(): def test_population_growth_before_its_first_entry_is_that_entry(): """The model backdates a parameter's first value, so a year before ons.population's first entry takes that entry, not zero: wine flat in - 2025 falls 1 / 1.02 - 1 per head when population's first entry, 2026, - is 2%.""" + 2025 falls 1 / 1.02 - 1 per head when population grows 2% in its first + entry, 2026.""" assert growth_in_effect({2026: 0.02, 2030: -0.005}, 2025) == 0.02 assert growth_in_effect({2026: 0.02, 2030: -0.005}, 2029) == 0.02 assert growth_in_effect({2026: 0.02, 2030: -0.005}, 2031) == -0.005 diff --git a/policyengine_uk/utils/import_obr_forecasts.py b/policyengine_uk/utils/import_obr_forecasts.py index 7e4538f178..0bafef16ec 100644 --- a/policyengine_uk/utils/import_obr_forecasts.py +++ b/policyengine_uk/utils/import_obr_forecasts.py @@ -688,10 +688,10 @@ class ClearanceSeriesSpec(ReceiptsSeriesSpec): """A line of the OBR's tobacco or alcohol clearance tables, which grows household quantity inputs per head (#2596). - The line counts quantities, not pounds, so its table has no "£ billion" - cell. ``unit`` is the unit as the table states it, in a row label or a - note, and the importer requires it in exactly one cell of the table in - both EFOs. Dataset projection grows household weights by + The OBR counts clearances in quantities, not pounds, so a clearance + table has no "£ billion" cell. ``unit`` is the unit as the OBR states it + in the table, in a row label or a note, and the importer requires it in + exactly one cell of the table in both EFOs (``validate_clearance_unit``). Dataset projection grows household weights by ``ons.population``, so ``render_receipts_growth_yaml`` writes the clearances' growth over one plus that growth (``per_head_growth``) and the weighted total grows with the clearances. After the OBR's forecast @@ -707,23 +707,25 @@ def __post_init__(self): # per_head_growth reads the previous EFO's line and the raw levels. if self.rates or self.outturn is not None: raise ValueError( - f"{name}: a clearance line counts quantities, so it takes no " - "rates and links to the previous EFO, not its own outturn" + f"{name}: the OBR counts clearances in quantities, so a " + "clearance line takes no rates and links to the previous EFO, " + "not its own outturn" ) TOBACCO_CLEARANCES = "tobacco clearances" ALCOHOL_CLEARANCES = "alcohol clearance forecasts" -# Table 4.11 (3.11 in March 2026) states each unit in a row label: the -# hand-rolled and other tobacco lines split "Non-cigarette clearances ('000s -# kg)". Table 4.12 (3.12) states its unit in a note. +# The OBR states each unit in Table 4.11 (3.11 in March 2026) in a row +# label, and splits "Non-cigarette clearances ('000s kg)" into the +# hand-rolled and other tobacco lines. It states Table 4.12's (3.12's) unit +# in a note. TOBACCO_STICKS = "(million sticks)" TOBACCO_KILOGRAMS = "('000s kg)" ALCOHOL_HECTOLITRES = "thousands of hectolitres of alcohol" -# The product groups the OBR forecasts. Table 4.12's wine line includes other -# fermented products and its spirits line spirit-based ready-to-drinks (its -# footnotes 1 and 2). +# The product groups the OBR forecasts. The OBR includes other fermented +# products in Table 4.12's wine line and spirit-based ready-to-drinks in its +# spirits line (the table's footnotes 1 and 2). CLEARANCE_SERIES_SPECS = ( ClearanceSeriesSpec( path=("per_capita", "tobacco_clearances", "cigarettes"), @@ -1033,9 +1035,9 @@ def growth_ratio( @dataclass(frozen=True) class ReceiptsLine: - """A receipts line as one EFO publishes it: £bn, or a clearance line's own - unit, by the calendar year in which each fiscal year starts, and which of - those years are outturn.""" + """A receipts line as the OBR publishes it in one EFO: £bn, or a + clearance line's own unit, by the calendar year in which each fiscal year + starts, and which of those years are outturn.""" table: str label: str @@ -1248,8 +1250,9 @@ def validate_clearance_unit( ): return raise ValueError( - f'{where} states the unit "{spec.unit}" of {name} in {text!r}, which is ' - "not the line's label, the label of the total it breaks down, or a note" + f'{where} gives the unit "{spec.unit}" of {name} in {text!r}, outside ' + "the line's label, the label of the total it breaks down and the " + "table's notes" ) @@ -1258,9 +1261,9 @@ def find_receipts_line( ) -> ReceiptsLine: """The one row labelled ``spec.row`` in a table titled ``spec.title``. - Only that table's rows are read: its cash unit, or a clearance line's - unit, its year headers and its "Outturn" and "Forecast" markers, which - may sit on one row or several. + The importer reads only that table's rows: its cash unit, or a clearance + line's unit, its year headers and its "Outturn" and "Forecast" markers, + which may sit on one row or several. """ matches = [] for sheet, rows in sheets.items(): @@ -1346,9 +1349,9 @@ def find_receipts_line( f"{where} heads more than one column " + ", ".join(fiscal_year_label(y) for y in repeated) ) - # The line's row holds its label and a value under each year, nothing - # else: a unit or note beside it would describe its values, and the - # importer would read them as if in the table's unit. + # The importer reads a line's label and a value under each year and + # refuses anything else on the row: a unit or note beside the label would + # describe the values, which the importer would read in the table's unit. beside = { column: value for column, value in row.items() @@ -1360,9 +1363,9 @@ def find_receipts_line( "a unit or note on the row needs deliberate importer support" ) if isinstance(spec, ClearanceSeriesSpec): - # A clearance table states each unit in a label or a note, so a cell - # beside its years, such as a unit column's heading, is a layout the - # importer does not read. + # The OBR states each clearance unit in a label or a note, so the + # importer does not read a cell beside the years, such as a unit + # column's heading. heading = { column: value for column, value in year_row.items() @@ -1370,8 +1373,8 @@ def find_receipts_line( } if heading: raise ValueError( - f"{where} holds {heading} in its fiscal-year row; a clearance " - "table states each unit in a label or a note" + f"{where} holds {heading} in its fiscal-year row; the OBR " + "states each clearance unit in a label or a note" ) validate_clearance_unit(table_rows, line_index, spec, where) else: @@ -1516,9 +1519,9 @@ def per_head_growth( """A series' growth per head, to six decimals: one plus each year's growth, over one plus that year's ``ons.population`` growth, minus one. - The growth is the series' own, before rounding where ``growth.ratios`` - holds it: a duty's base for a line with rates, whether it links to the - previous EFO or has its own outturn. Dataset projection grows each + It divides the series' own growth, before rounding where + ``growth.ratios`` holds it: a duty's base for a line with rates, whether + the importer links the line to the previous EFO or to its own outturn. Dataset projection grows each household's input by this and its weight by the population growth, so the weighted input grows by their product, the series' growth. """ @@ -1844,13 +1847,13 @@ def render_receipts_growth_yaml( """Rewrite each receipts series' values and references in a yoy_growth.yaml. - Years before this EFO's first year are kept, so a series' index keeps - its base year. Years after the OBR's receipts forecast follow - average_earnings as ``content`` holds it. A series under ``per_capita`` - is written per head of ``ons.population`` as ``content`` holds it, in - every year to the last that population or the tail states; a clearance - series holds the clearances flat after the forecast. The result is - checked to differ from ``content`` only in those values and references. + It keeps the years before this EFO's first year, so a series' index + keeps its base year, and follows average_earnings as ``content`` holds + it after the OBR's receipts forecast. It writes a series under + ``per_capita`` per head of ``ons.population`` as ``content`` holds it, + in every year to the last that population or the tail states, and holds + a clearance series' clearances flat after the forecast. It checks that + the result differs from ``content`` only in those values and references. """ before = yaml.safe_load(content) expected = copy.deepcopy(before) From f0f3d37e48acddbcba87707c643366a97f75bfbf Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sat, 31 Oct 2026 17:54:52 -0400 Subject: [PATCH 08/27] Wrap two importer docstrings; the importer holds the clearances flat Co-Authored-By: Claude Opus 5.5 --- policyengine_uk/utils/import_obr_forecasts.py | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/policyengine_uk/utils/import_obr_forecasts.py b/policyengine_uk/utils/import_obr_forecasts.py index 0bafef16ec..1586953f78 100644 --- a/policyengine_uk/utils/import_obr_forecasts.py +++ b/policyengine_uk/utils/import_obr_forecasts.py @@ -689,13 +689,15 @@ class ClearanceSeriesSpec(ReceiptsSeriesSpec): household quantity inputs per head (#2596). The OBR counts clearances in quantities, not pounds, so a clearance - table has no "£ billion" cell. ``unit`` is the unit as the OBR states it - in the table, in a row label or a note, and the importer requires it in - exactly one cell of the table in both EFOs (``validate_clearance_unit``). Dataset projection grows household weights by - ``ons.population``, so ``render_receipts_growth_yaml`` writes the - clearances' growth over one plus that growth (``per_head_growth``) and - the weighted total grows with the clearances. After the OBR's forecast - the series holds the clearances flat, as road fuel volume does. + table has no "£ billion" cell. ``unit`` holds the unit as the OBR states + it in the table, in a row label or a note, and the importer requires it + in exactly one cell of the table in both EFOs + (``validate_clearance_unit``). Dataset projection grows household + weights by ``ons.population``, so ``render_receipts_growth_yaml`` writes + the clearances' growth over one plus that growth (``per_head_growth``) + and the weighted total grows with the clearances. After the OBR's + forecast the importer holds the clearances flat, as ``road_fuel_volume`` + holds road fuel volume. """ unit: str = "" @@ -1521,9 +1523,10 @@ def per_head_growth( It divides the series' own growth, before rounding where ``growth.ratios`` holds it: a duty's base for a line with rates, whether - the importer links the line to the previous EFO or to its own outturn. Dataset projection grows each - household's input by this and its weight by the population growth, so - the weighted input grows by their product, the series' growth. + the importer links the line to the previous EFO or to its own outturn. + Dataset projection grows each household's input by this and its weight + by the population growth, so the weighted input grows by their product, + the series' growth. """ ratios = growth.ratios if ratios is None: From 58111769adebee3dca263391990105e4f81507d1 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 1 Nov 2026 02:10:52 -0500 Subject: [PATCH 09/27] Read clearance units as a class and scope notes by product The importer reads every clearance unit the OBR states in a table against a closed list (sticks, kg, litres or hectolitres of alcohol or of product, at the scales it knows) and compares units by measure and size. It reads a unit as a line's in the line's label, its total's label, another line's label that names its product, or a note that names its product or no product. It refuses a line when such a cell states another unit or scale, when a note states two units it cannot tie to products, when a note or a line's cell holds a quantity word outside the list, or when it finds the unit only in a note about another product. Each spec names its products. Tests: both round-2 witnesses refused by message; notes about other products; respellings that agree; a Hypothesis property over labels and notes in two releases. Test docstrings drop reviewer names. Co-Authored-By: Claude Opus 5.5 --- docs/engineering/excise-duties.md | 30 +- .../tests/test_import_obr_forecasts.py | 588 +++++++++++++++++- policyengine_uk/utils/import_obr_forecasts.py | 423 +++++++++++-- 3 files changed, 962 insertions(+), 79 deletions(-) diff --git a/docs/engineering/excise-duties.md b/docs/engineering/excise-duties.md index f4ac6ad3de..0ebb12d5c1 100644 --- a/docs/engineering/excise-duties.md +++ b/docs/engineering/excise-duties.md @@ -270,14 +270,28 @@ in 2024 and stand at: Past `PROJECTION_END_YEAR` (2032), policyengine-core scales each input by the index it declares. -`policyengine_uk/utils/import_obr_forecasts.py` writes the six series from -`CLEARANCE_SERIES_SPECS` in the same run as the other receipts series (see -the steps for a new EFO in the economic assumptions README), and reads -`ons.population` from the `yoy_growth.yaml` it rewrites. It finds each line's -unit in exactly one cell, where the OBR states it: the line's own label, the -label of the total the line breaks down, or a note. It stops before writing -anything if a line's row holds a cell beside its label and values, such as a -unit, so it never reads a line's values in a unit that another row states. +The importer, `policyengine_uk/utils/import_obr_forecasts.py`, writes the +six series from `CLEARANCE_SERIES_SPECS` in the same run as the other +receipts series (see the steps for a new EFO in the economic assumptions +README), and reads `ons.population` from the `yoy_growth.yaml` it rewrites. +It reads every unit the OBR states in a clearance table against a closed +list: sticks, kg, and litres or hectolitres of alcohol or of product, each +after an optional number and a scale such as "'000s", "thousands of" or +"million". It compares units by measure and size, so "'000s kg" and +"thousand kg" agree. It reads a unit as a line's when the OBR states it in +the line's own label, in the label of the total the line breaks down, in +another line's label that names the line's product, or in a note that names +the line's product or no product, and ignores a note that names only other +products. It stops before writing anything when: + +- the OBR states another unit or scale for a line; +- the OBR states two units in one note, and the importer cannot tie each to + the products named in its own clause; +- the OBR uses a word for a quantity or scale outside the list, such as + "tonnes", in a note or in a cell the importer reads as a line's; +- the OBR states a line's unit in none of its label, its total's label and a + note beginning "Note:" that names its product or no product; +- the OBR puts any cell beside a line's label and values. ### Budget measures and reforms diff --git a/policyengine_uk/tests/test_import_obr_forecasts.py b/policyengine_uk/tests/test_import_obr_forecasts.py index 4ba3ee4cf8..079333d163 100644 --- a/policyengine_uk/tests/test_import_obr_forecasts.py +++ b/policyengine_uk/tests/test_import_obr_forecasts.py @@ -16,7 +16,9 @@ from policyengine_uk.utils import import_obr_forecasts from policyengine_uk.utils.import_obr_forecasts import ( + ALCOHOL_CLEARANCES, CLEARANCE_SERIES_SPECS, + TOBACCO_CLEARANCES, RECEIPTS_SERIES_SPECS, SERIES_SPECS, STATUTORY_GAP_SPECS, @@ -41,6 +43,7 @@ per_head_growth, population_growth, read_calendar_values, + read_sheet_rows, receipts_label, read_workbook_sheets, render_forecast_gap_yaml, @@ -3047,6 +3050,12 @@ def test_clearance_comments_and_references_name_both_efos_and_the_population(): ] +UNIT_NEEDED = ( + "the importer needs the unit in its label, the label of the total it " + "breaks down or a note that applies to it, and found it only in " +) + + @pytest.mark.parametrize( "rows, message", [ @@ -3054,26 +3063,38 @@ def test_clearance_comments_and_references_name_both_efos_and_the_population(): # non-cigarette total for hand-rolled tobacco. ( [{"B": "Hand rolled tobacco clearances", "C": "3338", "D": "3113"}], - r"must state the unit \"\('000s kg\)\" .* found \[\]", + r"cannot read per_capita\.tobacco_clearances\.hand_rolling_tobacco in " + r"\"\('000s kg\)\" from Table 4\.11 \(sheet 4\.11\): " + + re.escape(UNIT_NEEDED) + + r"\[\]", ), + # A label naming the line's product in a unit the importer does not + # know. ( [ {"B": "Non-cigarette clearances (tonnes)", "C": "3.9", "D": "3.7"}, {"B": "Hand rolled tobacco clearances", "C": "3338", "D": "3113"}, ], - "must state the unit", + re.escape( + 'the importer does not know the unit or scale "tonnes" in ' + "'non-cigarette clearances (tonnes)'" + ), ), + # Without an "of which:" row, no total breaks the line down. ( [ {"B": "Non-cigarette clearances ('000s kg)", "C": "3940", "D": "3683"}, {"B": "Cigars ('000s kg)", "C": "50", "D": "45"}, {"B": "Hand rolled tobacco clearances", "C": "3338", "D": "3113"}, ], - "in exactly one cell", + re.escape( + UNIT_NEEDED + + '["non-cigarette clearances (\'000s kg)", "cigars (\'000s kg)"]' + ), ), ], ) -def test_a_clearance_line_needs_its_unit_stated_once(rows, message): +def test_a_clearance_line_needs_its_unit_where_the_obr_states_it(rows, message): sheet = {"4.11": [TOBACCO_TITLE, *TOBACCO_HEADER, *rows]} with pytest.raises(ValueError, match=message): find_receipts_line(sheet, CLEARANCE_SPECS["hand_rolling_tobacco"]) @@ -3107,6 +3128,12 @@ def test_a_clearance_spec_needs_a_unit_and_takes_no_rates_or_outturn(): replace(spec, rates=GAMBLING_SPECS[0].rates) with pytest.raises(ValueError, match="takes no rates"): replace(spec, outturn=GAMBLING_SPECS[0].outturn) + with pytest.raises(ValueError, match=r'does not know "\(tonnes\)" as one'): + replace(spec, unit="(tonnes)") + with pytest.raises(ValueError, match='does not know "million sticks or kg"'): + replace(spec, unit="million sticks or kg") + with pytest.raises(ValueError, match="needs the names a note may give"): + replace(spec, products=()) def test_real_october_2026_clearance_series(): @@ -3263,7 +3290,7 @@ def test_rendered_clearances_grow_the_weighted_total_with_the_clearances(years, def mixed_unit_workbook(rows): - """A one-sheet workbook of inline strings, as Sol's review built it.""" + """Build a one-sheet workbook of inline strings.""" table = rows[0]["B"].split()[0] return make_workbook( { @@ -3281,10 +3308,10 @@ def mixed_unit_workbook(rows): def test_a_unit_beside_a_clearance_line_is_refused_not_borrowed(): - """Sol's minimal case: the previous release holds other tobacco in - thousand kg, the current one in kg in a cell beside the line, while its - non-cigarette total still states thousand kg. Read as one unit, the line - grows 999-fold; the importer refuses it.""" + """The previous release holds other tobacco in thousand kg, the current + one in kg in a cell beside the line, while its non-cigarette total still + states thousand kg. Read as one unit, the line has growth of 999.0; the + importer refuses it.""" previous = mixed_unit_workbook( [ {"B": "3.11 Tobacco clearances"}, @@ -3347,9 +3374,9 @@ def test_a_unit_beside_a_clearance_line_is_refused_not_borrowed(): {"B": "Non-cigarette clearances ('000s kg)", "C": "3940", "D": "3683"}, {"B": "Hand rolled tobacco clearances", "C": "3338", "D": "3113"}, ], - "outside the line's label, the label of the total it breaks down", + re.escape(UNIT_NEEDED + '["non-cigarette clearances (\'000s kg)"]'), ), - # A total below the line describes other lines. + # A total below the line breaks down other lines. ( [ {"C": "2025-26", "D": "2026-27"}, @@ -3358,7 +3385,7 @@ def test_a_unit_beside_a_clearance_line_is_refused_not_borrowed(): {"B": "of which:"}, {"B": "Other tobacco products", "C": "602", "D": "571"}, ], - "outside the line's label", + re.escape(UNIT_NEEDED + '["non-cigarette clearances (\'000s kg)"]'), ), # A row without values between the "of which:" row and the line. ( @@ -3369,16 +3396,28 @@ def test_a_unit_beside_a_clearance_line_is_refused_not_borrowed(): {"B": "Heated tobacco is counted elsewhere"}, {"B": "Hand rolled tobacco clearances", "C": "3338", "D": "3113"}, ], - "outside the line's label", + re.escape(UNIT_NEEDED + '["non-cigarette clearances (\'000s kg)"]'), ), # Text that states the unit without opening "Note:". ( [ {"C": "2025-26", "D": "2026-27"}, {"B": "Hand rolled tobacco clearances", "C": "3338", "D": "3113"}, - {"B": "Source: HMRC, cigars in ('000s kg)"}, + {"B": "Source: HMRC, all in ('000s kg)"}, + ], + re.escape(UNIT_NEEDED + '["source: hmrc, all in (\'000s kg)"]'), + ), + # A note that states the unit for other products only. + ( + [ + {"C": "2025-26", "D": "2026-27"}, + {"B": "Hand rolled tobacco clearances", "C": "3338", "D": "3113"}, + {"B": "Note: Cigars are in ('000s kg)."}, ], - "outside the line's label", + re.escape( + "the importer found the unit only in " + '["note: cigars are in (\'000s kg)."], which name other products' + ), ), # A unit after the line's last year. ( @@ -3424,10 +3463,9 @@ def test_a_unit_note_beside_a_receipts_line_is_refused(): def put_unit_beside_real_line(workbook, table, label, unit, scale=1000): - """Sol's change to a real table: shift the sheet's cells from column C, - row 3, one column right, put ``unit`` in C beside the line labelled - ``label`` and scale that line's values. Every other archive entry, - including the shared strings, stays.""" + """Shift the sheet's cells from column C, row 3, one column right, put + ``unit`` in C beside the line labelled ``label`` and scale that line's + values. Every other archive entry, including the shared strings, stays.""" output = BytesIO() with ZipFile(BytesIO(workbook)) as source, ZipFile(output, "w") as dest: path = _sheet_paths(source)[table] @@ -3468,11 +3506,11 @@ def put_unit_beside_real_line(workbook, table, label, unit, scale=1000): def test_real_tables_with_a_kg_cell_beside_a_line_are_refused(tmp_path): - """The real October 2026 Table 4.11 changed as Sol's review changed it: - other tobacco in kg, stated beside the line, with its values 1,000 - times larger. Read in the total's thousand kg, the line would grow - 862-fold. The renderer refuses it, and a run on the real March 2026 - tables changed the same way stops before it writes anything.""" + """The helper writes other tobacco in kg beside its line in the real + October 2026 Table 4.11 and multiplies its values by 1,000. Reading them + in the total's thousand kg gives growth of 862.8112. The renderer refuses + it, and a run on the real March 2026 tables changed the same way stops + before it writes anything.""" changed = put_unit_beside_real_line( REAL_OCTOBER_2026_RECEIPTS, "4.11", "Other tobacco products", "kg" ) @@ -3584,7 +3622,11 @@ def test_a_clearance_line_reads_only_in_the_unit_its_table_states( parts = ("hand_rolling_tobacco", "other_tobacco_products") if key in parts and total != "('000s kg)": event("refused: total in another unit") - with pytest.raises(ValueError, match="must state the unit"): + with pytest.raises( + ValueError, + match=rf"cannot read {re.escape('.'.join(spec.path))} in .*" + rf"[\"']non-cigarette clearances {re.escape(total)}[\"']", + ): find_receipts_line(sheets, spec) return event("read") @@ -3596,6 +3638,500 @@ def test_a_clearance_line_reads_only_in_the_unit_its_table_states( ) +def rewrite_real_table(workbook, table, labels=(), scale=(), notes=()): + """Set the text of the cells in ``labels`` (as {"B6": text}), multiply + the values on each row in ``scale`` (as {6: 1000}) and add each of + ``notes`` in column B below the sheet's last row. Every other archive + entry, including the shared strings, stays.""" + labels, scale = dict(labels), dict(scale) + output = BytesIO() + with ZipFile(BytesIO(workbook)) as source, ZipFile(output, "w") as dest: + path = _sheet_paths(source)[table] + sheet = ET.fromstring(source.read(path)) + data = sheet.find("m:sheetData", XLSX_NS) + for cell in sheet.iter(f"{{{XLSX_MAIN}}}c"): + reference = cell.get("r") + row = int(reference.lstrip("ABCDEFGHIJKLMNOPQRSTUVWXYZ")) + if reference in labels: + for child in list(cell): + cell.remove(child) + cell.set("t", "inlineStr") + ET.SubElement( + ET.SubElement(cell, f"{{{XLSX_MAIN}}}is"), f"{{{XLSX_MAIN}}}t" + ).text = labels.pop(reference) + elif row in scale and cell.get("t") != "s": + value = cell.find("m:v", XLSX_NS) + if value is not None: + value.text = repr(float(value.text) * scale[row]) + assert not labels, labels + last = max(int(row.get("r")) for row in data) + for offset, note in enumerate(notes, start=1): + row = ET.SubElement(data, f"{{{XLSX_MAIN}}}row", {"r": str(last + offset)}) + cell = ET.SubElement( + row, + f"{{{XLSX_MAIN}}}c", + {"r": f"B{last + offset}", "t": "inlineStr"}, + ) + ET.SubElement( + ET.SubElement(cell, f"{{{XLSX_MAIN}}}is"), f"{{{XLSX_MAIN}}}t" + ).text = note + for entry in source.infolist(): + dest.writestr( + entry, + ET.tostring(sheet) if entry.filename == path else source.read(entry), + ) + return output.getvalue() + + +def real_levels(workbook, key): + """The levels the importer reads for ``key`` from its table alone in + ``workbook``, and from the real October 2026 table.""" + spec = CLEARANCE_SPECS[key] + table = "4.11" if spec.title == TOBACCO_CLEARANCES else "4.12" + return ( + find_receipts_line({table: read_sheet_rows(workbook, table)}, spec).levels, + find_receipts_line(REAL_OCTOBER_2026_SHEETS, spec).levels, + ) + + +def test_a_unit_note_cannot_override_the_total_a_line_reads_in(): + """The helper relabels the real October 2026 Table 4.11's non-cigarette + total in kg, multiplies its values and other tobacco's by 1,000, and + adds a note that puts hand-rolled tobacco in thousand kg and other + tobacco in kg. Read in the thousand kg the March 2026 table states, + other tobacco would have growth of 862.8112 in 2025. The total's label + and the note both state kg for other tobacco, the total kg for + hand-rolled tobacco too, so the importer refuses both lines and the + renderer refuses the tables; cigarettes read as before.""" + changed = rewrite_real_table( + REAL_OCTOBER_2026_RECEIPTS, + "4.11", + labels={"B6": "Non-cigarette clearances (kg)"}, + scale={6: 1000, 9: 1000}, + notes=[ + "Note: Hand rolled tobacco clearances are in ('000s kg); other " + "tobacco products are in kg." + ], + ) + (line,) = [ + row + for row in read_workbook_sheets(changed)["4.11"] + if receipts_label(row.get("B")) == "other tobacco products" + ] + previous = float(MARCH_2026_CLEARANCES["other_tobacco_products"][0]) + assert round(float(line["C"]) / previous - 1, 6) == 862.8112 + note = ( + "\"note: hand rolled tobacco clearances are in ('000s kg); other " + 'tobacco products are in kg."' + ) + with pytest.raises(ValueError) as refused: + compute_receipts_growth( + changed, + REAL_RECEIPTS[2026], + specs=(CLEARANCE_SPECS["other_tobacco_products"],), + ) + assert str(refused.value) == ( + "The importer cannot read " + 'per_capita.tobacco_clearances.other_tobacco_products in "(\'000s kg)" ' + 'from Table 4.11 (sheet 4.11): the OBR states "kg", another unit or ' + "scale, in 'non-cigarette clearances (kg)'; the OBR states \"kg\", " + f"another unit or scale, in {note}" + ) + with pytest.raises( + ValueError, + match=r"hand_rolling_tobacco in .*the OBR states \"kg\", another unit or " + r"scale, in 'non-cigarette clearances \(kg\)'", + ): + render( + get_yoy_growth_path().read_text(), + current=changed, + previous=REAL_RECEIPTS[2026], + **COMMITTED_RELEASES, + ) + read, real = real_levels(changed, "cigarettes") + assert read == real + + +@pytest.mark.parametrize( + "key, previous_rows, current_rows, message", + [ + # The label states million sticks, and a note that names no product + # puts the forecasts in thousand sticks. + ( + "cigarettes", + [ + {"B": "3.11 Tobacco clearances"}, + {"C": "Outturn"}, + {"C": "2024-25"}, + {"B": "Cigarette clearances (million sticks)", "C": 600}, + ], + [ + {"B": "4.11 Tobacco clearances"}, + {"C": "Outturn", "D": "Forecast"}, + {"C": "2025-26", "D": "2026-27"}, + {"B": "Cigarette clearances (million sticks)", "C": 600, "D": 600000}, + { + "B": "Note: Outturn clearances use the unit in the label; " + "forecasts are in thousand sticks." + }, + ], + "The importer cannot read per_capita.tobacco_clearances.cigarettes " + 'in "(million sticks)" from Table 4.11 (sheet 4.11): the OBR states ' + '"thousand sticks", another unit or scale, in \'note: outturn ' + "clearances use the unit in the label; forecasts are in thousand " + "sticks.'", + ), + # The total states thousand kg, and a note on other tobacco states kg. + ( + "other_tobacco_products", + [ + {"B": "3.11 Tobacco clearances"}, + {"C": "Outturn"}, + {"C": "2024-25"}, + {"B": "Non-cigarette clearances ('000s kg)", "C": 4000}, + {"B": "of which:"}, + {"B": "Other tobacco products", "C": 600}, + ], + [ + {"B": "4.11 Tobacco clearances"}, + {"C": "Outturn", "D": "Forecast"}, + {"C": "2025-26", "D": "2026-27"}, + {"B": "Non-cigarette clearances ('000s kg)", "C": 4000, "D": 3900}, + {"B": "of which:"}, + {"B": "Other tobacco products", "C": 600000, "D": 570000}, + {"B": "Note: Other tobacco products are in kg."}, + ], + "The importer cannot read " + "per_capita.tobacco_clearances.other_tobacco_products in " + '"(\'000s kg)" from Table 4.11 (sheet 4.11): the OBR states "kg", ' + "another unit or scale, in 'note: other tobacco products are in kg.'", + ), + ], +) +def test_a_note_that_describes_a_line_must_agree_with_its_label( + key, previous_rows, current_rows, message +): + """Read in its label's or its total's unit, either line would have growth + of 999.0 for clearances that did not change. A note that names the + line's product, or no product, describes the line, so the importer + refuses the line even though its label or its total states the line's + unit.""" + spec = CLEARANCE_SPECS[key] + with pytest.raises(ValueError) as refused: + compute_receipts_growth( + mixed_unit_workbook(current_rows), + mixed_unit_workbook(previous_rows), + specs=(spec,), + ) + assert str(refused.value) == message + + +def test_a_note_about_another_product_says_nothing_about_a_line(): + """A note on the real October 2026 Table 4.11 that puts hand-rolled + tobacco in million sticks describes hand-rolled tobacco alone: the + importer refuses that line, and reads cigarettes in their label's unit + and other tobacco in its total's, as before.""" + changed = rewrite_real_table( + REAL_OCTOBER_2026_RECEIPTS, + "4.11", + notes=["Note: Hand rolled tobacco clearances are in million sticks."], + ) + for key in ("cigarettes", "other_tobacco_products"): + read, real = real_levels(changed, key) + assert read == real, key + with pytest.raises(ValueError) as refused: + real_levels(changed, "hand_rolling_tobacco") + assert str(refused.value).endswith( + 'hand_rolling_tobacco in "(\'000s kg)" from Table 4.11 (sheet 4.11): ' + 'the OBR states "million sticks", another unit or scale, in ' + "'note: hand rolled tobacco clearances are in million sticks.'" + ) + + +def test_a_unit_only_in_a_note_about_another_product_is_refused(): + """With Table 4.12's note rewritten to state the unit for wine alone, the + real October 2026 table states no unit for beer and cider or for + spirits: the importer refuses both and reads wine as before.""" + note = "Note: Wine clearances are in thousands of hectolitres of alcohol." + changed = rewrite_real_table( + REAL_OCTOBER_2026_RECEIPTS, "4.12", labels={"B8": note} + ) + read, real = real_levels(changed, "wine") + assert read == real + for key in ("beer_and_cider", "spirits"): + with pytest.raises(ValueError) as refused: + real_levels(changed, key) + assert str(refused.value) == ( + f"The importer cannot read {'.'.join(CLEARANCE_SPECS[key].path)} in " + '"thousands of hectolitres of alcohol" from Table 4.12 (sheet 4.12): ' + "the importer found the unit only in ['note: wine clearances are in " + "thousands of hectolitres of alcohol.'], which name other products" + ) + + +@pytest.mark.parametrize( + "table, cell, text, refused", + [ + ("4.11", "B6", "Non-cigarette clearances (thousand kg)", None), + ("4.11", "B6", "Non-cigarette clearances (thousands of kilograms)", None), + ( + "4.12", + "B8", + "Note: All clearances are in '000s hectolitres of alcohol.", + None, + ), + ( + "4.12", + "B8", + "Note: All clearances are in hundred thousand litres of alcohol.", + None, + ), + ("4.11", "B6", "Non-cigarette clearances (million kg)", "million kg"), + ( + "4.12", + "B8", + "Note: All clearances are in hundreds of hectolitres of alcohol.", + "hundreds of hectolitres of alcohol", + ), + ( + "4.12", + "B8", + "Note: All clearances are in thousands of hectolitres of product.", + "thousands of hectolitres of product", + ), + ], +) +def test_a_clearance_unit_reads_by_measure_and_size(table, cell, text, refused): + """The importer compares units by measure and size: the same unit spelt + another way reads the real October 2026 lines as before, and another + size or measure is refused.""" + changed = rewrite_real_table(REAL_OCTOBER_2026_RECEIPTS, table, labels={cell: text}) + keys = [ + key + for key, spec in CLEARANCE_SPECS.items() + if (spec.title == TOBACCO_CLEARANCES) == (table == "4.11") + and key != "cigarettes" + ] + for key in keys: + if refused is None: + read, real = real_levels(changed, key) + assert read == real, key + continue + with pytest.raises(ValueError, match=f'states "{refused}", another unit'): + real_levels(changed, key) + + +# The unit each spelling states, as (measure, size): what the property below +# expects, written out rather than read by the importer. None marks a unit +# the importer does not know. +TABLE_UNITS = { + "million sticks": ("sticks", 10**6), + "millions of sticks": ("sticks", 10**6), + "thousand sticks": ("sticks", 10**3), + "'000s kg": ("kg", 10**3), + "thousand kg": ("kg", 10**3), + "thousands of kilograms": ("kg", 10**3), + "kg": ("kg", 1), + "million kg": ("kg", 10**6), + "thousands of hectolitres of alcohol": ("litres of alcohol", 10**5), + "thousand hectolitres of alcohol": ("litres of alcohol", 10**5), + "hectolitres of alcohol": ("litres of alcohol", 10**2), + "millions of litres of alcohol": ("litres of alcohol", 10**6), + "thousands of hectolitres of product": ("litres of product", 10**5), + "tonnes": None, + "grams": None, + "'000s tonnes": None, + "litres": None, + "hogsheads": None, + "thousands of hectolitres of pure alcohol": None, +} +# In a note only: it names cigarettes, which a label's scope would change. +NOTE_UNITS = {**TABLE_UNITS, "million cigarettes": None} +EXPECTED_UNITS = { + "(million sticks)": ("sticks", 10**6), + "('000s kg)": ("kg", 10**3), + "thousands of hectolitres of alcohol": ("litres of alcohol", 10**5), +} +# The names a note gives each product, and the lines they describe. +NOTE_SUBJECTS = { + TOBACCO_CLEARANCES: { + "Cigarette": {"cigarettes"}, + "Hand-rolled tobacco": {"hand_rolling_tobacco"}, + "Other tobacco products": {"other_tobacco_products"}, + "Cigars": {"other_tobacco_products"}, + "Non-cigarette products": {"hand_rolling_tobacco", "other_tobacco_products"}, + }, + ALCOHOL_CLEARANCES: { + "Beer and cider": {"beer_and_cider"}, + "Wine": {"wine"}, + "Other fermented products": {"wine"}, + "Spirits": {"spirits"}, + "Alcopops": {"spirits"}, + }, +} + + +@st.composite +def clearance_release(draw, key): + """A table's total and notes for one release, around the OBR's layout: + the label of a non-cigarette total, with or without its "of which:" row, + and up to two notes, each opening "Note:" or "Source: HMRC.", of one or + two clauses that each put some products, or all clearances, in a unit. + Half the draws start from the OBR's own unit cell: the non-cigarette + total's label in thousand kg, or Table 4.12's note.""" + spec = CLEARANCE_SPECS[key] + equivalent = [ + spelling + for spelling, unit in TABLE_UNITS.items() + if unit == EXPECTED_UNITS[spec.unit] + ] + unit = st.one_of(st.sampled_from(equivalent), st.sampled_from(sorted(NOTE_UNITS))) + total = of_which = None + published = draw(st.booleans()) + if spec.title == TOBACCO_CLEARANCES and published: + total, of_which = "'000s kg", True + elif spec.title == TOBACCO_CLEARANCES: + total = draw( + st.none() + | st.sampled_from(equivalent if key != "cigarettes" else ["'000s kg"]) + | st.sampled_from(sorted(TABLE_UNITS)) + ) + of_which = total is not None and draw(st.booleans()) + clause = st.tuples( + st.lists( + st.sampled_from(sorted(NOTE_SUBJECTS[spec.title])), max_size=2, unique=True + ), + unit, + ) + notes = draw( + st.lists( + st.tuples( + st.sampled_from(["Note:", "Source: HMRC."]), + st.lists(clause, min_size=1, max_size=2), + ), + max_size=2, + ) + ) + if spec.title == ALCOHOL_CLEARANCES and published: + notes = [("Note:", [([], "thousands of hectolitres of alcohol")]), *notes] + return total, of_which, notes + + +def clearance_release_rows(key, total, of_which, notes, first_year, levels): + """The table's rows for ``clearance_release``'s draw.""" + spec = CLEARANCE_SPECS[key] + chapter = 3 if first_year == 2024 else 4 + columns = "CD"[: len(levels)] + if spec.title == TOBACCO_CLEARANCES: + labels = ["Cigarette clearances (million sticks)"] + if total is not None: + labels.append(f"Non-cigarette clearances ({total})") + if of_which: + labels.append("of which:") + labels += ["Hand rolled tobacco clearances", "Other tobacco products"] + title = f"{chapter}.11 Tobacco clearances" + else: + labels = ["Beer and cider", "Wine1", "Spirits2"] + title = f"{chapter}.12 Alcohol clearance forecasts" + rows = [ + {"B": title}, + {"C": "Outturn", **({"D": "Forecast"} if len(levels) > 1 else {})}, + { + column: f"{first_year + i}-{str(first_year + i + 1)[2:]}" + for i, column in enumerate(columns) + }, + ] + for label in labels: + values = {} if label == "of which:" else dict(zip(columns, levels)) + rows.append({"B": label, **values}) + for prefix, clauses in notes: + rows.append( + { + "B": f"{prefix} " + + "; ".join( + f"{' and '.join(subjects) or 'All clearances'} are in {unit}" + for subjects, unit in clauses + ) + + "." + } + ) + if spec.title == ALCOHOL_CLEARANCES: + rows += [ + {"B": "1 Includes other fermented products such as fruit ciders."}, + {"B": "2 Includes spirit-based ‘ready-to-drinks’ such as alcopops."}, + ] + return rows + + +def clearance_release_reads(key, total, of_which, notes): + """Whether a release's table gives ``key`` its unit, written out from the + draw rather than read from the table's text: every unit that describes + the line is its unit, one of them sits where the OBR states it, every + note's units are ones the importer knows, and a note that states two + units ties each to products.""" + spec = CLEARANCE_SPECS[key] + expected = EXPECTED_UNITS[spec.unit] + described = [] # (unit, whether its cell is one that can state it) + if key == "cigarettes": + described.append((expected, True)) + if total is not None and key in ("hand_rolling_tobacco", "other_tobacco_products"): + described.append((TABLE_UNITS[total], bool(of_which))) + for prefix, clauses in notes: + units = [NOTE_UNITS[unit] for _, unit in clauses] + if None in units: + return False + lines = [ + set().union(*(NOTE_SUBJECTS[spec.title][s] for s in subjects)) + for subjects, _ in clauses + ] + if len(set(units)) > 1: + if not all(lines): + return False + ties = list(zip(units, lines)) + else: + ties = [(unit, set().union(*lines)) for unit in units] + described += [ + (unit, prefix == "Note:") + for unit, products in ties + if not products or key in products + ] + return all(unit == expected for unit, _ in described) and any( + can_state for _, can_state in described + ) + + +@settings(max_examples=200, deadline=None, derandomize=True) +@given(key=st.sampled_from(sorted(CLEARANCE_SPECS)), data=st.data()) +def test_a_clearance_line_reads_only_when_both_releases_state_its_unit(key, data): + """For any labels and notes in two releases, the importer reads a line + exactly when every unit each release states for it is the line's unit + and each states it where the OBR does. So it never reads a pair of + releases whose units disagree for the line, and whenever it reads one, + the growth is the clearances'.""" + previous = data.draw(clearance_release(key), label="previous") + current = data.draw(clearance_release(key), label="current") + spec = CLEARANCE_SPECS[key] + reads = clearance_release_reads(key, *previous) and clearance_release_reads( + key, *current + ) + try: + growth = compute_receipts_growth( + mixed_unit_workbook( + clearance_release_rows(key, *current, 2025, (1000, 950)) + ), + mixed_unit_workbook(clearance_release_rows(key, *previous, 2024, (1000,))), + specs=(spec,), + )[spec.path] + except ValueError as error: + event("refused") + assert not reads, str(error) + assert "cannot read" in str(error) + return + event("read") + assert reads + assert growth.values == {2025: 0.0, 2026: -0.05} + + # per_head_growth divides each series' own growth, a duty's base for a line # with rates, whether the importer links the line to the previous EFO or to # its own outturn. @@ -3606,7 +4142,7 @@ def test_a_clearance_line_reads_only_in_the_unit_its_table_states( def test_per_head_growth_divides_the_base_growth_the_series_holds(): - """Sol's minimal case: Remote Gaming Duty receipts double while the base, + """Remote Gaming Duty receipts double while the base, receipts over the rate, grows 10%, and population grows 1%. Per head the base grows 1.10 / 1.01 - 1, not 2 / 1.01 - 1 (receipts) or the growth the levels and the model's rates give, with or without a previous diff --git a/policyengine_uk/utils/import_obr_forecasts.py b/policyengine_uk/utils/import_obr_forecasts.py index d718d985c0..e8b71f5424 100644 --- a/policyengine_uk/utils/import_obr_forecasts.py +++ b/policyengine_uk/utils/import_obr_forecasts.py @@ -8,6 +8,7 @@ from dataclasses import dataclass, replace from datetime import date from decimal import Decimal +from fractions import Fraction from functools import lru_cache from io import BytesIO from pathlib import Path @@ -691,10 +692,11 @@ class ClearanceSeriesSpec(ReceiptsSeriesSpec): The OBR counts clearances in quantities, not pounds, so a clearance table has no "£ billion" cell. ``unit`` holds the unit as the OBR states - it in the table, in a row label or a note, and the importer requires it - in exactly one cell of the table in both EFOs - (``validate_clearance_unit``). Dataset projection grows household - weights by ``ons.population``, so ``render_receipts_growth_yaml`` writes + it in the table, in a row label or a note, and ``products`` the names a + note may give the line's product, lower case and without hyphens. In + both EFOs the importer requires every unit the OBR states for the line + to be this one (``validate_clearance_unit``). Dataset projection grows + household weights by ``ons.population``, so ``render_receipts_growth_yaml`` writes the clearances' growth over one plus that growth (``per_head_growth``) and the weighted total grows with the clearances. After the OBR's forecast the importer holds the clearances flat, as ``road_fuel_volume`` @@ -702,12 +704,23 @@ class ClearanceSeriesSpec(ReceiptsSeriesSpec): """ unit: str = "" + products: tuple[str, ...] = () def __post_init__(self): name = ".".join(self.path) if not self.unit: raise ValueError(f"{name}: a clearance line needs a unit") - # per_head_growth reads the previous EFO's line and the raw levels. + units, unknown = read_clearance_units(self.unit) + if len(units) != 1 or unknown: + raise ValueError( + f'{name}: the importer does not know "{self.unit}" as one ' + "clearance unit" + ) + if not self.products: + raise ValueError( + f"{name}: a clearance line needs the names a note may give its product" + ) + # The importer links clearance outturns to the previous EFO's outturn. if self.rates or self.outturn is not None: raise ValueError( f"{name}: the OBR counts clearances in quantities, so a " @@ -716,6 +729,195 @@ def __post_init__(self): ) +# Clearance units as a class (#2596). The importer knows a count of sticks, +# a mass in kg and a volume of alcohol or of product in litres or +# hectolitres, each after an optional number and any of the scales below, +# and compares two units by measure and size: "'000s kg" and "thousand kg" +# agree, "thousand sticks" and "million sticks" do not. It reads any other +# word for a quantity or a scale, such as "tonnes", "grams", "litres" +# without "of alcohol" or "of product", or a scale before no unit it knows, +# as a unit it does not know, which needs deliberate importer support. +CLEARANCE_SCALES = ( + (r"'?000'?s?", 1_000), + (r"thousands?", 1_000), + (r"hundreds?", 100), + (r"millions?|mn", 1_000_000), + (r"billions?|bn", 1_000_000_000), +) +CLEARANCE_MEASURES = ( + (r"sticks?", "sticks", 1), + (r"kgs?|kilogram(?:me)?s?", "kg", 1), + (r"(?:litre|liter)s? of alcohol", "litres of alcohol", 1), + (r"(?:hectolitre|hectoliter)s? of alcohol", "litres of alcohol", 100), + (r"(?:litre|liter)s? of product", "litres of product", 1), + (r"(?:hectolitre|hectoliter)s? of product", "litres of product", 100), +) +OTHER_QUANTITY_WORDS = ( + r"grams?|grammes?|g|mg|kilos?|tonnes?|tons?|lbs?|pounds?|ounces?|oz" + r"|(?:hecto|kilo|deca|deci|centi|milli)?(?:litre|liter)s?" + r"|hl|ml|cl|kl|l|hla|lpa|gallons?|pints?|barrels?" + r"|packs?|packets?|cartons?|pouches?|units|pence|per ?cent" +) +# The importer reads a number or a scale only after something other than a +# letter, digit, point, comma, apostrophe or hyphen, and a measure or another +# quantity word only after something other than a letter or a point, so it +# reads "10kg" and skips the "g" of "e.g.". +_CLEARANCE_SCALE = ( + r"(?\d[\d,]*(?:\.\d+)?)\s*)?" + rf"(?P(?:{_CLEARANCE_SCALE}(?:\s+of)?\s*)*)" + rf"(?P{_CLEARANCE_MEASURE})" +) +QUANTITY_WORD_RE = re.compile( + rf"{_CLEARANCE_SCALE}|(? str: + """A cell's text as the importer reads units and products in it: lower + case, single spaces and straight apostrophes.""" + return normalise_label(value).translate({0x2018: "'", 0x2019: "'"}) + + +def clearance_unit_size(match: re.Match) -> tuple[str, Fraction]: + """A unit's measure and size, such as ("kg", 1000) for "'000s kg".""" + number = match.group("number") + size = Fraction(number.replace(",", "")) if number else Fraction(1) + for scale in re.finditer(_CLEARANCE_SCALE, match.group("scales")): + size *= next( + factor + for pattern, factor in CLEARANCE_SCALES + if re.fullmatch(pattern, scale.group(0)) + ) + measure, factor = next( + (measure, factor) + for pattern, measure, factor in CLEARANCE_MEASURES + if re.fullmatch(pattern, match.group("measure")) + ) + return measure, size * factor + + +def read_clearance_units( + value: str | None, +) -> tuple[list[tuple[tuple[str, Fraction], str, int, int]], list[str]]: + """Each unit the importer knows in a cell's text, as (measure and size, + spelling, start, end), and each word for a quantity or scale outside + them: a word ``QUANTITY_WORD_RE`` matches outside every known unit, or + what follows "is in" or "are in" when no known unit starts there.""" + text = clearance_text(value) + units = [ + (clearance_unit_size(match), match.group(0), match.start(), match.end()) + for match in CLEARANCE_UNIT_RE.finditer(text) + ] + unknown = [ + match.group(0) + for match in QUANTITY_WORD_RE.finditer(text) + if not any( + start <= match.start() and match.end() <= end for _, _, start, end in units + ) + ] + for match in UNIT_STATEMENT_RE.finditer(text): + if not any(start == match.end() for _, _, start, _ in units): + unknown.append(CLAUSE_END_RE.split(text[match.end() :])[0].strip()) + return units, unknown + + +@dataclass(frozen=True) +class ClearanceStatement: + """The units the importer reads in a cell of a clearance table. + + ``units`` holds each unit the OBR states in the cell, as (measure and + size, spelling, the paths of the lines the importer ties it to, or + ``None`` for every line); ``products`` the paths of the lines whose + products the cell names, or ``None``; ``unknown`` each word for a + quantity or scale the importer does not know; ``untied`` whether the + importer cannot tie each of two or more units to products.""" + + units: tuple[tuple[tuple[str, Fraction], str, frozenset | None], ...] + products: frozenset | None + unknown: tuple[str, ...] + untied: bool + + +def read_clearance_statement( + value: str | None, products: dict[str, frozenset[tuple[str, ...]]] +) -> ClearanceStatement: + """Read the units in a cell, given ``products``, which maps each name a + cell may give a line's product to the lines' paths. + + When the OBR states one unit in a cell, however often, the importer ties + it to the lines whose products the cell names, or to every line when the + cell names none. When the OBR states two or more, the importer ties each + to the products named in its own clause, up to a semicolon or a full + stop, and cannot tie a clause with two units or no product. + """ + units, unknown = read_clearance_units(value) + text = list(clearance_text(value)) + for _, _, start, end in units: + text[start:end] = " " * (end - start) + text = "".join(text).replace("-", " ") + if not products: + raise ValueError("a clearance statement needs the names of products") + pattern = re.compile( + r"(? frozenset | None: + found = frozenset().union( + *(paths for position, paths in named if start <= position < end) + ) + return found or None + + whole = lines(0, len(text)) + if len({size for size, _, _, _ in units}) <= 1: + return ClearanceStatement( + units=tuple((size, spelling, whole) for size, spelling, _, _ in units), + products=whole, + unknown=tuple(unknown), + untied=False, + ) + tied = [] + start = 0 + for end in [match.end() for match in CLAUSE_END_RE.finditer(text)] + [len(text)]: + clause = [unit for unit in units if start <= unit[2] < end] + described = lines(start, end) + if clause and (described is None or len({unit[0] for unit in clause}) > 1): + return ClearanceStatement( + units=tuple(tied), products=whole, unknown=tuple(unknown), untied=True + ) + tied += [(size, spelling, described) for size, spelling, _, _ in clause] + start = end + return ClearanceStatement( + units=tuple(tied), products=whole, unknown=tuple(unknown), untied=False + ) + + TOBACCO_CLEARANCES = "tobacco clearances" ALCOHOL_CLEARANCES = "alcohol clearance forecasts" # The OBR states each unit in Table 4.11 (3.11 in March 2026) in a row @@ -728,43 +930,59 @@ def __post_init__(self): # The product groups the OBR forecasts. The OBR includes other fermented # products in Table 4.12's wine line and spirit-based ready-to-drinks in its -# spirits line (the table's footnotes 1 and 2). +# spirits line (the table's footnotes 1 and 2). It splits Table 4.11's +# non-cigarette total into hand-rolled tobacco and other tobacco products, +# so the importer reads "non-cigarette" as naming both, and other tobacco +# products as every non-cigarette product but hand-rolled tobacco, such as +# cigars and heated tobacco. CLEARANCE_SERIES_SPECS = ( ClearanceSeriesSpec( path=("per_capita", "tobacco_clearances", "cigarettes"), title=TOBACCO_CLEARANCES, row="cigarette clearances (million sticks)", unit=TOBACCO_STICKS, + products=("cigarettes", "cigarette"), ), ClearanceSeriesSpec( path=("per_capita", "tobacco_clearances", "hand_rolling_tobacco"), title=TOBACCO_CLEARANCES, row="hand rolled tobacco clearances", unit=TOBACCO_KILOGRAMS, + products=("hand rolled", "hand rolling", "non cigarette"), ), ClearanceSeriesSpec( path=("per_capita", "tobacco_clearances", "other_tobacco_products"), title=TOBACCO_CLEARANCES, row="other tobacco products", unit=TOBACCO_KILOGRAMS, + products=( + "other tobacco", + "cigars", + "cigar", + "heated tobacco", + "non cigarette", + ), ), ClearanceSeriesSpec( path=("per_capita", "alcohol_clearances", "beer_and_cider"), title=ALCOHOL_CLEARANCES, row="beer and cider", unit=ALCOHOL_HECTOLITRES, + products=("beer", "beers", "cider", "ciders"), ), ClearanceSeriesSpec( path=("per_capita", "alcohol_clearances", "wine"), title=ALCOHOL_CLEARANCES, row="wine", unit=ALCOHOL_HECTOLITRES, + products=("wine", "wines", "other fermented", "fruit cider", "fruit ciders"), ), ClearanceSeriesSpec( path=("per_capita", "alcohol_clearances", "spirits"), title=ALCOHOL_CLEARANCES, row="spirits", unit=ALCOHOL_HECTOLITRES, + products=("spirits", "spirit", "ready to drinks", "alcopops"), ), ) @@ -1204,13 +1422,15 @@ def validate_clearance_unit( table_rows: list[dict[str, str | None]], line_index: int, spec: ClearanceSeriesSpec, + years: dict[str, int], where: str, ) -> None: - """Require a clearance line's unit in exactly one cell of its table, and - that cell to describe the line. + """Require every unit the OBR states for a clearance line to be the + line's unit, and the unit where the OBR states it. - The OBR states the unit in one of three places, and only these describe - the line: + The importer reads every unit of its closed list (``CLEARANCE_UNIT_RE``) + wherever the OBR states it in the table, and every word for a quantity + or scale outside the list. It reads a unit as the line's in: - the line's own label, as Table 4.11's "Cigarette clearances (million sticks)"; @@ -1218,46 +1438,159 @@ def validate_clearance_unit( which:" row, with only the total's other parts between that and the line, as "Non-cigarette clearances ('000s kg)" for hand-rolled and other tobacco; - - a note on a row of its own, as Table 4.12's "Note: All clearances are - in thousands of hectolitres of alcohol." - - The first year's growth divides one EFO's level by another's, so both - must state the same unit there. A unit anywhere else, such as another - line's label or a unit column, describes something else; a new unit or + - another line's label, when it names this line's product + (``spec.products``); + - any other cell, such as a note, when it names this line's product or + no product, as Table 4.12's "Note: All clearances are in thousands of + hectolitres of alcohol". A note that names only other products says + nothing about this line. + + The importer needs the unit where the OBR states it: in the line's + label, the total's label or a note on a row of its own that opens + "Note:". It refuses the line when the OBR states another unit or scale + for it, or a word for a quantity or scale the importer does not know in + a cell the importer reads as the line's; when the OBR puts such a word + in any note, heading or footnote, or states two units there that the + importer cannot tie each to products in its own clause; and when the + importer finds the unit only somewhere else, such as a note about + another product. The first year's growth divides one EFO's level by + another's, so the OBR must state the same unit in both; a new unit or layout needs deliberate importer support. """ name = ".".join(spec.path) - found = [ - (index, text) - for index, row in enumerate(table_rows) - for _, text in row_cells(row) - if spec.unit in text - ] - if len(found) != 1: - raise ValueError( - f'{where} must state the unit "{spec.unit}" of {name} in exactly ' - f"one cell; found {[text for _, text in found]}" - ) - index, text = found[0] - cells = row_cells(table_rows[index]) - if index == line_index: - return - if len(cells) == 1 and text.startswith("note"): - return - between = table_rows[index + 2 : line_index] - if ( - index + 1 < line_index - and cells[0][1] == text - and [cell for _, cell in row_cells(table_rows[index + 1])] == ["of which:"] - and all(len(row_cells(row)) > 1 for row in between) + ((expected, _, _, _),), _ = read_clearance_units(spec.unit) + products: dict[str, frozenset] = {} + for line_spec in ( + *(other for other in CLEARANCE_SERIES_SPECS if other.title == spec.title), + spec, ): - return - raise ValueError( - f'{where} gives the unit "{spec.unit}" of {name} in {text!r}, outside ' - "the line's label, the label of the total it breaks down and the " - "table's notes" + for product in line_spec.products: + products[product] = products.get(product, frozenset()) | {line_spec.path} + first_year_column = min(column_index(column) for column in years) + year_rows = { + index for index, row in enumerate(table_rows) if _fiscal_year_columns(row) + } + + def is_line(index: int) -> bool: + cells = row_cells(table_rows[index]) + return ( + index > min(year_rows) + and index not in year_rows + and bool(cells) + and column_index(cells[0][0]) < first_year_column + and any(column in years for column, _ in cells) + ) + + lines = {index for index in range(len(table_rows)) if is_line(index)} + above = line_index - 1 + while above in lines: + above -= 1 + total = ( + above - 1 + if above - 1 in lines + and [text for _, text in row_cells(table_rows[above])] == ["of which:"] + else None ) + problems = [] + found = [] + other_products = [] + supplied = False + + def check(text: str, units, unknown) -> list: + """Each problem with the units a cell states for the line, and the + units that agree.""" + problems.extend( + f'the importer does not know the unit or scale "{word}" in {text!r}' + for word in dict.fromkeys(unknown) + ) + problems.extend( + f'the OBR states "{spelling}", another unit or scale, in {text!r}' + for size, spelling, _ in units + if size != expected + ) + return [unit for unit in units if unit[0] == expected] + + def unknown_in_brackets(text: str, unknown: list[str]) -> list[str]: + # The OBR states a label's unit in brackets, so the importer reads + # anything else there as a unit it does not know. + return [ + group + for group in re.findall(r"\(([^()]*)\)", clearance_text(text)) + if not CLEARANCE_UNIT_RE.fullmatch(group) + and not any(word in group for word in unknown) + ] + + # The line's own label, and the label of the total it breaks down. + for index in (line_index, total): + if index is None: + continue + text = row_cells(table_rows[index])[0][1] + units, unknown = read_clearance_units(text) + agree = check( + text, + [(size, spelling, None) for size, spelling, _, _ in units], + [*unknown, *unknown_in_brackets(text, unknown)], + ) + supplied = supplied or bool(agree) + + # Every other cell: another line's label, a heading, a note or a + # footnote. + for index, row in enumerate(table_rows): + if index in (line_index, total) or index in year_rows: + continue + cells = row_cells(row) + for _, text in cells[:1] if index in lines else cells: + statement = read_clearance_statement(text, products) + described = [ + unit + for unit in statement.units + if unit[2] is None or spec.path in unit[2] + ] + if index in lines: + # The importer reads another line's label as that line's, and + # as this line's only when it names this line's product. + if statement.products is None or spec.path not in statement.products: + if any(unit[0] == expected for unit in statement.units): + found.append(text) + continue + unknown = [ + *statement.unknown, + *unknown_in_brackets(text, statement.unknown), + ] + else: + unknown = statement.unknown + if statement.untied: + problems.append( + f"the importer cannot tie each unit in {text!r} to products " + "in its own clause" + ) + agree = check(text, described, unknown) + if agree and len(cells) == 1 and text.startswith("note"): + supplied = True + elif agree: + found.append(text) + elif any(unit[0] == expected for unit in statement.units): + other_products.append(text) + + if not supplied and not problems: + if other_products and not found: + problems.append( + f"the importer found the unit only in {other_products}, which " + "name other products" + ) + else: + problems.append( + "the importer needs the unit in its label, the label of the " + "total it breaks down or a note that applies to it, and found it " + f"only in {found + other_products}" + ) + if problems: + raise ValueError( + f'The importer cannot read {name} in "{spec.unit}" from {where}: ' + + "; ".join(problems) + ) + def find_receipts_line( sheets: dict[str, list[dict[str, str | None]]], spec: ReceiptsSeriesSpec @@ -1379,7 +1712,7 @@ def find_receipts_line( f"{where} holds {heading} in its fiscal-year row; the OBR " "states each clearance unit in a label or a note" ) - validate_clearance_unit(table_rows, line_index, spec, where) + validate_clearance_unit(table_rows, line_index, spec, years, where) else: validate_receipts_unit(header_rows, where) levels = {} From ade1df1ce9b52aea5add9ae25fd7d955a32e1f0c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 1 Nov 2026 02:16:12 -0500 Subject: [PATCH 10/27] Write the per-head formula as (1 + growth) / (1 + population) - 1 The README, the clearance spec's docstring and two test docstrings read as growth over one plus population growth; they now state one plus the growth, over one plus population growth, minus one. Co-Authored-By: Claude Opus 5.5 --- .../parameters/gov/economic_assumptions/README.md | 15 ++++++++------- .../tests/test_excise_quantity_projection.py | 7 ++++--- .../tests/test_import_obr_forecasts.py | 7 ++++--- policyengine_uk/utils/import_obr_forecasts.py | 7 ++++--- 4 files changed, 20 insertions(+), 16 deletions(-) diff --git a/policyengine_uk/parameters/gov/economic_assumptions/README.md b/policyengine_uk/parameters/gov/economic_assumptions/README.md index a57917ad42..4e73526ba2 100644 --- a/policyengine_uk/parameters/gov/economic_assumptions/README.md +++ b/policyengine_uk/parameters/gov/economic_assumptions/README.md @@ -72,13 +72,14 @@ Dataset projection grows the alcohol and tobacco quantity inputs (#2596) by the six clearance series under `per_capita`: `tobacco_clearances.cigarettes`, `hand_rolling_tobacco` and `other_tobacco_products`, and `alcohol_clearances.beer_and_cider`, `wine` -and `spirits`. The importer writes each as the growth in the OBR's -clearances of a product group over one plus `ons.population` growth, to six -decimals, because dataset projection grows household weights by -`ons.population`. It links each series' outturn year to the previous EFO's -outturn, as it does for the four receipts series, and holds the clearances -flat after the OBR's forecast. It reads `ons.population` from the -`yoy_growth.yaml` it rewrites. See +and `spirits`. From the OBR's clearances of each product group, the +importer writes +`(1 + aggregate clearance growth) / (1 + ons.population growth) - 1` to six +decimals. Dataset projection grows household weights by `ons.population`, +so the weighted quantities grow with the clearances. The importer links each +series' outturn year to the previous EFO's outturn, as it does for the four +receipts series, holds the clearances flat after the OBR's forecast, and +reads `ons.population` from the `yoy_growth.yaml` it rewrites. See [household excise duties](../../../../docs/engineering/excise-duties.md#projecting-alcohol-and-tobacco-quantities) for the mapping of inputs to groups, the arithmetic and the sources. diff --git a/policyengine_uk/tests/test_excise_quantity_projection.py b/policyengine_uk/tests/test_excise_quantity_projection.py index b408bd0e16..e30866ac45 100644 --- a/policyengine_uk/tests/test_excise_quantity_projection.py +++ b/policyengine_uk/tests/test_excise_quantity_projection.py @@ -3,9 +3,10 @@ Each input declares its product group's index under ``gov.economic_assumptions.indices.obr.per_capita``, and dataset projection -grows it by the same ``yoy_growth`` series: the growth in the OBR's clearances -of the group (October 2026 receipts Tables 4.11 and 4.12, after the March 2026 -EFO's 2024-25 outturn) over one plus ``ons.population`` growth. Dataset +grows it by the same ``yoy_growth`` series: one plus the growth in the OBR's +clearances of the group (October 2026 receipts Tables 4.11 and 4.12, after the +March 2026 EFO's 2024-25 outturn), over one plus ``ons.population`` growth, +minus one. Dataset projection grows household weights by ``ons.population``, so the weighted quantities grow with the clearances, and a duty-rate reform moves receipts through the rate alone. diff --git a/policyengine_uk/tests/test_import_obr_forecasts.py b/policyengine_uk/tests/test_import_obr_forecasts.py index 079333d163..e8cbc3c7ba 100644 --- a/policyengine_uk/tests/test_import_obr_forecasts.py +++ b/policyengine_uk/tests/test_import_obr_forecasts.py @@ -2984,9 +2984,10 @@ def test_clearance_lines_are_read_from_their_own_tables(): def test_clearance_growth_is_per_head_of_the_population_the_file_holds(): - """Each year's growth is the clearances' growth over one plus that year's - ons.population growth in the file being written; after the forecast the - clearances hold flat, so the series is 1 / (1 + population) - 1.""" + """The importer writes each year's value as one plus the clearances' + growth, over one plus that year's ons.population growth in the file it + writes, minus one; after the forecast it holds the clearances flat and + writes 1 / (1 + population) - 1.""" rendered = render() for key, path in CLEARANCE_PATHS.items(): published = { diff --git a/policyengine_uk/utils/import_obr_forecasts.py b/policyengine_uk/utils/import_obr_forecasts.py index e8b71f5424..ec85a672d3 100644 --- a/policyengine_uk/utils/import_obr_forecasts.py +++ b/policyengine_uk/utils/import_obr_forecasts.py @@ -696,9 +696,10 @@ class ClearanceSeriesSpec(ReceiptsSeriesSpec): note may give the line's product, lower case and without hyphens. In both EFOs the importer requires every unit the OBR states for the line to be this one (``validate_clearance_unit``). Dataset projection grows - household weights by ``ons.population``, so ``render_receipts_growth_yaml`` writes - the clearances' growth over one plus that growth (``per_head_growth``) - and the weighted total grows with the clearances. After the OBR's + household weights by ``ons.population``, so ``render_receipts_growth_yaml`` + writes one plus the clearances' growth, over one plus that growth, minus + one (``per_head_growth``), and the weighted total grows with the + clearances. After the OBR's forecast the importer holds the clearances flat, as ``road_fuel_volume`` holds road fuel volume. """ From 65f3b3f4aedd3b41d4869d3b21b7d7320c6baf39 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 1 Nov 2026 08:14:53 -0500 Subject: [PATCH 11/27] Check clearance tables against a closed list of read texts The importer no longer reads units or products from a clearance table's words. CLEARANCE_TABLE_TEXTS holds, for each table, the text of every cell in the March 2025, March 2026 and October 2026 tables, and the importer refuses any other text from the title's row to the next table's title (CLEARANCE_TABLE_ENDS), in every column, naming the table and the cell. Each line also needs the exact cell that states its unit. The unit parser, its product aliases and their tests go. New tests cover each witness class on the real tables, the list against the three releases, the receipts table below the alcohol clearances, and a property that inserts any text anywhere in a table. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_import_obr_forecasts.py | 1209 ++++++++--------- policyengine_uk/utils/import_obr_forecasts.py | 605 ++------- 2 files changed, 675 insertions(+), 1139 deletions(-) diff --git a/policyengine_uk/tests/test_import_obr_forecasts.py b/policyengine_uk/tests/test_import_obr_forecasts.py index e8cbc3c7ba..0749ce3767 100644 --- a/policyengine_uk/tests/test_import_obr_forecasts.py +++ b/policyengine_uk/tests/test_import_obr_forecasts.py @@ -2,6 +2,7 @@ import re from dataclasses import replace from datetime import date +from functools import lru_cache from pathlib import Path from io import BytesIO from textwrap import indent @@ -10,7 +11,7 @@ import pytest import yaml -from hypothesis import event, given, settings +from hypothesis import assume, event, given, settings from hypothesis import strategies as st from policyengine_uk.utils import import_obr_forecasts @@ -18,6 +19,8 @@ from policyengine_uk.utils.import_obr_forecasts import ( ALCOHOL_CLEARANCES, CLEARANCE_SERIES_SPECS, + CLEARANCE_TABLE_ENDS, + CLEARANCE_TABLE_TEXTS, TOBACCO_CLEARANCES, RECEIPTS_SERIES_SPECS, SERIES_SPECS, @@ -3051,56 +3054,6 @@ def test_clearance_comments_and_references_name_both_efos_and_the_population(): ] -UNIT_NEEDED = ( - "the importer needs the unit in its label, the label of the total it " - "breaks down or a note that applies to it, and found it only in " -) - - -@pytest.mark.parametrize( - "rows, message", - [ - # The OBR states the unit in the cigarette label, and in the - # non-cigarette total for hand-rolled tobacco. - ( - [{"B": "Hand rolled tobacco clearances", "C": "3338", "D": "3113"}], - r"cannot read per_capita\.tobacco_clearances\.hand_rolling_tobacco in " - r"\"\('000s kg\)\" from Table 4\.11 \(sheet 4\.11\): " - + re.escape(UNIT_NEEDED) - + r"\[\]", - ), - # A label naming the line's product in a unit the importer does not - # know. - ( - [ - {"B": "Non-cigarette clearances (tonnes)", "C": "3.9", "D": "3.7"}, - {"B": "Hand rolled tobacco clearances", "C": "3338", "D": "3113"}, - ], - re.escape( - 'the importer does not know the unit or scale "tonnes" in ' - "'non-cigarette clearances (tonnes)'" - ), - ), - # Without an "of which:" row, no total breaks the line down. - ( - [ - {"B": "Non-cigarette clearances ('000s kg)", "C": "3940", "D": "3683"}, - {"B": "Cigars ('000s kg)", "C": "50", "D": "45"}, - {"B": "Hand rolled tobacco clearances", "C": "3338", "D": "3113"}, - ], - re.escape( - UNIT_NEEDED - + '["non-cigarette clearances (\'000s kg)", "cigars (\'000s kg)"]' - ), - ), - ], -) -def test_a_clearance_line_needs_its_unit_where_the_obr_states_it(rows, message): - sheet = {"4.11": [TOBACCO_TITLE, *TOBACCO_HEADER, *rows]} - with pytest.raises(ValueError, match=message): - find_receipts_line(sheet, CLEARANCE_SPECS["hand_rolling_tobacco"]) - - def test_a_cigarette_line_in_another_unit_is_not_read(): # The OBR states the unit in the cigarette label, so a new unit renames # the row. @@ -3113,28 +3066,28 @@ def test_a_cigarette_line_in_another_unit_is_not_read(): def test_an_alcohol_table_in_another_unit_is_refused_before_any_write(tmp_path): yoy, gap_dir = write_tree(tmp_path) before = {path: path.read_bytes() for path in [yoy, *gap_dir.iterdir()]} - receipts = make_receipts_xlsx( - alcohol_note="Note: All clearances are in millions of litres of alcohol." - ) - with pytest.raises(ValueError, match="thousands of hectolitres of alcohol"): + note = "Note: All clearances are in millions of litres of alcohol." + receipts = make_receipts_xlsx(alcohol_note=note) + with pytest.raises(ValueError, match=re.escape(f"not read: {note!r} in column B")): run_main(tmp_path, make_test_xlsx(), "--yaml-path", str(yoy), receipts=receipts) assert {path: path.read_bytes() for path in before} == before -def test_a_clearance_spec_needs_a_unit_and_takes_no_rates_or_outturn(): - with pytest.raises(ValueError, match="needs a unit"): - ClearanceSeriesSpec(path=("per_capita", "x"), title="t", row="r") +def test_a_clearance_spec_needs_a_read_unit_cell_and_takes_no_rates_or_outturn(): spec = CLEARANCE_SPECS["wine"] + with pytest.raises(ValueError, match="needs the cell that states its unit"): + ClearanceSeriesSpec(path=("per_capita", "x"), title=ALCOHOL_CLEARANCES, row="r") + with pytest.raises(ValueError, match=r"and '\(tonnes\)' is not one"): + replace(spec, unit_cell="(tonnes)") + # Each table's list holds its own cells. + with pytest.raises(ValueError, match="is not one"): + replace(spec, unit_cell=CLEARANCE_SPECS["cigarettes"].unit_cell) + with pytest.raises(ValueError, match="is not one"): + replace(spec, title="alcohol receipts forecasts") with pytest.raises(ValueError, match="takes no rates"): replace(spec, rates=GAMBLING_SPECS[0].rates) with pytest.raises(ValueError, match="takes no rates"): replace(spec, outturn=GAMBLING_SPECS[0].outturn) - with pytest.raises(ValueError, match=r'does not know "\(tonnes\)" as one'): - replace(spec, unit="(tonnes)") - with pytest.raises(ValueError, match='does not know "million sticks or kg"'): - replace(spec, unit="million sticks or kg") - with pytest.raises(ValueError, match="needs the names a note may give"): - replace(spec, products=()) def test_real_october_2026_clearance_series(): @@ -3275,19 +3228,8 @@ def test_rendered_clearances_grow_the_weighted_total_with_the_clearances(years, assert render(rendered, current=current, previous=previous) == rendered -# A unit stated beside a line, or anywhere that does not describe it, must -# stop the run: the importer would otherwise read the line's values in the -# unit some other cell states. - -TOBACCO_UNITS = ( - "kg", - "'000s kg", - "('000s kg)", - "tonnes", - "(million sticks)", - "£ billion", - "thousands of hectolitres of alcohol", -) +# A unit beside a line must stop the run: the importer would otherwise read +# the line's values in the unit another cell states. def mixed_unit_workbook(rows): @@ -3334,7 +3276,7 @@ def test_a_unit_beside_a_clearance_line_is_refused_not_borrowed(): ] ) spec = CLEARANCE_SPECS["other_tobacco_products"] - with pytest.raises(ValueError, match=r"holds \{'C': 'kg'\} beside"): + with pytest.raises(ValueError, match="not read: 'kg' in column C"): compute_receipts_growth(current, previous, specs=(spec,)) # Without the kg cell the same layout reads. reconciled = mixed_unit_workbook( @@ -3354,95 +3296,6 @@ def test_a_unit_beside_a_clearance_line_is_refused_not_borrowed(): ) -@pytest.mark.parametrize( - "rows, message", - [ - # A unit column: its heading sits in the fiscal-year row. - ( - [ - {"C": "Unit", "D": "2025-26", "E": "2026-27"}, - {"B": "Non-cigarette clearances ('000s kg)", "D": "3940", "E": "3683"}, - {"B": "of which:"}, - {"B": "Hand rolled tobacco clearances", "D": "3338", "E": "3113"}, - ], - "in its fiscal-year row", - ), - # The OBR states the unit in the non-cigarette total, but no "of - # which:" row puts the line among its parts. - ( - [ - {"C": "2025-26", "D": "2026-27"}, - {"B": "Non-cigarette clearances ('000s kg)", "C": "3940", "D": "3683"}, - {"B": "Hand rolled tobacco clearances", "C": "3338", "D": "3113"}, - ], - re.escape(UNIT_NEEDED + '["non-cigarette clearances (\'000s kg)"]'), - ), - # A total below the line breaks down other lines. - ( - [ - {"C": "2025-26", "D": "2026-27"}, - {"B": "Hand rolled tobacco clearances", "C": "3338", "D": "3113"}, - {"B": "Non-cigarette clearances ('000s kg)", "C": "3940", "D": "3683"}, - {"B": "of which:"}, - {"B": "Other tobacco products", "C": "602", "D": "571"}, - ], - re.escape(UNIT_NEEDED + '["non-cigarette clearances (\'000s kg)"]'), - ), - # A row without values between the "of which:" row and the line. - ( - [ - {"C": "2025-26", "D": "2026-27"}, - {"B": "Non-cigarette clearances ('000s kg)", "C": "3940", "D": "3683"}, - {"B": "of which:"}, - {"B": "Heated tobacco is counted elsewhere"}, - {"B": "Hand rolled tobacco clearances", "C": "3338", "D": "3113"}, - ], - re.escape(UNIT_NEEDED + '["non-cigarette clearances (\'000s kg)"]'), - ), - # Text that states the unit without opening "Note:". - ( - [ - {"C": "2025-26", "D": "2026-27"}, - {"B": "Hand rolled tobacco clearances", "C": "3338", "D": "3113"}, - {"B": "Source: HMRC, all in ('000s kg)"}, - ], - re.escape(UNIT_NEEDED + '["source: hmrc, all in (\'000s kg)"]'), - ), - # A note that states the unit for other products only. - ( - [ - {"C": "2025-26", "D": "2026-27"}, - {"B": "Hand rolled tobacco clearances", "C": "3338", "D": "3113"}, - {"B": "Note: Cigars are in ('000s kg)."}, - ], - re.escape( - "the importer found the unit only in " - '["note: cigars are in (\'000s kg)."], which name other products' - ), - ), - # A unit after the line's last year. - ( - [ - {"C": "2025-26", "D": "2026-27"}, - {"B": "Non-cigarette clearances ('000s kg)", "C": "3940", "D": "3683"}, - {"B": "of which:"}, - { - "B": "Hand rolled tobacco clearances", - "C": "3338", - "D": "3113", - "E": "tonnes", - }, - ], - r"holds \{'E': 'tonnes'\} beside", - ), - ], -) -def test_a_clearance_unit_must_describe_its_line(rows, message): - sheet = {"4.11": [TOBACCO_TITLE, {"C": "Outturn", "D": "Forecast"}, *rows]} - with pytest.raises(ValueError, match=message): - find_receipts_line(sheet, CLEARANCE_SPECS["hand_rolling_tobacco"]) - - def test_a_unit_note_beside_a_receipts_line_is_refused(): """The importer refuses a cell beside a pound line's label and values too: a "£ million" cell there would put its values in another magnitude.""" @@ -3524,7 +3377,7 @@ def test_real_tables_with_a_kg_cell_beside_a_line_are_refused(tmp_path): assert line["D"] == repr( float(OCTOBER_2026_CLEARANCES["other_tobacco_products"][0]) * 1000 ) - with pytest.raises(ValueError, match=r"holds \{'C': 'kg'\} beside"): + with pytest.raises(ValueError, match="not read: 'kg' in column C"): render( get_yoy_growth_path().read_text(), current=changed, @@ -3533,7 +3386,7 @@ def test_real_tables_with_a_kg_cell_beside_a_line_are_refused(tmp_path): ) yoy, gap_dir = write_tree(tmp_path) before = {path: path.read_bytes() for path in [yoy, *gap_dir.iterdir()]} - with pytest.raises(ValueError, match=r"holds \{'C': 'kg'\} beside"): + with pytest.raises(ValueError, match="not read: 'kg' in column C"): run_main( tmp_path, make_test_xlsx(), @@ -3566,116 +3419,52 @@ def test_real_tables_with_a_kg_cell_beside_a_line_are_refused(tmp_path): REAL_OCTOBER_2026_SHEETS = read_workbook_sheets(REAL_OCTOBER_2026_RECEIPTS) -@settings(max_examples=60, deadline=None, derandomize=True) -@given( - key=st.sampled_from(sorted(CLEARANCE_LEVELS)), - unit=st.one_of( - st.sampled_from(TOBACCO_UNITS), - # Any other text, but not a fiscal year or a marker, which would make - # the row a heading rather than a line. - st.text(min_size=1, max_size=12).filter( - lambda text: ( - text.strip() - and not re.fullmatch(r"\s*\d{4}-\d{2}\s*", text) - and receipts_label(text) not in ("outturn", "forecast") - ) - ), - ), - beside=st.sampled_from(["none", "before", "after"]), - total=st.sampled_from(["('000s kg)", "(tonnes)", "(kg)", "('000s tonnes)"]), -) -def test_a_clearance_line_reads_only_in_the_unit_its_table_states( - key, unit, beside, total -): - """For any unit text beside any real October 2026 clearance line, before - its years or after them, the importer refuses the line. With nothing - beside it, a tobacco part reads only under a non-cigarette total that - states thousand kg, and then reads the real values.""" - spec = CLEARANCE_SPECS[key] - table = "4.11" if "tobacco" in spec.path[1] else "4.12" - rows = [dict(row) for row in REAL_OCTOBER_2026_SHEETS[table]] - for row in rows: - label = receipts_label(row.get("B")) - if label.startswith("non-cigarette clearances"): - row["B"] = f"Non-cigarette clearances {total}" - line = next( - row - for row in rows - if receipts_label(row.get("B")) == spec.row - and any(column != "B" for column in row) - ) - if beside == "before": - for row in rows[1:]: - for column in sorted( - (column for column in row if len(column) == 1 and column >= "C"), - reverse=True, - ): - row[chr(ord(column) + 1)] = row.pop(column) - line["C"] = unit - elif beside == "after": - line["J"] = unit - sheets = {table: rows} - if beside != "none": - event("refused: beside") - with pytest.raises(ValueError, match="beside"): - find_receipts_line(sheets, spec) - return - parts = ("hand_rolling_tobacco", "other_tobacco_products") - if key in parts and total != "('000s kg)": - event("refused: total in another unit") - with pytest.raises( - ValueError, - match=rf"cannot read {re.escape('.'.join(spec.path))} in .*" - rf"[\"']non-cigarette clearances {re.escape(total)}[\"']", - ): - find_receipts_line(sheets, spec) - return - event("read") - assert find_receipts_line(sheets, spec).levels == dict( - zip( - range(2025, 2032), - (float(level) for level in OCTOBER_2026_CLEARANCES[key]), - ) - ) - - -def rewrite_real_table(workbook, table, labels=(), scale=(), notes=()): - """Set the text of the cells in ``labels`` (as {"B6": text}), multiply - the values on each row in ``scale`` (as {6: 1000}) and add each of +def edit_real_table(workbook, table, cells=(), scale=(), notes=()): + """Set each cell in ``cells`` (as {"B6": text}) to its text, its number + or, for None, nothing, adding the cell and its row where the sheet lacks + them; multiply each cell in ``scale`` (as {"D5": 0.1}); and add each of ``notes`` in column B below the sheet's last row. Every other archive entry, including the shared strings, stays.""" - labels, scale = dict(labels), dict(scale) + cells, scale = dict(cells), dict(scale) output = BytesIO() with ZipFile(BytesIO(workbook)) as source, ZipFile(output, "w") as dest: path = _sheet_paths(source)[table] sheet = ET.fromstring(source.read(path)) data = sheet.find("m:sheetData", XLSX_NS) - for cell in sheet.iter(f"{{{XLSX_MAIN}}}c"): - reference = cell.get("r") - row = int(reference.lstrip("ABCDEFGHIJKLMNOPQRSTUVWXYZ")) - if reference in labels: - for child in list(cell): - cell.remove(child) - cell.set("t", "inlineStr") - ET.SubElement( - ET.SubElement(cell, f"{{{XLSX_MAIN}}}is"), f"{{{XLSX_MAIN}}}t" - ).text = labels.pop(reference) - elif row in scale and cell.get("t") != "s": - value = cell.find("m:v", XLSX_NS) - if value is not None: - value.text = repr(float(value.text) * scale[row]) - assert not labels, labels last = max(int(row.get("r")) for row in data) for offset, note in enumerate(notes, start=1): - row = ET.SubElement(data, f"{{{XLSX_MAIN}}}row", {"r": str(last + offset)}) - cell = ET.SubElement( - row, - f"{{{XLSX_MAIN}}}c", - {"r": f"B{last + offset}", "t": "inlineStr"}, - ) - ET.SubElement( - ET.SubElement(cell, f"{{{XLSX_MAIN}}}is"), f"{{{XLSX_MAIN}}}t" - ).text = note + cells[f"B{last + offset}"] = note + + def cell(reference): + number = int(reference.lstrip("ABCDEFGHIJKLMNOPQRSTUVWXYZ")) + rows = {int(row.get("r")): row for row in data} + if number not in rows: + rows[number] = ET.Element(f"{{{XLSX_MAIN}}}row", {"r": str(number)}) + data.insert(sum(other < number for other in rows), rows[number]) + found = rows[number].find(f"m:c[@r='{reference}']", XLSX_NS) + if found is None: + found = ET.SubElement( + rows[number], f"{{{XLSX_MAIN}}}c", {"r": reference} + ) + return found + + for reference, factor in scale.items(): + target = cell(reference) + assert target.get("t") is None, reference + value = target.find("m:v", XLSX_NS) + value.text = repr(float(value.text) * factor) + for reference, value in cells.items(): + target = cell(reference) + for child in list(target): + target.remove(child) + target.attrib.pop("t", None) + if isinstance(value, str): + target.set("t", "inlineStr") + ET.SubElement( + ET.SubElement(target, f"{{{XLSX_MAIN}}}is"), f"{{{XLSX_MAIN}}}t" + ).text = value + elif value is not None: + ET.SubElement(target, f"{{{XLSX_MAIN}}}v").text = repr(float(value)) for entry in source.infolist(): dest.writestr( entry, @@ -3695,442 +3484,510 @@ def real_levels(workbook, key): ) -def test_a_unit_note_cannot_override_the_total_a_line_reads_in(): - """The helper relabels the real October 2026 Table 4.11's non-cigarette - total in kg, multiplies its values and other tobacco's by 1,000, and - adds a note that puts hand-rolled tobacco in thousand kg and other - tobacco in kg. Read in the thousand kg the March 2026 table states, - other tobacco would have growth of 862.8112 in 2025. The total's label - and the note both state kg for other tobacco, the total kg for - hand-rolled tobacco too, so the importer refuses both lines and the - renderer refuses the tables; cigarettes read as before.""" - changed = rewrite_real_table( - REAL_OCTOBER_2026_RECEIPTS, - "4.11", - labels={"B6": "Non-cigarette clearances (kg)"}, - scale={6: 1000, 9: 1000}, - notes=[ - "Note: Hand rolled tobacco clearances are in ('000s kg); other " - "tobacco products are in kg." - ], +# The importer reads no words in a clearance table: each cell must hold a +# number, a fiscal year or a text someone has read (CLEARANCE_TABLE_TEXTS). + +TOBACCO_LINES = ("cigarettes", "hand_rolling_tobacco", "other_tobacco_products") +ALCOHOL_LINES = ("beer_and_cider", "wine", "spirits") +REAL_CLEARANCE_TABLES = { + (release, title): (workbook, f"{chapter}.{number}") + for release, workbook, chapter in ( + ("March 2025", REAL_RECEIPTS[2025], 3), + ("March 2026", REAL_RECEIPTS[2026], 3), + ("October 2026", REAL_OCTOBER_2026_RECEIPTS, 4), ) - (line,) = [ - row - for row in read_workbook_sheets(changed)["4.11"] - if receipts_label(row.get("B")) == "other tobacco products" + for title, number in ((TOBACCO_CLEARANCES, 11), (ALCOHOL_CLEARANCES, 12)) +} + + +@lru_cache(maxsize=None) +def real_clearance_rows(release, title): + """The rows of a real clearance table's sheet, and of its table: from + its title's row to the row above the next table's title, or to the + sheet's end, with that title in lower case, without its number.""" + workbook, sheet = REAL_CLEARANCE_TABLES[release, title] + rows = read_sheet_rows(workbook, sheet) + titles = [" ".join(str(row.get("B") or "").split()) for row in rows] + (start,) = [ + i for i, text in enumerate(titles) if text.lower() == f"{sheet} {title}" ] - previous = float(MARCH_2026_CLEARANCES["other_tobacco_products"][0]) - assert round(float(line["C"]) / previous - 1, 6) == 862.8112 - note = ( - "\"note: hand rolled tobacco clearances are in ('000s kg); other " - 'tobacco products are in kg."' - ) - with pytest.raises(ValueError) as refused: - compute_receipts_growth( - changed, - REAL_RECEIPTS[2026], - specs=(CLEARANCE_SPECS["other_tobacco_products"],), - ) - assert str(refused.value) == ( - "The importer cannot read " - 'per_capita.tobacco_clearances.other_tobacco_products in "(\'000s kg)" ' - 'from Table 4.11 (sheet 4.11): the OBR states "kg", another unit or ' - "scale, in 'non-cigarette clearances (kg)'; the OBR states \"kg\", " - f"another unit or scale, in {note}" + for end in range(start + 1, len(rows)): + if re.match(r"\d+\.\d+ ", titles[end]): + return rows, start, end, titles[end].split(" ", 1)[1].lower() + return rows, start, len(rows), None + + +def unread(table, title, *cells): + """The importer's refusal of real Table ``table`` for the unread + ``cells``, each as (text, column).""" + return ( + f"Table {table} (sheet {table}) holds text the importer has not read: " + + "; ".join(f"{text!r} in column {column}" for text, column in cells) + + ". Read each one. If it states another unit, convert the table's " + "figures to the units the read texts state. Then add the text, as " + f'receipts_label gives it, to CLEARANCE_TABLE_TEXTS["{title}"], or the ' + "title of a table that follows the clearances to CLEARANCE_TABLE_ENDS." ) - with pytest.raises( - ValueError, - match=r"hand_rolling_tobacco in .*the OBR states \"kg\", another unit or " - r"scale, in 'non-cigarette clearances \(kg\)'", + + +def test_both_real_release_pairs_read_every_series(): + """March 2026 over March 2025 and October 2026 over March 2026 each read + all 17 series, the six clearance lines among them.""" + for current, previous in ( + (REAL_RECEIPTS[2026], REAL_RECEIPTS[2025]), + (REAL_OCTOBER_2026_RECEIPTS, REAL_RECEIPTS[2026]), ): - render( - get_yoy_growth_path().read_text(), - current=changed, - previous=REAL_RECEIPTS[2026], - **COMMITTED_RELEASES, - ) - read, real = real_levels(changed, "cigarettes") - assert read == real + growth = compute_receipts_growth(current, previous) + assert len(growth) == len(RECEIPTS_SERIES_SPECS) == 17 + assert set(CLEARANCE_PATHS.values()) <= set(growth) + + +def test_the_read_texts_are_the_texts_of_the_three_releases(): + """Each clearance table's list holds the text of every cell in the three + releases' tables, below the title and above the next table's title, but + numbers and fiscal years, and nothing else. The list of ends holds the + titles that follow them: the alcohol receipts forecasts, on Table 4.12's + sheet (3.12's in March), and nothing on Table 4.11's.""" + texts = {title: set() for title in CLEARANCE_TABLE_TEXTS} + ends = {title: set() for title in CLEARANCE_TABLE_TEXTS} + for release, title in REAL_CLEARANCE_TABLES: + rows, start, end, following = real_clearance_rows(release, title) + assert [value for value in rows[start].values() if value][1:] == [] + if following is not None: + ends[title].add(following) + for row in rows[start + 1 : end]: + for value in row.values(): + cell = " ".join(str(value or "").split()) + if not cell or re.fullmatch(r"\d{4}-\d{2}", cell): + continue + try: + float(cell) + except ValueError: + texts[title].add(receipts_label(cell)) + assert texts == { + title: set(listed) for title, listed in CLEARANCE_TABLE_TEXTS.items() + } + assert ends == { + title: set(listed) for title, listed in CLEARANCE_TABLE_ENDS.items() + } + assert ends[ALCOHOL_CLEARANCES] == {"alcohol receipts forecasts"} -@pytest.mark.parametrize( - "key, previous_rows, current_rows, message", - [ - # The label states million sticks, and a note that names no product - # puts the forecasts in thousand sticks. - ( - "cigarettes", - [ - {"B": "3.11 Tobacco clearances"}, - {"C": "Outturn"}, - {"C": "2024-25"}, - {"B": "Cigarette clearances (million sticks)", "C": 600}, - ], - [ - {"B": "4.11 Tobacco clearances"}, - {"C": "Outturn", "D": "Forecast"}, - {"C": "2025-26", "D": "2026-27"}, - {"B": "Cigarette clearances (million sticks)", "C": 600, "D": 600000}, - { - "B": "Note: Outturn clearances use the unit in the label; " - "forecasts are in thousand sticks." - }, - ], - "The importer cannot read per_capita.tobacco_clearances.cigarettes " - 'in "(million sticks)" from Table 4.11 (sheet 4.11): the OBR states ' - '"thousand sticks", another unit or scale, in \'note: outturn ' - "clearances use the unit in the label; forecasts are in thousand " - "sticks.'", - ), - # The total states thousand kg, and a note on other tobacco states kg. - ( - "other_tobacco_products", - [ - {"B": "3.11 Tobacco clearances"}, - {"C": "Outturn"}, - {"C": "2024-25"}, - {"B": "Non-cigarette clearances ('000s kg)", "C": 4000}, - {"B": "of which:"}, - {"B": "Other tobacco products", "C": 600}, - ], - [ - {"B": "4.11 Tobacco clearances"}, - {"C": "Outturn", "D": "Forecast"}, - {"C": "2025-26", "D": "2026-27"}, - {"B": "Non-cigarette clearances ('000s kg)", "C": 4000, "D": 3900}, - {"B": "of which:"}, - {"B": "Other tobacco products", "C": 600000, "D": 570000}, - {"B": "Note: Other tobacco products are in kg."}, - ], - "The importer cannot read " - "per_capita.tobacco_clearances.other_tobacco_products in " - '"(\'000s kg)" from Table 4.11 (sheet 4.11): the OBR states "kg", ' - "another unit or scale, in 'note: other tobacco products are in kg.'", - ), - ], +# Each edit to the real October 2026 Table 4.11 or 4.12 below puts text no +# one has read in the table. Table 4.11 runs from row 2 to the sheet's end, +# so a note below the sheet's last row falls in it; Table 4.12's clearances +# run from row 2 to row 11, above the receipts forecasts' title. +OTHER_TOBACCO_NOTE = "Note: Other tobacco products {}" +FULL_NOTE = ( + "Note: Hand rolled tobacco clearances are in ('000s kg); other tobacco " + "products are in kg." ) -def test_a_note_that_describes_a_line_must_agree_with_its_label( - key, previous_rows, current_rows, message -): - """Read in its label's or its total's unit, either line would have growth - of 999.0 for clearances that did not change. A note that names the - line's product, or no product, describes the line, so the importer - refuses the line even though its label or its total states the line's - unit.""" - spec = CLEARANCE_SPECS[key] +UNREAD_EDITS = { + # The OBR's cigarette forecasts in ten million sticks, and other tobacco + # in kilotonnes, with the figures to match. + "spelled-scale-real-release": ( + "4.11", + {}, + {f"{column}5": 0.1 for column in "DEFGHI"}, + ["Note: Cigarette forecasts shown in ten million sticks."], + ), + "new-unit-real-release": ( + "4.11", + {}, + {f"{column}9": 0.001 for column in "CDEFGHI"}, + [OTHER_TOBACCO_NOTE.format("shown in kilotonnes.")], + ), + **{ + f"unknown-note-{ending}": ("4.11", {}, {}, [OTHER_TOBACCO_NOTE.format(ending)]) + for ending in ( + "are in milligrams.", + "are in (milligrams).", + "shown in milligrams.", + "(milligrams).", + "shown in kilotonnes.", + "shown in tens of thousands of kilograms.", + ) + }, + "unknown-label-brackets": ( + "4.11", + {"B6": "Non-cigarette clearances (milligrams)"}, + {}, + [], + ), + **{ + f"product-negation-{product}": ( + "4.11", + {}, + {}, + [f"Note: {product} are in million sticks."], + ) + for product in ("Non-cigarette products", "Non-cigarettes") + }, + **{ + f"product-only-note-{product}": ( + "4.12", + { + "B8": f"Note: {product} clearances are in thousands of " + "hectolitres of alcohol." + }, + {}, + [], + ) + for product in ("Wine", "Sake", "swine") + }, + "full-note": ( + "4.11", + {"B6": "Non-cigarette clearances (kg)"}, + {f"{column}{row}": 1000 for column in "CDEFGHI" for row in (6, 9)}, + [FULL_NOTE], + ), + "competing-cigarettes": ( + "4.11", + {}, + {}, + [ + "Note: Outturn clearances use the unit in the label; forecasts are " + "in thousand sticks." + ], + ), + "competing-other-tobacco": ( + "4.11", + {}, + {}, + [OTHER_TOBACCO_NOTE.format("are in kg.")], + ), + # The listed units spelt another way. + "same-unit-thousand-kg": ( + "4.11", + {"B6": "Non-cigarette clearances (thousand kg)"}, + {}, + [], + ), + "same-unit-000s-hectolitres": ( + "4.12", + {"B8": "Note: All clearances are in '000s hectolitres of alcohol."}, + {}, + [], + ), + # A listed text with more words. + "extra-words-label": ( + "4.11", + {"B6": "Non-cigarette clearances ('000s kg) in tens"}, + {}, + [], + ), + "extra-words-note": ( + "4.12", + { + "B8": "Note: All clearances are in thousands of hectolitres of " + "alcohol. Spirits are in litres." + }, + {}, + [], + ), + # Units outside column B: in a new row, the "of which:" row, beside a + # line after its years, and in the title, marker and year rows. + "non-first-column-note": ( + "4.11", + {"E40": "Note: Cigarette forecasts shown in ten million sticks."}, + {}, + [], + ), + "non-first-column-of-which": ("4.11", {"K7": "kg"}, {}, []), + "after-the-years": ("4.11", {"J5": "ten million sticks"}, {}, []), + "title-row": ("4.11", {"D2": "Cigarettes in ten million sticks"}, {}, []), + "marker-row": ("4.11", {"K3": "Million sticks"}, {}, []), + "year-row": ("4.11", {"J4": "Unit"}, {}, []), + "alcohol-last-row": ( + "4.12", + {"B11": "Note: Spirits clearances are in litres."}, + {}, + [], + ), + # New lines: a product neither list names, and wine in Table 4.11. + "new-line-heated-tobacco": ( + "4.11", + {"B10": "Heated tobacco clearances", **{f"{c}10": 50 for c in "CDEFGHI"}}, + {}, + [], + ), + "new-line-sake": ( + "4.12", + {"B11": "Sake", **{f"{c}11": 10 for c in "CDEFGHI"}}, + {}, + [], + ), + "other-table-text": ("4.11", {"B10": "Wine"}, {}, []), + # Text where a number belongs. + "not-a-number": ("4.11", {"D5": "nan"}, {}, []), +} + + +@pytest.mark.parametrize("name", sorted(UNREAD_EDITS)) +def test_text_no_one_has_read_stops_the_real_tables(name): + """For each edit, the importer refuses every line of the edited table, + naming each unread cell, and the run stops before it renders anything; + the other table's lines read as before.""" + table, cells, scale, notes = UNREAD_EDITS[name] + changed = edit_real_table( + REAL_OCTOBER_2026_RECEIPTS, table, cells=cells, scale=scale, notes=notes + ) + title = TOBACCO_CLEARANCES if table == "4.11" else ALCOHOL_CLEARANCES + added = [(text, reference[0]) for reference, text in cells.items()] + expected = unread( + table, + title, + *[(text, column) for text, column in added if isinstance(text, str)], + *[(note, "B") for note in notes], + ) + sheets = {sheet: read_sheet_rows(changed, sheet) for sheet in ("4.11", "4.12")} + for key, spec in CLEARANCE_SPECS.items(): + if spec.title != title: + assert ( + find_receipts_line(sheets, spec).levels + == find_receipts_line(REAL_OCTOBER_2026_SHEETS, spec).levels + ), key + continue + with pytest.raises(ValueError) as refused: + find_receipts_line(sheets, spec) + assert str(refused.value) == expected, key with pytest.raises(ValueError) as refused: compute_receipts_growth( - mixed_unit_workbook(current_rows), - mixed_unit_workbook(previous_rows), - specs=(spec,), + changed, REAL_RECEIPTS[2026], specs=CLEARANCE_SERIES_SPECS ) - assert str(refused.value) == message + assert str(refused.value) == expected -def test_a_note_about_another_product_says_nothing_about_a_line(): - """A note on the real October 2026 Table 4.11 that puts hand-rolled - tobacco in million sticks describes hand-rolled tobacco alone: the - importer refuses that line, and reads cigarettes in their label's unit - and other tobacco in its total's, as before.""" - changed = rewrite_real_table( - REAL_OCTOBER_2026_RECEIPTS, - "4.11", - notes=["Note: Hand rolled tobacco clearances are in million sticks."], +def test_the_scale_edits_change_the_growth_the_figures_give(): + """Read in the units the real tables state, the two rescaled tables' + figures would give cigarette clearances growth of -91.0% in 2026-27 + for -10.3%, and other tobacco -99.9% in 2025-26 for -13.6%.""" + + def growth(levels, key): + # 2025-26 over March 2026's 2024-25 outturn, then 2026-27 over 2025-26. + before = float(MARCH_2026_CLEARANCES[key][0]) + return [round(levels[0] / before - 1, 6), round(levels[1] / levels[0] - 1, 6)] + + for name, key, year, scaled, real in ( + ("spelled-scale-real-release", "cigarettes", 1, -0.910322, -0.103219), + ("new-unit-real-release", "other_tobacco_products", 0, -0.999136, -0.136189), + ): + table, _, scale, _ = UNREAD_EDITS[name] + changed = edit_real_table(REAL_OCTOBER_2026_RECEIPTS, table, scale=scale) + (line,) = [ + row + for row in read_sheet_rows(changed, table) + if receipts_label(row.get("B")) == CLEARANCE_SPECS[key].row + ] + levels = [float(line[column]) for column in "CD"] + published = [float(level) for level in OCTOBER_2026_CLEARANCES[key][:2]] + assert growth(levels, key)[year] == scaled, name + assert growth(published, key)[year] == real, name + + +def test_a_real_table_without_its_unit_cell_refuses_the_lines_it_describes(): + """Cleared, the non-cigarette total's label and Table 4.12's note leave + hand-rolled tobacco, other tobacco and the alcohol lines without the + cell that states their unit, and the importer refuses them; cigarettes + read as before.""" + for table, cell in (("4.11", "B6"), ("4.12", "B8")): + changed = edit_real_table(REAL_OCTOBER_2026_RECEIPTS, table, cells={cell: None}) + for key, spec in CLEARANCE_SPECS.items(): + if spec.title != ( + TOBACCO_CLEARANCES if table == "4.11" else ALCOHOL_CLEARANCES + ): + continue + if key == "cigarettes": + read, real = real_levels(changed, key) + assert read == real + continue + with pytest.raises(ValueError) as refused: + real_levels(changed, key) + assert str(refused.value) == ( + f"Table {table} (sheet {table}) lacks {spec.unit_cell!r}, the cell " + f"in which the OBR states the unit of {'.'.join(spec.path)}" + ) + + +def test_the_alcohol_receipts_table_stays_outside_the_clearance_table(): + """On the real October 2026 sheet 4.12, the alcohol receipts forecasts + follow the clearances, with "£ billion" and "Note: All taxes are on an + accrued basis.". The clearance table ends at their title: a note in the + receipts table leaves the clearance lines reading, the same note in the + clearance table's last row stops them, and so does a receipts title the + list of ends does not name, since the clearance table then runs through + the receipts table to the sheet's end.""" + rows, start, end, following = real_clearance_rows( + "October 2026", ALCOHOL_CLEARANCES + ) + assert (start, end, following) == (1, 11, "alcohol receipts forecasts") + assert rows[12]["C"] == "£ billion" + assert rows[18]["B"] == "Note: All taxes are on an accrued basis." + note = "Note: Spirits clearances are in litres." + for cells in ({"B22": note}, {"J13": note}, {"C13": "£ million"}): + changed = edit_real_table(REAL_OCTOBER_2026_RECEIPTS, "4.12", cells=cells) + for key in ALCOHOL_LINES: + read, real = real_levels(changed, key) + assert read == real, (cells, key) + changed = edit_real_table(REAL_OCTOBER_2026_RECEIPTS, "4.12", cells={"B11": note}) + with pytest.raises(ValueError) as refused: + real_levels(changed, "wine") + assert str(refused.value) == unread("4.12", ALCOHOL_CLEARANCES, (note, "B")) + retitled = "4.12 Alcohol receipts forecasts (accrued)" + changed = edit_real_table( + REAL_OCTOBER_2026_RECEIPTS, "4.12", cells={"B12": retitled} ) - for key in ("cigarettes", "other_tobacco_products"): - read, real = real_levels(changed, key) - assert read == real, key with pytest.raises(ValueError) as refused: - real_levels(changed, "hand_rolling_tobacco") - assert str(refused.value).endswith( - 'hand_rolling_tobacco in "(\'000s kg)" from Table 4.11 (sheet 4.11): ' - 'the OBR states "million sticks", another unit or scale, in ' - "'note: hand rolled tobacco clearances are in million sticks.'" + real_levels(changed, "wine") + assert str(refused.value) == unread( + "4.12", + ALCOHOL_CLEARANCES, + (retitled, "B"), + ("£ billion", "C"), + ("Note: All taxes are on an accrued basis.", "B"), ) -def test_a_unit_only_in_a_note_about_another_product_is_refused(): - """With Table 4.12's note rewritten to state the unit for wine alone, the - real October 2026 table states no unit for beer and cider or for - spirits: the importer refuses both and reads wine as before.""" - note = "Note: Wine clearances are in thousands of hectolitres of alcohol." - changed = rewrite_real_table( - REAL_OCTOBER_2026_RECEIPTS, "4.12", labels={"B8": note} +def unread_text(text, title): + """Whether the importer must refuse ``text`` in a clearance table titled + ``title``: written out from the lists, not read by the importer, and + leaving out blanks, numbers, fiscal years and table titles a list + names.""" + label = receipts_label(text) + if not label or label in CLEARANCE_TABLE_TEXTS[title]: + return False + if re.fullmatch(r"\d{4}-\d{2}", label): + return False + try: + float(label) + return False + except ValueError: + pass + heading = re.match(r"(\d+\.\d+)\s+(.+)$", " ".join(text.split())) + return not ( + heading + and receipts_label(heading.group(2)) + in {*CLEARANCE_TABLE_ENDS[title], TOBACCO_CLEARANCES, ALCOHOL_CLEARANCES} ) - read, real = real_levels(changed, "wine") - assert read == real - for key in ("beer_and_cider", "spirits"): - with pytest.raises(ValueError) as refused: - real_levels(changed, key) - assert str(refused.value) == ( - f"The importer cannot read {'.'.join(CLEARANCE_SPECS[key].path)} in " - '"thousands of hectolitres of alcohol" from Table 4.12 (sheet 4.12): ' - "the importer found the unit only in ['note: wine clearances are in " - "thousands of hectolitres of alcohol.'], which name other products" - ) -@pytest.mark.parametrize( - "table, cell, text, refused", - [ - ("4.11", "B6", "Non-cigarette clearances (thousand kg)", None), - ("4.11", "B6", "Non-cigarette clearances (thousands of kilograms)", None), - ( - "4.12", - "B8", - "Note: All clearances are in '000s hectolitres of alcohol.", - None, - ), - ( - "4.12", - "B8", - "Note: All clearances are in hundred thousand litres of alcohol.", - None, - ), - ("4.11", "B6", "Non-cigarette clearances (million kg)", "million kg"), - ( - "4.12", - "B8", - "Note: All clearances are in hundreds of hectolitres of alcohol.", - "hundreds of hectolitres of alcohol", +UNIT_SHAPED = st.one_of( + st.sampled_from( + [ + "kg", + "'000s kg", + "(kg)", + "(tonnes)", + "Unit", + "£ billion", + "million sticks", + "(million sticks)", + "thousands of hectolitres of alcohol", + "Of which: cigars", + "Cigarette clearances (billion sticks)", + "Non-cigarette clearances (thousand kg)", + "Heated tobacco clearances", + "Note: Non-cigarettes are in million sticks.", + "Note: Sake clearances are in thousands of hectolitres of alcohol.", + "Note: All clearances are in millions of litres of alcohol.", + ] + ), + st.builds( + "{}{}{}{}".format, + st.sampled_from( + [ + "", + "(", + "Note: ", + "Note: All clearances are in ", + "Note: Cigarette forecasts shown in ", + "Note: Other tobacco products are in ", + ] ), - ( - "4.12", - "B8", - "Note: All clearances are in thousands of hectolitres of product.", - "thousands of hectolitres of product", + st.sampled_from(["", "ten ", "tens of ", "thousand ", "'000s ", "million "]), + st.sampled_from( + [ + "kg", + "sticks", + "million sticks", + "kilotonnes", + "tonnes", + "litres", + "hectolitres of alcohol", + "litres of product", + ] ), - ], + st.sampled_from(["", ".", ")"]), + ), ) -def test_a_clearance_unit_reads_by_measure_and_size(table, cell, text, refused): - """The importer compares units by measure and size: the same unit spelt - another way reads the real October 2026 lines as before, and another - size or measure is refused.""" - changed = rewrite_real_table(REAL_OCTOBER_2026_RECEIPTS, table, labels={cell: text}) - keys = [ - key - for key, spec in CLEARANCE_SPECS.items() - if (spec.title == TOBACCO_CLEARANCES) == (table == "4.11") - and key != "cigarettes" - ] - for key in keys: - if refused is None: - read, real = real_levels(changed, key) - assert read == real, key - continue - with pytest.raises(ValueError, match=f'states "{refused}", another unit'): - real_levels(changed, key) - - -# The unit each spelling states, as (measure, size): what the property below -# expects, written out rather than read by the importer. None marks a unit -# the importer does not know. -TABLE_UNITS = { - "million sticks": ("sticks", 10**6), - "millions of sticks": ("sticks", 10**6), - "thousand sticks": ("sticks", 10**3), - "'000s kg": ("kg", 10**3), - "thousand kg": ("kg", 10**3), - "thousands of kilograms": ("kg", 10**3), - "kg": ("kg", 1), - "million kg": ("kg", 10**6), - "thousands of hectolitres of alcohol": ("litres of alcohol", 10**5), - "thousand hectolitres of alcohol": ("litres of alcohol", 10**5), - "hectolitres of alcohol": ("litres of alcohol", 10**2), - "millions of litres of alcohol": ("litres of alcohol", 10**6), - "thousands of hectolitres of product": ("litres of product", 10**5), - "tonnes": None, - "grams": None, - "'000s tonnes": None, - "litres": None, - "hogsheads": None, - "thousands of hectolitres of pure alcohol": None, -} -# In a note only: it names cigarettes, which a label's scope would change. -NOTE_UNITS = {**TABLE_UNITS, "million cigarettes": None} -EXPECTED_UNITS = { - "(million sticks)": ("sticks", 10**6), - "('000s kg)": ("kg", 10**3), - "thousands of hectolitres of alcohol": ("litres of alcohol", 10**5), -} -# The names a note gives each product, and the lines they describe. -NOTE_SUBJECTS = { - TOBACCO_CLEARANCES: { - "Cigarette": {"cigarettes"}, - "Hand-rolled tobacco": {"hand_rolling_tobacco"}, - "Other tobacco products": {"other_tobacco_products"}, - "Cigars": {"other_tobacco_products"}, - "Non-cigarette products": {"hand_rolling_tobacco", "other_tobacco_products"}, - }, - ALCOHOL_CLEARANCES: { - "Beer and cider": {"beer_and_cider"}, - "Wine": {"wine"}, - "Other fermented products": {"wine"}, - "Spirits": {"spirits"}, - "Alcopops": {"spirits"}, - }, -} -@st.composite -def clearance_release(draw, key): - """A table's total and notes for one release, around the OBR's layout: - the label of a non-cigarette total, with or without its "of which:" row, - and up to two notes, each opening "Note:" or "Source: HMRC.", of one or - two clauses that each put some products, or all clearances, in a unit. - Half the draws start from the OBR's own unit cell: the non-cigarette - total's label in thousand kg, or Table 4.12's note.""" - spec = CLEARANCE_SPECS[key] - equivalent = [ - spelling - for spelling, unit in TABLE_UNITS.items() - if unit == EXPECTED_UNITS[spec.unit] - ] - unit = st.one_of(st.sampled_from(equivalent), st.sampled_from(sorted(NOTE_UNITS))) - total = of_which = None - published = draw(st.booleans()) - if spec.title == TOBACCO_CLEARANCES and published: - total, of_which = "'000s kg", True - elif spec.title == TOBACCO_CLEARANCES: - total = draw( - st.none() - | st.sampled_from(equivalent if key != "cigarettes" else ["'000s kg"]) - | st.sampled_from(sorted(TABLE_UNITS)) - ) - of_which = total is not None and draw(st.booleans()) - clause = st.tuples( - st.lists( - st.sampled_from(sorted(NOTE_SUBJECTS[spec.title])), max_size=2, unique=True - ), - unit, - ) - notes = draw( - st.lists( - st.tuples( - st.sampled_from(["Note:", "Source: HMRC."]), - st.lists(clause, min_size=1, max_size=2), - ), - max_size=2, +@settings(max_examples=200, deadline=None, derandomize=True) +@given(table=st.sampled_from(sorted(REAL_CLEARANCE_TABLES)), data=st.data()) +def test_any_text_no_one_has_read_stops_a_real_clearance_table(table, data): + """Put one text in a real table, in an empty cell of any row from its + title down, in any column, or on a new row. Any text outside the table's + list, from st.text() or shaped like a unit, stops every line of the + table: the importer names the cell, or finds no line or table where the + text took a label's or the title's place. A text from the list, other + than a line's label or a marker, in a cell outside the lines and the + year row, leaves every line reading as before.""" + release, title = table + _, sheet = REAL_CLEARANCE_TABLES[table] + real, start, end, _ = real_clearance_rows(release, title) + rows = [dict(row) for row in real] + keys = [key for key, spec in CLEARANCE_SPECS.items() if spec.title == title] + listed = data.draw(st.booleans(), label="listed") + if listed: + labels = {CLEARANCE_SPECS[key].row for key in keys} | {"outturn", "forecast"} + text = data.draw( + st.sampled_from(sorted(CLEARANCE_TABLE_TEXTS[title] - labels)), label="text" ) + else: + text = data.draw(st.one_of(st.text(min_size=1, max_size=40), UNIT_SHAPED)) + assume(unread_text(text, title)) + year_row = next( + i + for i in range(start, end) + if any(re.fullmatch(r"\d{4}-\d{2}", str(v or "")) for v in rows[i].values()) ) - if spec.title == ALCOHOL_CLEARANCES and published: - notes = [("Note:", [([], "thousands of hectolitres of alcohol")]), *notes] - return total, of_which, notes - - -def clearance_release_rows(key, total, of_which, notes, first_year, levels): - """The table's rows for ``clearance_release``'s draw.""" - spec = CLEARANCE_SPECS[key] - chapter = 3 if first_year == 2024 else 4 - columns = "CD"[: len(levels)] - if spec.title == TOBACCO_CLEARANCES: - labels = ["Cigarette clearances (million sticks)"] - if total is not None: - labels.append(f"Non-cigarette clearances ({total})") - if of_which: - labels.append("of which:") - labels += ["Hand rolled tobacco clearances", "Other tobacco products"] - title = f"{chapter}.11 Tobacco clearances" + line_rows = { + i + for i in range(year_row + 1, end) + if any(re.fullmatch(r"[\d.]+", str(v or "")) for v in rows[i].values()) + } + column = data.draw(st.sampled_from("ABCDEFGHIJKLM"), label="column") + if data.draw(st.booleans(), label="new row"): + lowest = year_row + 1 if listed else start + 1 + rows.insert(data.draw(st.integers(lowest, end), label="row"), {column: text}) else: - labels = ["Beer and cider", "Wine1", "Spirits2"] - title = f"{chapter}.12 Alcohol clearance forecasts" - rows = [ - {"B": title}, - {"C": "Outturn", **({"D": "Forecast"} if len(levels) > 1 else {})}, - { - column: f"{first_year + i}-{str(first_year + i + 1)[2:]}" - for i, column in enumerate(columns) - }, - ] - for label in labels: - values = {} if label == "of which:" else dict(zip(columns, levels)) - rows.append({"B": label, **values}) - for prefix, clauses in notes: - rows.append( - { - "B": f"{prefix} " - + "; ".join( - f"{' and '.join(subjects) or 'All clearances'} are in {unit}" - for subjects, unit in clauses - ) - + "." - } - ) - if spec.title == ALCOHOL_CLEARANCES: - rows += [ - {"B": "1 Includes other fermented products such as fruit ciders."}, - {"B": "2 Includes spirit-based ‘ready-to-drinks’ such as alcopops."}, - ] - return rows - - -def clearance_release_reads(key, total, of_which, notes): - """Whether a release's table gives ``key`` its unit, written out from the - draw rather than read from the table's text: every unit that describes - the line is its unit, one of them sits where the OBR states it, every - note's units are ones the importer knows, and a note that states two - units ties each to products.""" - spec = CLEARANCE_SPECS[key] - expected = EXPECTED_UNITS[spec.unit] - described = [] # (unit, whether its cell is one that can state it) - if key == "cigarettes": - described.append((expected, True)) - if total is not None and key in ("hand_rolling_tobacco", "other_tobacco_products"): - described.append((TABLE_UNITS[total], bool(of_which))) - for prefix, clauses in notes: - units = [NOTE_UNITS[unit] for _, unit in clauses] - if None in units: - return False - lines = [ - set().union(*(NOTE_SUBJECTS[spec.title][s] for s in subjects)) - for subjects, _ in clauses + empty = [ + i for i in range(start, end) if not str(rows[i].get(column) or "").strip() ] - if len(set(units)) > 1: - if not all(lines): - return False - ties = list(zip(units, lines)) + if listed: + empty = [ + i + for i in empty + if i not in line_rows and i != year_row and (i != start or column > "B") + ] + assume(empty) + rows[data.draw(st.sampled_from(empty), label="row")][column] = text + for key in keys: + spec = CLEARANCE_SPECS[key] + if listed: + event("listed: read") + assert ( + find_receipts_line({sheet: rows}, spec).levels + == find_receipts_line({sheet: real}, spec).levels + ) + continue + with pytest.raises(ValueError) as refused: + find_receipts_line({sheet: rows}, spec) + if str(refused.value).startswith(f'Expected one "{spec.row}" row'): + event("unread: no line or table") else: - ties = [(unit, set().union(*lines)) for unit in units] - described += [ - (unit, prefix == "Note:") - for unit, products in ties - if not products or key in products - ] - return all(unit == expected for unit, _ in described) and any( - can_state for _, can_state in described - ) - - -@settings(max_examples=200, deadline=None, derandomize=True) -@given(key=st.sampled_from(sorted(CLEARANCE_SPECS)), data=st.data()) -def test_a_clearance_line_reads_only_when_both_releases_state_its_unit(key, data): - """For any labels and notes in two releases, the importer reads a line - exactly when every unit each release states for it is the line's unit - and each states it where the OBR does. So it never reads a pair of - releases whose units disagree for the line, and whenever it reads one, - the growth is the clearances'.""" - previous = data.draw(clearance_release(key), label="previous") - current = data.draw(clearance_release(key), label="current") - spec = CLEARANCE_SPECS[key] - reads = clearance_release_reads(key, *previous) and clearance_release_reads( - key, *current - ) - try: - growth = compute_receipts_growth( - mixed_unit_workbook( - clearance_release_rows(key, *current, 2025, (1000, 950)) - ), - mixed_unit_workbook(clearance_release_rows(key, *previous, 2024, (1000,))), - specs=(spec,), - )[spec.path] - except ValueError as error: - event("refused") - assert not reads, str(error) - assert "cannot read" in str(error) - return - event("read") - assert reads - assert growth.values == {2025: 0.0, 2026: -0.05} + event("unread: named") + assert str(refused.value) == unread(sheet, title, (text, column)) # per_head_growth divides each series' own growth, a duty's base for a line diff --git a/policyengine_uk/utils/import_obr_forecasts.py b/policyengine_uk/utils/import_obr_forecasts.py index ec85a672d3..cb17a614f0 100644 --- a/policyengine_uk/utils/import_obr_forecasts.py +++ b/policyengine_uk/utils/import_obr_forecasts.py @@ -8,7 +8,6 @@ from dataclasses import dataclass, replace from datetime import date from decimal import Decimal -from fractions import Fraction from functools import lru_cache from io import BytesIO from pathlib import Path @@ -679,9 +678,10 @@ class ReceiptsSeriesSpec: def per_head(self) -> bool: """Whether the series sits under ``per_capita``. Dataset projection grows household weights by ``ons.population``, so - ``render_receipts_growth_yaml`` writes such a series' growth over one - plus that growth (``per_head_growth``, ``per_head_tail``), and the - weighted total grows with the line in every year.""" + ``render_receipts_growth_yaml`` writes such a series as + (1 + aggregate growth) / (1 + ons.population growth) - 1 + (``per_head_growth``, ``per_head_tail``), and the weighted total grows + with the line in every year.""" return self.path[0] == "per_capita" @@ -691,35 +691,27 @@ class ClearanceSeriesSpec(ReceiptsSeriesSpec): household quantity inputs per head (#2596). The OBR counts clearances in quantities, not pounds, so a clearance - table has no "£ billion" cell. ``unit`` holds the unit as the OBR states - it in the table, in a row label or a note, and ``products`` the names a - note may give the line's product, lower case and without hyphens. In - both EFOs the importer requires every unit the OBR states for the line - to be this one (``validate_clearance_unit``). Dataset projection grows - household weights by ``ons.population``, so ``render_receipts_growth_yaml`` - writes one plus the clearances' growth, over one plus that growth, minus - one (``per_head_growth``), and the weighted total grows with the - clearances. After the OBR's - forecast the importer holds the clearances flat, as ``road_fuel_volume`` - holds road fuel volume. + table has no "£ billion" cell. ``unit_cell`` holds the text of the cell + in which the OBR states the line's unit, as ``CLEARANCE_TABLE_TEXTS`` + lists it. In both EFOs the importer requires that cell and refuses any + text in the table that the list lacks (``validate_clearance_table``). + Dataset projection grows household weights by ``ons.population``, so + ``render_receipts_growth_yaml`` writes + (1 + aggregate clearance growth) / (1 + ons.population growth) - 1 + (``per_head_growth``), and the weighted total grows with the + clearances. After the OBR's forecast the importer holds the clearances + flat, as ``road_fuel_volume`` holds road fuel volume. """ - unit: str = "" - products: tuple[str, ...] = () + unit_cell: str = "" def __post_init__(self): name = ".".join(self.path) - if not self.unit: - raise ValueError(f"{name}: a clearance line needs a unit") - units, unknown = read_clearance_units(self.unit) - if len(units) != 1 or unknown: + if self.unit_cell not in CLEARANCE_TABLE_TEXTS.get(self.title, ()): raise ValueError( - f'{name}: the importer does not know "{self.unit}" as one ' - "clearance unit" - ) - if not self.products: - raise ValueError( - f"{name}: a clearance line needs the names a note may give its product" + f"{name}: a clearance line needs the cell that states its " + f"unit among its table's read texts, and {self.unit_cell!r} " + "is not one" ) # The importer links clearance outturns to the previous EFO's outturn. if self.rates or self.outturn is not None: @@ -730,264 +722,102 @@ def __post_init__(self): ) -# Clearance units as a class (#2596). The importer knows a count of sticks, -# a mass in kg and a volume of alcohol or of product in litres or -# hectolitres, each after an optional number and any of the scales below, -# and compares two units by measure and size: "'000s kg" and "thousand kg" -# agree, "thousand sticks" and "million sticks" do not. It reads any other -# word for a quantity or a scale, such as "tonnes", "grams", "litres" -# without "of alcohol" or "of product", or a scale before no unit it knows, -# as a unit it does not know, which needs deliberate importer support. -CLEARANCE_SCALES = ( - (r"'?000'?s?", 1_000), - (r"thousands?", 1_000), - (r"hundreds?", 100), - (r"millions?|mn", 1_000_000), - (r"billions?|bn", 1_000_000_000), -) -CLEARANCE_MEASURES = ( - (r"sticks?", "sticks", 1), - (r"kgs?|kilogram(?:me)?s?", "kg", 1), - (r"(?:litre|liter)s? of alcohol", "litres of alcohol", 1), - (r"(?:hectolitre|hectoliter)s? of alcohol", "litres of alcohol", 100), - (r"(?:litre|liter)s? of product", "litres of product", 1), - (r"(?:hectolitre|hectoliter)s? of product", "litres of product", 100), -) -OTHER_QUANTITY_WORDS = ( - r"grams?|grammes?|g|mg|kilos?|tonnes?|tons?|lbs?|pounds?|ounces?|oz" - r"|(?:hecto|kilo|deca|deci|centi|milli)?(?:litre|liter)s?" - r"|hl|ml|cl|kl|l|hla|lpa|gallons?|pints?|barrels?" - r"|packs?|packets?|cartons?|pouches?|units|pence|per ?cent" -) -# The importer reads a number or a scale only after something other than a -# letter, digit, point, comma, apostrophe or hyphen, and a measure or another -# quantity word only after something other than a letter or a point, so it -# reads "10kg" and skips the "g" of "e.g.". -_CLEARANCE_SCALE = ( - r"(?\d[\d,]*(?:\.\d+)?)\s*)?" - rf"(?P(?:{_CLEARANCE_SCALE}(?:\s+of)?\s*)*)" - rf"(?P{_CLEARANCE_MEASURE})" -) -QUANTITY_WORD_RE = re.compile( - rf"{_CLEARANCE_SCALE}|(? str: - """A cell's text as the importer reads units and products in it: lower - case, single spaces and straight apostrophes.""" - return normalise_label(value).translate({0x2018: "'", 0x2019: "'"}) - - -def clearance_unit_size(match: re.Match) -> tuple[str, Fraction]: - """A unit's measure and size, such as ("kg", 1000) for "'000s kg".""" - number = match.group("number") - size = Fraction(number.replace(",", "")) if number else Fraction(1) - for scale in re.finditer(_CLEARANCE_SCALE, match.group("scales")): - size *= next( - factor - for pattern, factor in CLEARANCE_SCALES - if re.fullmatch(pattern, scale.group(0)) - ) - measure, factor = next( - (measure, factor) - for pattern, measure, factor in CLEARANCE_MEASURES - if re.fullmatch(pattern, match.group("measure")) - ) - return measure, size * factor - - -def read_clearance_units( - value: str | None, -) -> tuple[list[tuple[tuple[str, Fraction], str, int, int]], list[str]]: - """Each unit the importer knows in a cell's text, as (measure and size, - spelling, start, end), and each word for a quantity or scale outside - them: a word ``QUANTITY_WORD_RE`` matches outside every known unit, or - what follows "is in" or "are in" when no known unit starts there.""" - text = clearance_text(value) - units = [ - (clearance_unit_size(match), match.group(0), match.start(), match.end()) - for match in CLEARANCE_UNIT_RE.finditer(text) - ] - unknown = [ - match.group(0) - for match in QUANTITY_WORD_RE.finditer(text) - if not any( - start <= match.start() and match.end() <= end for _, _, start, end in units - ) - ] - for match in UNIT_STATEMENT_RE.finditer(text): - if not any(start == match.end() for _, _, start, _ in units): - unknown.append(CLAUSE_END_RE.split(text[match.end() :])[0].strip()) - return units, unknown - - -@dataclass(frozen=True) -class ClearanceStatement: - """The units the importer reads in a cell of a clearance table. - - ``units`` holds each unit the OBR states in the cell, as (measure and - size, spelling, the paths of the lines the importer ties it to, or - ``None`` for every line); ``products`` the paths of the lines whose - products the cell names, or ``None``; ``unknown`` each word for a - quantity or scale the importer does not know; ``untied`` whether the - importer cannot tie each of two or more units to products.""" - - units: tuple[tuple[tuple[str, Fraction], str, frozenset | None], ...] - products: frozenset | None - unknown: tuple[str, ...] - untied: bool - - -def read_clearance_statement( - value: str | None, products: dict[str, frozenset[tuple[str, ...]]] -) -> ClearanceStatement: - """Read the units in a cell, given ``products``, which maps each name a - cell may give a line's product to the lines' paths. - - When the OBR states one unit in a cell, however often, the importer ties - it to the lines whose products the cell names, or to every line when the - cell names none. When the OBR states two or more, the importer ties each - to the products named in its own clause, up to a semicolon or a full - stop, and cannot tie a clause with two units or no product. - """ - units, unknown = read_clearance_units(value) - text = list(clearance_text(value)) - for _, _, start, end in units: - text[start:end] = " " * (end - start) - text = "".join(text).replace("-", " ") - if not products: - raise ValueError("a clearance statement needs the names of products") - pattern = re.compile( - r"(? frozenset | None: - found = frozenset().union( - *(paths for position, paths in named if start <= position < end) - ) - return found or None - - whole = lines(0, len(text)) - if len({size for size, _, _, _ in units}) <= 1: - return ClearanceStatement( - units=tuple((size, spelling, whole) for size, spelling, _, _ in units), - products=whole, - unknown=tuple(unknown), - untied=False, - ) - tied = [] - start = 0 - for end in [match.end() for match in CLAUSE_END_RE.finditer(text)] + [len(text)]: - clause = [unit for unit in units if start <= unit[2] < end] - described = lines(start, end) - if clause and (described is None or len({unit[0] for unit in clause}) > 1): - return ClearanceStatement( - units=tuple(tied), products=whole, unknown=tuple(unknown), untied=True - ) - tied += [(size, spelling, described) for size, spelling, _, _ in clause] - start = end - return ClearanceStatement( - units=tuple(tied), products=whole, unknown=tuple(unknown), untied=False - ) - - TOBACCO_CLEARANCES = "tobacco clearances" ALCOHOL_CLEARANCES = "alcohol clearance forecasts" -# The OBR states each unit in Table 4.11 (3.11 in March 2026) in a row -# label, and splits "Non-cigarette clearances ('000s kg)" into the -# hand-rolled and other tobacco lines. It states Table 4.12's (3.12's) unit -# in a note. -TOBACCO_STICKS = "(million sticks)" -TOBACCO_KILOGRAMS = "('000s kg)" -ALCOHOL_HECTOLITRES = "thousands of hectolitres of alcohol" +# The cells in which the OBR states each clearance unit: the cigarette label +# for cigarettes, the label of the non-cigarette total that Table 4.11 (3.11 +# in March 2026) splits into hand-rolled and other tobacco, and Table 4.12's +# (3.12's) note for alcohol. +TOBACCO_STICKS = "cigarette clearances (million sticks)" +TOBACCO_KILOGRAMS = "non-cigarette clearances ('000s kg)" +ALCOHOL_HECTOLITRES = "note: all clearances are in thousands of hectolitres of alcohol." +# The text of every cell below each clearance table's title in Tables 3.11 +# and 3.12 of the March 2025 and March 2026 EFOs and Tables 4.11 and 4.12 of +# October 2026, but numbers and fiscal years. Someone read each one. The +# importer compares a cell's text as receipts_label gives it: lower case, in +# single spaces and without a footnote number after a label's last letter, +# so " Wine1" gives "wine". It refuses any other text +# (validate_clearance_table), so it reads no unit or product from words no +# one has read. Add a text only after reading it, and when it states another +# unit, convert the table's figures to the units these texts state first. +CLEARANCE_TABLE_TEXTS = { + TOBACCO_CLEARANCES: frozenset( + { + "outturn", + "forecast", + TOBACCO_STICKS, + TOBACCO_KILOGRAMS, + "of which:", + "hand rolled tobacco clearances", + "other tobacco products", + } + ), + ALCOHOL_CLEARANCES: frozenset( + { + "outturn", + "forecast", + "beer and cider", + "wine", + "spirits", + ALCOHOL_HECTOLITRES, + "1 includes spirit-based ‘ready-to-drinks’ such as alcopops.", + "1 includes other fermented products such as fruit ciders.", + "2 includes spirit-based ‘ready-to-drinks’ such as alcopops.", + } + ), +} +# The titles, as row_title gives them, of the tables that may follow each +# clearance table on its sheet. The importer checks a clearance table from +# its title down to the row above one of these titles, or to the sheet's +# end. In all three EFOs the alcohol receipts forecasts follow the alcohol +# clearances, and no table follows the tobacco clearances. +CLEARANCE_TABLE_ENDS = { + TOBACCO_CLEARANCES: frozenset(), + ALCOHOL_CLEARANCES: frozenset({"alcohol receipts forecasts"}), +} # The product groups the OBR forecasts. The OBR includes other fermented # products in Table 4.12's wine line and spirit-based ready-to-drinks in its -# spirits line (the table's footnotes 1 and 2). It splits Table 4.11's -# non-cigarette total into hand-rolled tobacco and other tobacco products, -# so the importer reads "non-cigarette" as naming both, and other tobacco -# products as every non-cigarette product but hand-rolled tobacco, such as -# cigars and heated tobacco. +# spirits line (the table's footnotes 1 and 2). CLEARANCE_SERIES_SPECS = ( ClearanceSeriesSpec( path=("per_capita", "tobacco_clearances", "cigarettes"), title=TOBACCO_CLEARANCES, row="cigarette clearances (million sticks)", - unit=TOBACCO_STICKS, - products=("cigarettes", "cigarette"), + unit_cell=TOBACCO_STICKS, ), ClearanceSeriesSpec( path=("per_capita", "tobacco_clearances", "hand_rolling_tobacco"), title=TOBACCO_CLEARANCES, row="hand rolled tobacco clearances", - unit=TOBACCO_KILOGRAMS, - products=("hand rolled", "hand rolling", "non cigarette"), + unit_cell=TOBACCO_KILOGRAMS, ), ClearanceSeriesSpec( path=("per_capita", "tobacco_clearances", "other_tobacco_products"), title=TOBACCO_CLEARANCES, row="other tobacco products", - unit=TOBACCO_KILOGRAMS, - products=( - "other tobacco", - "cigars", - "cigar", - "heated tobacco", - "non cigarette", - ), + unit_cell=TOBACCO_KILOGRAMS, ), ClearanceSeriesSpec( path=("per_capita", "alcohol_clearances", "beer_and_cider"), title=ALCOHOL_CLEARANCES, row="beer and cider", - unit=ALCOHOL_HECTOLITRES, - products=("beer", "beers", "cider", "ciders"), + unit_cell=ALCOHOL_HECTOLITRES, ), ClearanceSeriesSpec( path=("per_capita", "alcohol_clearances", "wine"), title=ALCOHOL_CLEARANCES, row="wine", - unit=ALCOHOL_HECTOLITRES, - products=("wine", "wines", "other fermented", "fruit cider", "fruit ciders"), + unit_cell=ALCOHOL_HECTOLITRES, ), ClearanceSeriesSpec( path=("per_capita", "alcohol_clearances", "spirits"), title=ALCOHOL_CLEARANCES, row="spirits", - unit=ALCOHOL_HECTOLITRES, - products=("spirits", "spirit", "ready to drinks", "alcopops"), + unit_cell=ALCOHOL_HECTOLITRES, ), ) - GAMBLING_RATES = "gov.hmrc.gambling_duty" MACHINE_GAMES = f"{GAMBLING_RATES}.machine_games" @@ -1410,186 +1240,70 @@ def validate_receipts_unit( ) -def row_cells(row: dict[str, str | None]) -> list[tuple[str, str]]: - """A row's non-empty cells, left to right, as (column, normalised text).""" - return [ - (column, normalise_label(value)) - for column, value in sorted(row.items(), key=lambda item: column_index(item[0])) - if normalise_label(value) - ] +# A number as a worksheet stores it, such as "1272.4562775587838" or "1E-3". +NUMBER_RE = re.compile(r"[-+]?(?:\d+\.?\d*|\.\d+)(?:e[-+]?\d+)?") -def validate_clearance_unit( - table_rows: list[dict[str, str | None]], - line_index: int, +def validate_clearance_table( + rows: list[dict[str, str | None]], + number: str, spec: ClearanceSeriesSpec, - years: dict[str, int], where: str, ) -> None: - """Require every unit the OBR states for a clearance line to be the - line's unit, and the unit where the OBR states it. - - The importer reads every unit of its closed list (``CLEARANCE_UNIT_RE``) - wherever the OBR states it in the table, and every word for a quantity - or scale outside the list. It reads a unit as the line's in: - - - the line's own label, as Table 4.11's "Cigarette clearances (million - sticks)"; - - the label of the total the line breaks down: the row above an "of - which:" row, with only the total's other parts between that and the - line, as "Non-cigarette clearances ('000s kg)" for hand-rolled and - other tobacco; - - another line's label, when it names this line's product - (``spec.products``); - - any other cell, such as a note, when it names this line's product or - no product, as Table 4.12's "Note: All clearances are in thousands of - hectolitres of alcohol". A note that names only other products says - nothing about this line. - - The importer needs the unit where the OBR states it: in the line's - label, the total's label or a note on a row of its own that opens - "Note:". It refuses the line when the OBR states another unit or scale - for it, or a word for a quantity or scale the importer does not know in - a cell the importer reads as the line's; when the OBR puts such a word - in any note, heading or footnote, or states two units there that the - importer cannot tie each to products in its own clause; and when the - importer finds the unit only somewhere else, such as a note about - another product. The first year's growth divides one EFO's level by - another's, so the OBR must state the same unit in both; a new unit or - layout needs deliberate importer support. + """Refuse a clearance table that holds text no one has read. + + The importer checks every cell of the table numbered ``number`` and + titled ``spec.title`` on a sheet's ``rows``, in every column, from the + title's row down to the row above a title ``CLEARANCE_TABLE_ENDS`` + lists, or to the sheet's end. It skips the title, numbers and fiscal + years, and needs every other text in ``CLEARANCE_TABLE_TEXTS`` and the + line's ``unit_cell`` among them. It interprets no words, so a note that + states another unit, a line for another product or a unit column stops + the run until someone reads the text and lists it. """ - name = ".".join(spec.path) - ((expected, _, _, _),), _ = read_clearance_units(spec.unit) - products: dict[str, frozenset] = {} - for line_spec in ( - *(other for other in CLEARANCE_SERIES_SPECS if other.title == spec.title), - spec, - ): - for product in line_spec.products: - products[product] = products.get(product, frozenset()) | {line_spec.path} - first_year_column = min(column_index(column) for column in years) - year_rows = { - index for index, row in enumerate(table_rows) if _fiscal_year_columns(row) - } - - def is_line(index: int) -> bool: - cells = row_cells(table_rows[index]) - return ( - index > min(year_rows) - and index not in year_rows - and bool(cells) - and column_index(cells[0][0]) < first_year_column - and any(column in years for column, _ in cells) - ) - - lines = {index for index in range(len(table_rows)) if is_line(index)} - above = line_index - 1 - while above in lines: - above -= 1 - total = ( - above - 1 - if above - 1 in lines - and [text for _, text in row_cells(table_rows[above])] == ["of which:"] - else None + start = next( + index + for index, row in enumerate(rows) + if row_title(row) == (number, spec.title) ) - - problems = [] - found = [] - other_products = [] - supplied = False - - def check(text: str, units, unknown) -> list: - """Each problem with the units a cell states for the line, and the - units that agree.""" - problems.extend( - f'the importer does not know the unit or scale "{word}" in {text!r}' - for word in dict.fromkeys(unknown) - ) - problems.extend( - f'the OBR states "{spelling}", another unit or scale, in {text!r}' - for size, spelling, _ in units - if size != expected - ) - return [unit for unit in units if unit[0] == expected] - - def unknown_in_brackets(text: str, unknown: list[str]) -> list[str]: - # The OBR states a label's unit in brackets, so the importer reads - # anything else there as a unit it does not know. - return [ - group - for group in re.findall(r"\(([^()]*)\)", clearance_text(text)) - if not CLEARANCE_UNIT_RE.fullmatch(group) - and not any(word in group for word in unknown) + block = [rows[start]] + for row in rows[start + 1 :]: + title = row_title(row) + if title is not None and title[1] in CLEARANCE_TABLE_ENDS[spec.title]: + break + block.append(row) + read, unread = set(), [] + for index, row in enumerate(block): + years = _fiscal_year_columns(row) + cells = [ + (column, value) + for column, value in sorted( + row.items(), key=lambda item: column_index(item[0]) + ) + if receipts_label(value) ] - - # The line's own label, and the label of the total it breaks down. - for index in (line_index, total): - if index is None: - continue - text = row_cells(table_rows[index])[0][1] - units, unknown = read_clearance_units(text) - agree = check( - text, - [(size, spelling, None) for size, spelling, _, _ in units], - [*unknown, *unknown_in_brackets(text, unknown)], + # The first text on the title's row is the title. + for column, value in cells[1:] if index == 0 else cells: + text = receipts_label(value) + if column in years or NUMBER_RE.fullmatch(text): + continue + read.add(text) + if text not in CLEARANCE_TABLE_TEXTS[spec.title]: + unread.append(f"{value!r} in column {column}") + if unread: + raise ValueError( + f"{where} holds text the importer has not read: " + + "; ".join(unread) + + ". Read each one. If it states another unit, convert the " + "table's figures to the units the read texts state. Then add the " + "text, as receipts_label gives it, to " + f'CLEARANCE_TABLE_TEXTS["{spec.title}"], or the title of a table ' + "that follows the clearances to CLEARANCE_TABLE_ENDS." ) - supplied = supplied or bool(agree) - - # Every other cell: another line's label, a heading, a note or a - # footnote. - for index, row in enumerate(table_rows): - if index in (line_index, total) or index in year_rows: - continue - cells = row_cells(row) - for _, text in cells[:1] if index in lines else cells: - statement = read_clearance_statement(text, products) - described = [ - unit - for unit in statement.units - if unit[2] is None or spec.path in unit[2] - ] - if index in lines: - # The importer reads another line's label as that line's, and - # as this line's only when it names this line's product. - if statement.products is None or spec.path not in statement.products: - if any(unit[0] == expected for unit in statement.units): - found.append(text) - continue - unknown = [ - *statement.unknown, - *unknown_in_brackets(text, statement.unknown), - ] - else: - unknown = statement.unknown - if statement.untied: - problems.append( - f"the importer cannot tie each unit in {text!r} to products " - "in its own clause" - ) - agree = check(text, described, unknown) - if agree and len(cells) == 1 and text.startswith("note"): - supplied = True - elif agree: - found.append(text) - elif any(unit[0] == expected for unit in statement.units): - other_products.append(text) - - if not supplied and not problems: - if other_products and not found: - problems.append( - f"the importer found the unit only in {other_products}, which " - "name other products" - ) - else: - problems.append( - "the importer needs the unit in its label, the label of the " - "total it breaks down or a note that applies to it, and found it " - f"only in {found + other_products}" - ) - if problems: + if spec.unit_cell not in read: raise ValueError( - f'The importer cannot read {name} in "{spec.unit}" from {where}: ' - + "; ".join(problems) + f"{where} lacks {spec.unit_cell!r}, the cell in which the OBR " + f"states the unit of {'.'.join(spec.path)}" ) @@ -1598,9 +1312,11 @@ def find_receipts_line( ) -> ReceiptsLine: """The one row labelled ``spec.row`` in a table titled ``spec.title``. - The importer reads only that table's rows: its cash unit, or a clearance - line's unit, its year headers and its "Outturn" and "Forecast" markers, - which may sit on one row or several. + The importer reads only that table's rows: its cash unit, its year + headers and its "Outturn" and "Forecast" markers, which may sit on one + row or several. For a clearance line it first checks every cell of the + line's table against the texts someone has read + (``validate_clearance_table``). """ matches = [] for sheet, rows in sheets.items(): @@ -1609,14 +1325,12 @@ def find_receipts_line( continue where = f"Table {table} (sheet {sheet})" years: dict[str, int] = {} - year_row: dict[str, str | None] = {} markers: dict[str, int] = {} header_rows = [] - for index, row in enumerate(table_rows): + for row in table_rows: row_years = _fiscal_year_columns(row) if row_years: years = row_years - year_row = row continue if not years: header_rows.append(row) @@ -1646,18 +1360,7 @@ def find_receipts_line( ] if labels and receipts_label(labels[0][1]) == spec.row: matches.append( - ( - table, - sheet, - labels[0], - row, - years, - markers, - header_rows, - table_rows, - index, - year_row, - ) + (table, sheet, labels[0], row, years, markers, header_rows) ) name = ".".join(spec.path) @@ -1667,19 +1370,10 @@ def find_receipts_line( f'Expected one "{spec.row}" row in the "{spec.title}" table for ' f"{name}; found {found}" ) - ( - table, - sheet, - (label_column, label), - row, - years, - markers, - header_rows, - table_rows, - line_index, - year_row, - ) = matches[0] + table, sheet, (label_column, label), row, years, markers, header_rows = matches[0] where = f"Table {table} (sheet {sheet})" + if isinstance(spec, ClearanceSeriesSpec): + validate_clearance_table(sheets[sheet], table, spec, where) repeated = sorted({y for y in years.values() if list(years.values()).count(y) > 1}) if repeated: raise ValueError( @@ -1699,22 +1393,7 @@ def find_receipts_line( f"{where} holds {beside} beside the {name} row's label and values; " "a unit or note on the row needs deliberate importer support" ) - if isinstance(spec, ClearanceSeriesSpec): - # The OBR states each clearance unit in a label or a note, so the - # importer does not read a cell beside the years, such as a unit - # column's heading. - heading = { - column: value - for column, value in year_row.items() - if normalise_label(value) and column not in years - } - if heading: - raise ValueError( - f"{where} holds {heading} in its fiscal-year row; the OBR " - "states each clearance unit in a label or a note" - ) - validate_clearance_unit(table_rows, line_index, spec, years, where) - else: + if not isinstance(spec, ClearanceSeriesSpec): validate_receipts_unit(header_rows, where) levels = {} for column, year in years.items(): From 7817dbcbb8df33b639a142ed0340b55207f4d6ae Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 1 Nov 2026 08:27:45 -0500 Subject: [PATCH 12/27] Describe the closed list of clearance texts in the guide and README The excise duties guide and the economic assumptions README now say what the importer does: it interprets no words in the clearance tables, checks every cell against the texts someone has read, needs each line's unit cell, and names the table and cell it refuses. The README keeps the check that each alcohol line covers the same products in both EFOs, and the changelog states the projection. Co-Authored-By: Claude Opus 5.5 --- changelog.d/2596.fixed.md | 2 +- docs/engineering/excise-duties.md | 39 +++++++++++-------- .../gov/economic_assumptions/README.md | 28 +++++++------ 3 files changed, 40 insertions(+), 29 deletions(-) diff --git a/changelog.d/2596.fixed.md b/changelog.d/2596.fixed.md index ffbdac2355..b993355a07 100644 --- a/changelog.d/2596.fixed.md +++ b/changelog.d/2596.fixed.md @@ -1 +1 @@ -Dataset projection now grows the alcohol and tobacco quantity inputs by the OBR's clearance forecasts per person, one index for each product group, so later years no longer charge duty on base-year quantities. +Dataset projection grows alcohol and tobacco quantity inputs with the OBR's forecasts of clearances per person, using one index per product group. The model charges later duty rates on those projected quantities. diff --git a/docs/engineering/excise-duties.md b/docs/engineering/excise-duties.md index 0ebb12d5c1..e6531baaca 100644 --- a/docs/engineering/excise-duties.md +++ b/docs/engineering/excise-duties.md @@ -274,25 +274,30 @@ The importer, `policyengine_uk/utils/import_obr_forecasts.py`, writes the six series from `CLEARANCE_SERIES_SPECS` in the same run as the other receipts series (see the steps for a new EFO in the economic assumptions README), and reads `ons.population` from the `yoy_growth.yaml` it rewrites. -It reads every unit the OBR states in a clearance table against a closed -list: sticks, kg, and litres or hectolitres of alcohol or of product, each -after an optional number and a scale such as "'000s", "thousands of" or -"million". It compares units by measure and size, so "'000s kg" and -"thousand kg" agree. It reads a unit as a line's when the OBR states it in -the line's own label, in the label of the total the line breaks down, in -another line's label that names the line's product, or in a note that names -the line's product or no product, and ignores a note that names only other -products. It stops before writing anything when: - -- the OBR states another unit or scale for a line; -- the OBR states two units in one note, and the importer cannot tie each to - the products named in its own clause; -- the OBR uses a word for a quantity or scale outside the list, such as - "tonnes", in a note or in a cell the importer reads as a line's; -- the OBR states a line's unit in none of its label, its total's label and a - note beginning "Note:" that names its product or no product; +It interprets no words in the clearance tables. `CLEARANCE_TABLE_TEXTS` +lists each text, other than titles, numbers and fiscal years, in Tables 3.11 +and 3.12 of the March 2025 and March 2026 EFOs and Tables 4.11 and 4.12 of +October 2026, from each title down to the next table's title: the line +labels, the "Outturn" and "Forecast" markers, the "of which:" row, Table +4.12's note "All clearances are in thousands of hectolitres of alcohol" and +its footnotes. On Table 4.12's sheet the OBR follows the alcohol clearances +with the alcohol receipts forecasts. The importer ends the clearance table +at their title (`CLEARANCE_TABLE_ENDS`), so its check leaves out the +receipts table's "£ billion" cell and note. The importer checks every cell +of each clearance table it reads, in every column, skips numbers and fiscal +years, and stops before writing anything when: + +- a cell holds any other text, such as a note that states another unit or + scale, a line for another product or a unit column's heading; +- the table lacks the cell that states a line's unit: the cigarette label's + "(million sticks)", the non-cigarette total's "('000s kg)" for hand-rolled + and other tobacco, or Table 4.12's note for alcohol; - the OBR puts any cell beside a line's label and values. +The importer names the table and each unread cell. Whoever runs it reads +the text, converts the table's figures to the listed units if it states +another, and adds the text to the list. + ### Budget measures and reforms The OBR incorporates in its October 2026 forecast the measures the diff --git a/policyengine_uk/parameters/gov/economic_assumptions/README.md b/policyengine_uk/parameters/gov/economic_assumptions/README.md index 4e73526ba2..3f015b74f3 100644 --- a/policyengine_uk/parameters/gov/economic_assumptions/README.md +++ b/policyengine_uk/parameters/gov/economic_assumptions/README.md @@ -511,17 +511,23 @@ the forecast levels, though not their growth. The same run also refreshes the six alcohol and tobacco clearance series under `per_capita` from the new EFO's tobacco clearances and alcohol clearance forecasts, and the previous EFO's for the outturn year's base. - The importer finds each line's unit in both EFOs where the OBR states - it: in the line's own label, in the label of the total the line breaks - down, or in a note. It takes million sticks for cigarettes, thousand - kilograms for the other tobacco lines, and thousand hectolitres of - alcohol for alcohol. It stops before writing on any other unit, and on - any cell beside a line's label and values, such as a unit column. It - does not read the lines' footnotes, so check that each alcohol line - covers the same products in both EFOs: in March 2026 the OBR added - "Includes other fermented products" to the wine line. The importer - divides each year's growth by `ons.population` as `yoy_growth.yaml` - holds it, so rerun it whenever that series changes. Until then the six + The importer interprets no words in these tables. It checks every cell + of both EFOs' tables, but numbers and fiscal years, against the texts + someone has read (`CLEARANCE_TABLE_TEXTS`). It also needs the cell that + states each line's unit: million sticks in the cigarette label, + thousand kilograms in the non-cigarette total's label and thousand + hectolitres of alcohol in Table 4.12's note. It stops before writing on + any other text and names the table and the cell. Read that text, convert + the table's figures to these units if it states another, and add the + text to the list. The importer also stops on any cell beside a line's + label and values. The list holds each EFO's footnotes as the OBR wrote + them, and the importer does not compare the products one EFO's line + covers with another's. Check that each alcohol line covers the same + products in both EFOs: in March 2026 the OBR added "Includes other + fermented products" to the wine line. The importer writes + `(1 + aggregate clearance growth) / (1 + ons.population growth) - 1`, + using the population in `yoy_growth.yaml`, so rerun it whenever that + series changes. Until then the six clearance series and the seven gambling bases keep the values written for the old population, and the byte-for-byte test fails. Then update the published levels in From 45e935ddf8ce66cec584c2391ffa64e527fe6232 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 1 Nov 2026 08:47:33 -0500 Subject: [PATCH 13/27] Give the clearance check's text active verbs; test a title-shaped note The spec's error says which texts lack the unit cell, the docstrings name who acts, and the per-head docstrings write the ratio as one plus the aggregate growth over one plus population growth. The refusal test adds a note shaped like a table's title, and the property draws two such texts, since sheet_tables starts a table at one. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_import_obr_forecasts.py | 106 ++++++++++-------- policyengine_uk/utils/import_obr_forecasts.py | 14 +-- 2 files changed, 67 insertions(+), 53 deletions(-) diff --git a/policyengine_uk/tests/test_import_obr_forecasts.py b/policyengine_uk/tests/test_import_obr_forecasts.py index 0749ce3767..42fa44e991 100644 --- a/policyengine_uk/tests/test_import_obr_forecasts.py +++ b/policyengine_uk/tests/test_import_obr_forecasts.py @@ -3077,12 +3077,12 @@ def test_a_clearance_spec_needs_a_read_unit_cell_and_takes_no_rates_or_outturn() spec = CLEARANCE_SPECS["wine"] with pytest.raises(ValueError, match="needs the cell that states its unit"): ClearanceSeriesSpec(path=("per_capita", "x"), title=ALCOHOL_CLEARANCES, row="r") - with pytest.raises(ValueError, match=r"and '\(tonnes\)' is not one"): + with pytest.raises(ValueError, match=r"texts lack '\(tonnes\)'"): replace(spec, unit_cell="(tonnes)") # Each table's list holds its own cells. - with pytest.raises(ValueError, match="is not one"): + with pytest.raises(ValueError, match="texts lack 'cigarette clearances"): replace(spec, unit_cell=CLEARANCE_SPECS["cigarettes"].unit_cell) - with pytest.raises(ValueError, match="is not one"): + with pytest.raises(ValueError, match="needs the cell that states its unit"): replace(spec, title="alcohol receipts forecasts") with pytest.raises(ValueError, match="takes no rates"): replace(spec, rates=GAMBLING_SPECS[0].rates) @@ -3474,8 +3474,8 @@ def cell(reference): def real_levels(workbook, key): - """The levels the importer reads for ``key`` from its table alone in - ``workbook``, and from the real October 2026 table.""" + """Read the levels of ``key`` from its table alone in ``workbook``, and + from the real October 2026 table.""" spec = CLEARANCE_SPECS[key] table = "4.11" if spec.title == TOBACCO_CLEARANCES else "4.12" return ( @@ -3484,7 +3484,7 @@ def real_levels(workbook, key): ) -# The importer reads no words in a clearance table: each cell must hold a +# The importer interprets no words in a clearance table: each cell must hold a # number, a fiscal year or a text someone has read (CLEARANCE_TABLE_TEXTS). TOBACCO_LINES = ("cigarettes", "hand_rolling_tobacco", "other_tobacco_products") @@ -3502,9 +3502,9 @@ def real_levels(workbook, key): @lru_cache(maxsize=None) def real_clearance_rows(release, title): - """The rows of a real clearance table's sheet, and of its table: from - its title's row to the row above the next table's title, or to the - sheet's end, with that title in lower case, without its number.""" + """Read a real clearance table's sheet and find the table: its title's + row, the row of the next table's title or the sheet's end, and that + title in lower case without its number.""" workbook, sheet = REAL_CLEARANCE_TABLES[release, title] rows = read_sheet_rows(workbook, sheet) titles = [" ".join(str(row.get("B") or "").split()) for row in rows] @@ -3518,8 +3518,8 @@ def real_clearance_rows(release, title): def unread(table, title, *cells): - """The importer's refusal of real Table ``table`` for the unread - ``cells``, each as (text, column).""" + """Write out the importer's refusal of real Table ``table`` for the + unread ``cells``, each as (text, column).""" return ( f"Table {table} (sheet {table}) holds text the importer has not read: " + "; ".join(f"{text!r} in column {column}" for text, column in cells) @@ -3531,8 +3531,9 @@ def unread(table, title, *cells): def test_both_real_release_pairs_read_every_series(): - """March 2026 over March 2025 and October 2026 over March 2026 each read - all 17 series, the six clearance lines among them.""" + """The importer reads all 17 series, the six clearance lines among + them, from each real pair: March 2026 over March 2025 and October 2026 + over March 2026.""" for current, previous in ( (REAL_RECEIPTS[2026], REAL_RECEIPTS[2025]), (REAL_OCTOBER_2026_RECEIPTS, REAL_RECEIPTS[2026]), @@ -3720,6 +3721,13 @@ def test_the_read_texts_are_the_texts_of_the_three_releases(): "other-table-text": ("4.11", {"B10": "Wine"}, {}, []), # Text where a number belongs. "not-a-number": ("4.11", {"D5": "nan"}, {}, []), + # A note shaped like a table's title, which sheet_tables reads as one. + "title-shaped-note": ( + "4.11", + {}, + {}, + ["1.1 Note: Cigarette forecasts shown in ten million sticks."], + ), } @@ -3727,7 +3735,7 @@ def test_the_read_texts_are_the_texts_of_the_three_releases(): def test_text_no_one_has_read_stops_the_real_tables(name): """For each edit, the importer refuses every line of the edited table, naming each unread cell, and the run stops before it renders anything; - the other table's lines read as before.""" + the importer reads the other table's lines as before.""" table, cells, scale, notes = UNREAD_EDITS[name] changed = edit_real_table( REAL_OCTOBER_2026_RECEIPTS, table, cells=cells, scale=scale, notes=notes @@ -3759,9 +3767,10 @@ def test_text_no_one_has_read_stops_the_real_tables(name): def test_the_scale_edits_change_the_growth_the_figures_give(): - """Read in the units the real tables state, the two rescaled tables' - figures would give cigarette clearances growth of -91.0% in 2026-27 - for -10.3%, and other tobacco -99.9% in 2025-26 for -13.6%.""" + """In the units the real tables state, the two rescaled tables' figures + give cigarette clearances growth of -91.0% in 2026-27, where the + published figures give -10.3%, and other tobacco growth of -99.9% in + 2025-26, where they give -13.6%.""" def growth(levels, key): # 2025-26 over March 2026's 2024-25 outturn, then 2026-27 over 2025-26. @@ -3786,10 +3795,10 @@ def growth(levels, key): def test_a_real_table_without_its_unit_cell_refuses_the_lines_it_describes(): - """Cleared, the non-cigarette total's label and Table 4.12's note leave - hand-rolled tobacco, other tobacco and the alcohol lines without the - cell that states their unit, and the importer refuses them; cigarettes - read as before.""" + """The helper clears the non-cigarette total's label and, separately, + Table 4.12's note. Each leaves its lines, hand-rolled and other tobacco + or the three alcohol lines, without the cell that states their unit, + and the importer refuses them; it reads cigarettes as before.""" for table, cell in (("4.11", "B6"), ("4.12", "B8")): changed = edit_real_table(REAL_OCTOBER_2026_RECEIPTS, table, cells={cell: None}) for key, spec in CLEARANCE_SPECS.items(): @@ -3810,13 +3819,14 @@ def test_a_real_table_without_its_unit_cell_refuses_the_lines_it_describes(): def test_the_alcohol_receipts_table_stays_outside_the_clearance_table(): - """On the real October 2026 sheet 4.12, the alcohol receipts forecasts - follow the clearances, with "£ billion" and "Note: All taxes are on an - accrued basis.". The clearance table ends at their title: a note in the - receipts table leaves the clearance lines reading, the same note in the - clearance table's last row stops them, and so does a receipts title the - list of ends does not name, since the clearance table then runs through - the receipts table to the sheet's end.""" + """On the real October 2026 sheet 4.12 the OBR follows the alcohol + clearances with the alcohol receipts forecasts, which hold "£ billion" + and "Note: All taxes are on an accrued basis.". The importer ends the + clearance table at their title. It reads the clearance lines with a note + or "£ million" in the receipts table, and refuses them with the same + note in the clearance table's last row, or under a receipts title + CLEARANCE_TABLE_ENDS does not name, since the clearance table then runs + to the sheet's end.""" rows, start, end, following = real_clearance_rows( "October 2026", ALCOHOL_CLEARANCES ) @@ -3849,10 +3859,9 @@ def test_the_alcohol_receipts_table_stays_outside_the_clearance_table(): def unread_text(text, title): - """Whether the importer must refuse ``text`` in a clearance table titled - ``title``: written out from the lists, not read by the importer, and - leaving out blanks, numbers, fiscal years and table titles a list - names.""" + """Decide from the lists, without the importer, whether it must refuse + ``text`` in a clearance table titled ``title``. The test leaves out + blanks, numbers, fiscal years and table titles a list names.""" label = receipts_label(text) if not label or label in CLEARANCE_TABLE_TEXTS[title]: return False @@ -3890,6 +3899,9 @@ def unread_text(text, title): "Note: Non-cigarettes are in million sticks.", "Note: Sake clearances are in thousands of hectolitres of alcohol.", "Note: All clearances are in millions of litres of alcohol.", + # Shaped like a table's title, so sheet_tables starts a table there. + "1.1 Note: Cigarette forecasts shown in ten million sticks.", + "4.13 Tobacco clearances in ten million sticks", ] ), st.builds( @@ -3931,7 +3943,7 @@ def test_any_text_no_one_has_read_stops_a_real_clearance_table(table, data): table: the importer names the cell, or finds no line or table where the text took a label's or the title's place. A text from the list, other than a line's label or a marker, in a cell outside the lines and the - year row, leaves every line reading as before.""" + year row, changes nothing: the importer reads every line as before.""" release, title = table _, sheet = REAL_CLEARANCE_TABLES[table] real, start, end, _ = real_clearance_rows(release, title) @@ -4000,11 +4012,11 @@ def test_any_text_no_one_has_read_stops_a_real_clearance_table(table, data): def test_per_head_growth_divides_the_base_growth_the_series_holds(): - """Remote Gaming Duty receipts double while the base, - receipts over the rate, grows 10%, and population grows 1%. Per head - the base grows 1.10 / 1.01 - 1, not 2 / 1.01 - 1 (receipts) or the - growth the levels and the model's rates give, with or without a previous - line.""" + """Remote Gaming Duty receipts double and its base, receipts over the + rate, grows 10%, while population grows 1%. per_head_growth returns + 1.10 / 1.01 - 1 for the base's growth per head, with or without a + previous line, not 2 / 1.01 - 1 (receipts) or the growth the levels and + the model's rates give.""" line = ReceiptsLine( table="4.10", label="Remote Gaming Duty", @@ -4037,9 +4049,10 @@ def test_per_head_growth_divides_the_base_growth_the_series_holds(): def test_real_remote_gaming_base_per_head_rounds_once(): - """The real line has its own outturn and no previous line. Per head, each - year divides the base's growth before rounding by one plus population - growth and rounds once, as the committed series holds it.""" + """The real line has its own outturn and no previous line. Each year the + importer divides one plus the base's growth before rounding by one plus + population growth, subtracts one and rounds once, as the committed + series holds it.""" growth = compute_receipts_growth( REAL_OCTOBER_2026_RECEIPTS, REAL_RECEIPTS[2026], specs=(REMOTE_GAMING,) )[REMOTE_GAMING.path] @@ -4065,11 +4078,12 @@ def test_real_remote_gaming_base_per_head_rounds_once(): def test_per_head_growth_times_population_growth_is_the_series_growth( ratios, population ): - """For any growth before rounding and any population growth: per head, - one plus the series times one plus population growth is the growth to - within the six-decimal rounding, 5e-7 times one plus population growth; - the per-head series rounds its own ratios; and a series without its - ratios divides one plus its six-decimal values instead.""" + """For any growth before rounding and any population growth, + (1 + per-head growth) × (1 + population growth) matches one plus the + aggregate growth to within the six-decimal rounding, 5e-7 × + (1 + population growth). The per-head series rounds its own ratios, and + for a series without its ratios per_head_growth divides one plus its + six-decimal values instead.""" years = range(2025, 2025 + len(ratios)) by_year = dict(zip(years, ratios)) population_by_year = dict(zip(range(2025, 2033), population)) diff --git a/policyengine_uk/utils/import_obr_forecasts.py b/policyengine_uk/utils/import_obr_forecasts.py index cb17a614f0..f9dbfcacaf 100644 --- a/policyengine_uk/utils/import_obr_forecasts.py +++ b/policyengine_uk/utils/import_obr_forecasts.py @@ -710,8 +710,7 @@ def __post_init__(self): if self.unit_cell not in CLEARANCE_TABLE_TEXTS.get(self.title, ()): raise ValueError( f"{name}: a clearance line needs the cell that states its " - f"unit among its table's read texts, and {self.unit_cell!r} " - "is not one" + f"unit, and its table's read texts lack {self.unit_cell!r}" ) # The importer links clearance outturns to the previous EFO's outturn. if self.rates or self.outturn is not None: @@ -731,9 +730,10 @@ def __post_init__(self): TOBACCO_STICKS = "cigarette clearances (million sticks)" TOBACCO_KILOGRAMS = "non-cigarette clearances ('000s kg)" ALCOHOL_HECTOLITRES = "note: all clearances are in thousands of hectolitres of alcohol." -# The text of every cell below each clearance table's title in Tables 3.11 -# and 3.12 of the March 2025 and March 2026 EFOs and Tables 4.11 and 4.12 of -# October 2026, but numbers and fiscal years. Someone read each one. The +# The text of every cell from each clearance table's title down to the next +# table's title, in Tables 3.11 and 3.12 of the March 2025 and March 2026 +# EFOs and Tables 4.11 and 4.12 of October 2026, but the title, numbers and +# fiscal years. Someone read each one. The # importer compares a cell's text as receipts_label gives it: lower case, in # single spaces and without a footnote number after a label's last letter, # so " Wine1" gives "wine". It refuses any other text @@ -1314,8 +1314,8 @@ def find_receipts_line( The importer reads only that table's rows: its cash unit, its year headers and its "Outturn" and "Forecast" markers, which may sit on one - row or several. For a clearance line it first checks every cell of the - line's table against the texts someone has read + row or several. Before it reads a clearance line's values, it checks + every cell of the line's table against the texts someone has read (``validate_clearance_table``). """ matches = [] From 86d2497a9ca6b205c06aae7e29fa876f2e052558 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 1 Nov 2026 08:51:45 -0500 Subject: [PATCH 14/27] Write the excise projection test's formula as the importer does Co-Authored-By: Claude Opus 5.5 --- policyengine_uk/tests/test_excise_quantity_projection.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/policyengine_uk/tests/test_excise_quantity_projection.py b/policyengine_uk/tests/test_excise_quantity_projection.py index e30866ac45..92ab84a26d 100644 --- a/policyengine_uk/tests/test_excise_quantity_projection.py +++ b/policyengine_uk/tests/test_excise_quantity_projection.py @@ -3,10 +3,10 @@ Each input declares its product group's index under ``gov.economic_assumptions.indices.obr.per_capita``, and dataset projection -grows it by the same ``yoy_growth`` series: one plus the growth in the OBR's -clearances of the group (October 2026 receipts Tables 4.11 and 4.12, after the -March 2026 EFO's 2024-25 outturn), over one plus ``ons.population`` growth, -minus one. Dataset +grows it by the same ``yoy_growth`` series, +(1 + aggregate clearance growth) / (1 + ons.population growth) - 1, from the +OBR's clearances of the group (October 2026 receipts Tables 4.11 and 4.12, +after the March 2026 EFO's 2024-25 outturn). Dataset projection grows household weights by ``ons.population``, so the weighted quantities grow with the clearances, and a duty-rate reform moves receipts through the rate alone. From 1f4249a9711e4b29076c11689b46789a2fae6012 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 1 Nov 2026 08:52:53 -0500 Subject: [PATCH 15/27] Reflow the comment on the read clearance texts Co-Authored-By: Claude Opus 5.5 --- policyengine_uk/utils/import_obr_forecasts.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/policyengine_uk/utils/import_obr_forecasts.py b/policyengine_uk/utils/import_obr_forecasts.py index f9dbfcacaf..a629bf49ca 100644 --- a/policyengine_uk/utils/import_obr_forecasts.py +++ b/policyengine_uk/utils/import_obr_forecasts.py @@ -733,13 +733,13 @@ def __post_init__(self): # The text of every cell from each clearance table's title down to the next # table's title, in Tables 3.11 and 3.12 of the March 2025 and March 2026 # EFOs and Tables 4.11 and 4.12 of October 2026, but the title, numbers and -# fiscal years. Someone read each one. The -# importer compares a cell's text as receipts_label gives it: lower case, in -# single spaces and without a footnote number after a label's last letter, -# so " Wine1" gives "wine". It refuses any other text -# (validate_clearance_table), so it reads no unit or product from words no -# one has read. Add a text only after reading it, and when it states another -# unit, convert the table's figures to the units these texts state first. +# fiscal years. Someone read each one. The importer compares a cell's text as +# receipts_label gives it: lower case, in single spaces and without a +# footnote number after a label's last letter, so " Wine1" gives "wine". It +# refuses any other text (validate_clearance_table), so it reads no unit or +# product from words no one has read. Add a text only after reading it, and +# when it states another unit, convert the table's figures to the units these +# texts state first. CLEARANCE_TABLE_TEXTS = { TOBACCO_CLEARANCES: frozenset( { From 107eca4fae99969e85c37fc83064ee6d432eaa8a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 1 Nov 2026 08:54:59 -0500 Subject: [PATCH 16/27] Name the title row's skip in the clearance check Co-Authored-By: Claude Opus 5.5 --- policyengine_uk/utils/import_obr_forecasts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/policyengine_uk/utils/import_obr_forecasts.py b/policyengine_uk/utils/import_obr_forecasts.py index a629bf49ca..4c46ae35ee 100644 --- a/policyengine_uk/utils/import_obr_forecasts.py +++ b/policyengine_uk/utils/import_obr_forecasts.py @@ -1282,7 +1282,7 @@ def validate_clearance_table( ) if receipts_label(value) ] - # The first text on the title's row is the title. + # Skip the title, the first text on the title's row. for column, value in cells[1:] if index == 0 else cells: text = receipts_label(value) if column in years or NUMBER_RE.fullmatch(text): From a5511a523250bf3f833d1b514353f6b4d14b413f Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 1 Nov 2026 13:37:09 -0500 Subject: [PATCH 17/27] Read every run of a string; refuse a repeated clearance heading The XLSX reader returned only the first text run of an inline string, so the clearance check compared a prefix of a formatted cell with its list. Inline and shared strings now share one helper that joins every . validate_clearance_table checked the first block with the table's number and title while find_receipts_line could take a line from a later block with the same heading. It now refuses a sheet that repeats the heading and names the worksheet rows, which read_sheet_rows keeps as SheetRow.number. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_import_obr_forecasts.py | 241 +++++++++++++++++- policyengine_uk/utils/import_obr_forecasts.py | 74 ++++-- 2 files changed, 289 insertions(+), 26 deletions(-) diff --git a/policyengine_uk/tests/test_import_obr_forecasts.py b/policyengine_uk/tests/test_import_obr_forecasts.py index 42fa44e991..d3afe9ff10 100644 --- a/policyengine_uk/tests/test_import_obr_forecasts.py +++ b/policyengine_uk/tests/test_import_obr_forecasts.py @@ -1824,6 +1824,7 @@ def test_october_2026_tables_reproduce_the_committed_series(): COMMITTED_RELEASES = {"release": ("October", 2026), "previous_release": ("March", 2026)} XLSX_MAIN = "http://schemas.openxmlformats.org/spreadsheetml/2006/main" XLSX_NS = {"m": XLSX_MAIN} +XML_NAMESPACE = "http://www.w3.org/XML/1998/namespace" def mutate_real_receipts(workbook, table, unit, scale=1): @@ -3420,11 +3421,12 @@ def test_real_tables_with_a_kg_cell_beside_a_line_are_refused(tmp_path): def edit_real_table(workbook, table, cells=(), scale=(), notes=()): - """Set each cell in ``cells`` (as {"B6": text}) to its text, its number - or, for None, nothing, adding the cell and its row where the sheet lacks - them; multiply each cell in ``scale`` (as {"D5": 0.1}); and add each of - ``notes`` in column B below the sheet's last row. Every other archive - entry, including the shared strings, stays.""" + """Set each cell in ``cells`` to its text, number or, for None, nothing; + a tuple of texts gives one string with a formatting run for each. Add + the cell and its row where the sheet lacks them. Multiply the cells in + ``scale`` by their factors, then append ``notes`` in column B below the + sheet's last row. Preserve every other archive entry, including the + shared strings.""" cells, scale = dict(cells), dict(scale) output = BytesIO() with ZipFile(BytesIO(workbook)) as source, ZipFile(output, "w") as dest: @@ -3463,6 +3465,9 @@ def cell(reference): ET.SubElement( ET.SubElement(target, f"{{{XLSX_MAIN}}}is"), f"{{{XLSX_MAIN}}}t" ).text = value + elif isinstance(value, tuple): + target.set("t", "inlineStr") + target.append(formatting_runs("is", value)) elif value is not None: ET.SubElement(target, f"{{{XLSX_MAIN}}}v").text = repr(float(value)) for entry in source.infolist(): @@ -3473,6 +3478,23 @@ def cell(reference): return output.getvalue() +def formatting_runs(tag, texts): + """Build a ``tag`` string element with a formatting run for each of + ``texts``, the later runs bold, as a spreadsheet writes a cell whose + formatting changes.""" + string = ET.Element(f"{{{XLSX_MAIN}}}{tag}") + for index, part in enumerate(texts): + run = ET.SubElement(string, f"{{{XLSX_MAIN}}}r") + if index: + ET.SubElement( + ET.SubElement(run, f"{{{XLSX_MAIN}}}rPr"), f"{{{XLSX_MAIN}}}b" + ) + ET.SubElement( + run, f"{{{XLSX_MAIN}}}t", {f"{{{XML_NAMESPACE}}}space": "preserve"} + ).text = part + return string + + def real_levels(workbook, key): """Read the levels of ``key`` from its table alone in ``workbook``, and from the real October 2026 table.""" @@ -4002,6 +4024,215 @@ def test_any_text_no_one_has_read_stops_a_real_clearance_table(table, data): assert str(refused.value) == unread(sheet, title, (text, column)) +# A spreadsheet can split a cell's text into formatting runs. The check must +# see the whole text: each first run below passes alone, as a listed text, a +# number or a fiscal year. +TEN_MILLION_STICKS = "Note: Cigarette forecasts shown in ten million sticks." +FIRST_RUNS = { + "rich-listed-prefix": "of which:", + "rich-numeric-prefix": "1", + "rich-fiscal-year-prefix": "2026-27", +} + + +@pytest.mark.parametrize("name", sorted(FIRST_RUNS)) +def test_the_check_reads_every_formatting_run_of_a_cell(name): + """The helper divides the real October 2026 cigarette forecasts by ten + and writes B10 of Table 4.11 as one inline string in two runs: the + first ("of which:", "1" or "2026-27"), then " Note: Cigarette forecasts + shown in ten million sticks." The reader returns both runs, and the + importer refuses every tobacco line, naming the whole cell, and stops + before it renders anything.""" + runs = (FIRST_RUNS[name], " " + TEN_MILLION_STICKS) + changed = edit_real_table( + REAL_OCTOBER_2026_RECEIPTS, + "4.11", + cells={"B10": runs}, + scale={f"{column}5": 0.1 for column in "DEFGHI"}, + ) + with ZipFile(BytesIO(changed)) as archive: + sheet = ET.fromstring(archive.read(_sheet_paths(archive)["4.11"])) + cell = sheet.find("m:sheetData/m:row/m:c[@r='B10']", XLSX_NS) + assert [ + run.findtext("m:t", namespaces=XLSX_NS) + for run in cell.findall("m:is/m:r", XLSX_NS) + ] == list(runs) + rows = read_sheet_rows(changed, "4.11") + (note,) = [row["B"] for row in rows if row.number == 10] + assert note == "".join(runs) + expected = unread("4.11", TOBACCO_CLEARANCES, (note, "B")) + for key in TOBACCO_LINES: + with pytest.raises(ValueError) as refused: + find_receipts_line({"4.11": rows}, CLEARANCE_SPECS[key]) + assert str(refused.value) == expected, key + with pytest.raises(ValueError) as refused: + render( + get_yoy_growth_path().read_text(), + current=changed, + previous=REAL_RECEIPTS[2026], + **COMMITTED_RELEASES, + ) + assert str(refused.value) == expected + + +def runs_workbook(texts): + """Build a one-sheet workbook that holds ``texts`` in A1 as an inline + string and in A2 as a shared string, each with a formatting run for each + text, and their join in A3 as an inline string of one ````.""" + sheet = make_sheet( + { + 1: [ + '' + + ET.tostring(formatting_runs("is", texts), encoding="unicode") + + "" + ], + 2: ['0'], + 3: [make_inline_cell("A3", "")], + } + ) + sheet = ET.fromstring(sheet) + joined = sheet.find("m:sheetData/m:row/m:c[@r='A3']/m:is/m:t", XLSX_NS) + joined.text = "".join(texts) + strings = ET.Element(f"{{{XLSX_MAIN}}}sst") + strings.append(formatting_runs("si", texts)) + output = BytesIO() + with ZipFile(BytesIO(make_workbook({"1": ET.tostring(sheet)}))) as source: + with ZipFile(output, "w") as dest: + for entry in source.infolist(): + dest.writestr(entry, source.read(entry)) + dest.writestr("xl/sharedStrings.xml", ET.tostring(strings)) + return output.getvalue() + + +@settings(max_examples=100, deadline=None, derandomize=True) +@given( + texts=st.lists( + st.text(st.characters(min_codepoint=0x20, max_codepoint=0xD7FF)), + min_size=1, + max_size=4, + ) +) +def test_a_string_reads_whole_in_any_formatting_runs(texts): + """However a spreadsheet splits a text into formatting runs, the reader + returns the whole text, from an inline string and from a shared string + alike, as it does from one ````.""" + rows = read_sheet_rows(runs_workbook(texts), "1") + assert [(row.number, row["A"]) for row in rows] == [ + (number, "".join(texts)) for number in (1, 2, 3) + ] + + +# A sheet that repeats a clearance table's heading must stop the run: the +# importer would check the text under one and could read a line under the +# other. +TEN_THOUSAND_HECTOLITRES = ( + "Note: Spirits forecasts shown in ten thousand hectolitres of alcohol." +) + + +def repeat_alcohol_heading(workbook, sheet): + """Clear the lines of the real alcohol clearance table on ``sheet``, rows + 5 to 7, and repeat its rows 2 to 10 below the sheet's last row, so a + second copy of its heading follows the alcohol receipts forecasts. + Divide the repeated spirits forecasts by ten and put a note in ten + thousand hectolitres below the repeated rows. Return the workbook and + the second heading's row.""" + rows = read_sheet_rows(workbook, sheet) + shift = rows[-1].number + 1 + cells = {f"{column}{number}": None for column in "BCDEFGHI" for number in (5, 6, 7)} + for row in rows: + if not 2 <= row.number <= 10: + continue + for column, value in row.items(): + if not str(value or "").strip(): + continue + try: + value = float(value) + except ValueError: + pass + else: + if row.number == 7 and column > "C": + value /= 10 + cells[f"{column}{row.number + shift}"] = value + cells[f"B{11 + shift}"] = TEN_THOUSAND_HECTOLITRES + return edit_real_table(workbook, sheet, cells=cells), 2 + shift + + +def repeated(table, row): + """Write out the importer's refusal of real Table ``table`` with its + heading in rows 2 and ``row``.""" + return ( + f"Table {table} (sheet {table}) repeats its heading, in rows 2 and " + f"{row}. The importer would check the text under one and could read a " + "line under another, so keep one heading." + ) + + +def test_a_repeated_clearance_heading_stops_the_run(tmp_path): + """The helper clears the lines of the real October 2026 Table 4.12 and + repeats the table below the alcohol receipts forecasts, under a second + "4.12 Alcohol clearance forecasts" heading in row 53. It divides the + repeated spirits forecasts by ten and states ten thousand hectolitres in + a note below them. Each line then sits once, under the second heading, + and the first keeps the table's listed note. Read in thousand + hectolitres, the repeated figures give 2026-27 spirits growth of -90.1%, + where the published figures give -0.8%. The importer refuses every + alcohol line and names both headings' rows, the renderer gets no growth, + and a run on the real March 2026 tables changed the same way stops + before it writes anything. The importer reads the tobacco lines as + before.""" + changed, row = repeat_alcohol_heading(REAL_OCTOBER_2026_RECEIPTS, "4.12") + assert row == 53 + rows = read_sheet_rows(changed, "4.12") + (spirits,) = [line for line in rows if line.number == 58] + assert receipts_label(spirits["B"]) == "spirits" + published = [float(level) for level in OCTOBER_2026_CLEARANCES["spirits"][:2]] + assert round(float(spirits["D"]) / float(spirits["C"]) - 1, 6) == -0.900793 + assert round(published[1] / published[0] - 1, 6) == -0.007927 + sheets = {"4.11": REAL_OCTOBER_2026_SHEETS["4.11"], "4.12": rows} + for key, spec in CLEARANCE_SPECS.items(): + if spec.title == TOBACCO_CLEARANCES: + assert ( + find_receipts_line(sheets, spec).levels + == find_receipts_line(REAL_OCTOBER_2026_SHEETS, spec).levels + ), key + continue + with pytest.raises(ValueError) as refused: + find_receipts_line(sheets, spec) + assert str(refused.value) == repeated("4.12", 53), key + # Rows built as plain dicts carry no number, so the refusal counts their + # places: row 53 is the 47th row the sheet stores. + with pytest.raises(ValueError) as refused: + find_receipts_line( + {"4.12": [dict(line) for line in rows]}, CLEARANCE_SPECS["spirits"] + ) + assert str(refused.value) == repeated("4.12", 47) + with pytest.raises(ValueError) as refused: + render( + get_yoy_growth_path().read_text(), + current=changed, + previous=REAL_RECEIPTS[2026], + **COMMITTED_RELEASES, + ) + assert str(refused.value) == repeated("4.12", 53) + march, row = repeat_alcohol_heading(REAL_RECEIPTS[2026], "3.12") + yoy, gap_dir = write_tree(tmp_path) + before = {path: path.read_bytes() for path in [yoy, *gap_dir.iterdir()]} + with pytest.raises(ValueError) as refused: + run_main( + tmp_path, + make_test_xlsx(), + "--yaml-path", + str(yoy), + receipts=march, + previous_receipts=REAL_RECEIPTS[2025], + ) + assert str(refused.value) == repeated("3.12", row) + assert row == 38 + assert {path: path.read_bytes() for path in before} == before + assert not list(yoy.parent.rglob("*.staged")) + + # per_head_growth divides each series' own growth, a duty's base for a line # with rates, whether the importer links the line to the previous EFO or to # its own outturn. diff --git a/policyengine_uk/utils/import_obr_forecasts.py b/policyengine_uk/utils/import_obr_forecasts.py index 4c46ae35ee..363b2c6b10 100644 --- a/policyengine_uk/utils/import_obr_forecasts.py +++ b/policyengine_uk/utils/import_obr_forecasts.py @@ -227,16 +227,19 @@ def extract_economy_workbook_bytes( return Path(workbook_name).name, archive.read(workbook_name) +def _string_text(item: ET.Element) -> str: + """Join the text of every ```` in a shared or inline string, so the + reader returns the whole text of a string split into formatting runs + (````).""" + return "".join(node.text or "" for node in item.iterfind(".//main:t", WORKBOOK_NS)) + + def _load_shared_strings(archive: ZipFile) -> list[str]: if "xl/sharedStrings.xml" not in archive.namelist(): return [] root = ET.fromstring(archive.read("xl/sharedStrings.xml")) - values: list[str] = [] - for item in root.findall("main:si", WORKBOOK_NS): - parts = [node.text or "" for node in item.iterfind(".//main:t", WORKBOOK_NS)] - values.append("".join(parts)) - return values + return [_string_text(item) for item in root.findall("main:si", WORKBOOK_NS)] def _sheet_paths(archive: ZipFile) -> dict[str, str]: @@ -264,22 +267,36 @@ def _cell_value(cell: ET.Element, shared_strings: list[str]) -> str | None: if cell_type == "s" and value_node is not None: return shared_strings[int(value_node.text)] if cell_type == "inlineStr" and inline_node is not None: - text_node = inline_node.find(".//main:t", WORKBOOK_NS) - return text_node.text if text_node is not None else "" + return _string_text(inline_node) if value_node is not None: return value_node.text return None -def read_sheet_rows(xlsx_bytes: bytes, sheet_name: str) -> list[dict[str, str | None]]: +class SheetRow(dict): + """A worksheet row as ``read_sheet_rows`` reads it: each cell's text by + column letter, and ``number``, the row's number on the worksheet.""" + + def __init__(self, number: int): + super().__init__() + self.number = number + + +def read_sheet_rows(xlsx_bytes: bytes, sheet_name: str) -> list[SheetRow]: + """Read the rows a worksheet stores, in order. A worksheet can skip + rows, as October 2026's sheet 4.12 skips row 33, so a row's place in the + list can differ from its number.""" with ZipFile(BytesIO(xlsx_bytes)) as archive: shared_strings = _load_shared_strings(archive) sheet_path = _sheet_paths(archive)[sheet_name] sheet = ET.fromstring(archive.read(sheet_path)) rows = [] + number = 0 for row in sheet.findall(".//main:sheetData/main:row", WORKBOOK_NS): - values: dict[str, str | None] = {} + # A row without a number takes the one after the row before it. + number = int(row.attrib.get("r", number + 1)) + values = SheetRow(number) for cell in row.findall("main:c", WORKBOOK_NS): ref = cell.attrib["r"] column = "".join(ch for ch in ref if ch.isalpha()) @@ -676,10 +693,10 @@ class ReceiptsSeriesSpec: @property def per_head(self) -> bool: - """Whether the series sits under ``per_capita``. Dataset projection - grows household weights by ``ons.population``, so - ``render_receipts_growth_yaml`` writes such a series as - (1 + aggregate growth) / (1 + ons.population growth) - 1 + """Return whether the renderer writes this series under + ``per_capita``. Dataset projection grows household weights by + ``ons.population``, so ``render_receipts_growth_yaml`` writes such a + series as (1 + aggregate growth) / (1 + ons.population growth) - 1 (``per_head_growth``, ``per_head_tail``), and the weighted total grows with the line in every year.""" return self.path[0] == "per_capita" @@ -687,8 +704,8 @@ def per_head(self) -> bool: @dataclass(frozen=True) class ClearanceSeriesSpec(ReceiptsSeriesSpec): - """A line of the OBR's tobacco or alcohol clearance tables, which grows - household quantity inputs per head (#2596). + """Describe a clearance line the importer uses to grow household + quantity inputs per head (#2596). The OBR counts clearances in quantities, not pounds, so a clearance table has no "£ billion" cell. ``unit_cell`` holds the text of the cell @@ -1260,12 +1277,26 @@ def validate_clearance_table( line's ``unit_cell`` among them. It interprets no words, so a note that states another unit, a line for another product or a unit column stops the run until someone reads the text and lists it. + + The importer also refuses a sheet that repeats the table's number and + title, and names the rows (``SheetRow.number``, or a row's place in + ``rows`` counted from 1 for a row built as a plain dict). It would check + the text under one heading and could read the line under another. """ - start = next( + starts = [ index for index, row in enumerate(rows) if row_title(row) == (number, spec.title) - ) + ] + if len(starts) > 1: + numbers = [str(getattr(rows[index], "number", index + 1)) for index in starts] + raise ValueError( + f"{where} repeats its heading, in rows " + f"{', '.join(numbers[:-1])} and {numbers[-1]}. The importer would " + "check the text under one and could read a line under another, so " + "keep one heading." + ) + start = starts[0] block = [rows[start]] for row in rows[start + 1 :]: title = row_title(row) @@ -1310,13 +1341,14 @@ def validate_clearance_table( def find_receipts_line( sheets: dict[str, list[dict[str, str | None]]], spec: ReceiptsSeriesSpec ) -> ReceiptsLine: - """The one row labelled ``spec.row`` in a table titled ``spec.title``. + """Find the row labelled ``spec.row`` in the table titled + ``spec.title``. The importer reads only that table's rows: its cash unit, its year headers and its "Outturn" and "Forecast" markers, which may sit on one - row or several. Before it reads a clearance line's values, it checks - every cell of the line's table against the texts someone has read - (``validate_clearance_table``). + row or several. Before it reads a clearance line's values, it checks the + line's table, from its title through its end, against the texts someone + has read (``validate_clearance_table``). """ matches = [] for sheet, rows in sheets.items(): From d168994ad21535776ee87e75d6aebe8efba5238b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 1 Nov 2026 14:00:01 -0500 Subject: [PATCH 18/27] State the clearance check's span and the repeated-heading stop The README and the guide now say the importer compares text cells from each clearance table's title through its configured end, and checks no cell above the title or below the end. They add the stop on a repeated heading and the reading of every formatting run, and split the guide's sentence on duty per unit cleared. Co-Authored-By: Claude Opus 5.5 --- docs/engineering/excise-duties.md | 32 +++++++++++-------- .../gov/economic_assumptions/README.md | 17 ++++++---- 2 files changed, 30 insertions(+), 19 deletions(-) diff --git a/docs/engineering/excise-duties.md b/docs/engineering/excise-duties.md index e6531baaca..4cacde27d2 100644 --- a/docs/engineering/excise-duties.md +++ b/docs/engineering/excise-duties.md @@ -283,20 +283,26 @@ labels, the "Outturn" and "Forecast" markers, the "of which:" row, Table its footnotes. On Table 4.12's sheet the OBR follows the alcohol clearances with the alcohol receipts forecasts. The importer ends the clearance table at their title (`CLEARANCE_TABLE_ENDS`), so its check leaves out the -receipts table's "£ billion" cell and note. The importer checks every cell -of each clearance table it reads, in every column, skips numbers and fiscal -years, and stops before writing anything when: +receipts table's "£ billion" cell and note. From each clearance table's +title through that configured end, or the sheet's last row, the importer +compares text cells in every column with the list. It skips the title cell, +numbers and fiscal years, and stops before writing anything when: - a cell holds any other text, such as a note that states another unit or scale, a line for another product or a unit column's heading; - the table lacks the cell that states a line's unit: the cigarette label's "(million sticks)", the non-cigarette total's "('000s kg)" for hand-rolled and other tobacco, or Table 4.12's note for alcohol; -- the OBR puts any cell beside a line's label and values. +- the OBR puts any cell beside a line's label and values; +- the sheet repeats the table's heading, since the importer would check the + text under one heading and could read a line under another. -The importer names the table and each unread cell. Whoever runs it reads -the text, converts the table's figures to the listed units if it states -another, and adds the text to the list. +The importer reads every formatting run of a cell's text. It names the +table and each unread cell, or the rows that repeat the heading. Whoever +runs it reads the text, converts the table's figures to the listed units if +it states another, and adds the text to the list. The importer checks no +cell above a table's title or below its end, so that person reads any note +there too. ### Budget measures and reforms @@ -341,12 +347,12 @@ UK's 2024-25 clearances shows the effect. It treats beer and cider as beer at 4% with no draught relief, wine and other fermented products as wine at 12%, spirits at 40%, other tobacco products as other smoking tobacco, and cigarettes at the default price. On that mix the model's duty per unit -cleared rises 5.7% for alcohol by 2026-27 and 19.0% for tobacco by 2027-28, -with the rate rises of February 2025 and February 2026 for alcohol and of -October 2024, November 2025 and October 2026 for tobacco, each of which -reaches its first full model year after 2024-25, and then holds. The OBR's -receipts per unit cleared keep rising, by 22.1% for alcohol and 48.2% for -tobacco by 2031-32. +cleared rises 5.7% for alcohol by 2026-27 and 19.0% for tobacco by 2027-28. +The alcohol increases take effect in February 2025 and February 2026, and +the tobacco increases in October 2024, November 2025 and October 2026. Each +reaches its first full model year after 2024-25. After that, product mix +alone moves model duty per unit. The OBR's receipts per unit cleared keep +rising, by 22.1% for alcohol and 48.2% for tobacco by 2031-32. On this construct, the model's alcohol duty exceeds the OBR's receipts by 7.6% in 2024-25 and 7.9% in 2026-27, then by less each year: by 3.5% in diff --git a/policyengine_uk/parameters/gov/economic_assumptions/README.md b/policyengine_uk/parameters/gov/economic_assumptions/README.md index 3f015b74f3..dbd0e40765 100644 --- a/policyengine_uk/parameters/gov/economic_assumptions/README.md +++ b/policyengine_uk/parameters/gov/economic_assumptions/README.md @@ -511,18 +511,23 @@ the forecast levels, though not their growth. The same run also refreshes the six alcohol and tobacco clearance series under `per_capita` from the new EFO's tobacco clearances and alcohol clearance forecasts, and the previous EFO's for the outturn year's base. - The importer interprets no words in these tables. It checks every cell - of both EFOs' tables, but numbers and fiscal years, against the texts - someone has read (`CLEARANCE_TABLE_TEXTS`). It also needs the cell that + The importer interprets no words in these tables. From each table's + title through its configured end, the row above a title + `CLEARANCE_TABLE_ENDS` lists or the sheet's last row, the importer + compares text cells with `CLEARANCE_TABLE_TEXTS`, in both EFOs. It skips + the title cell, numbers and fiscal years. It also needs the cell that states each line's unit: million sticks in the cigarette label, thousand kilograms in the non-cigarette total's label and thousand hectolitres of alcohol in Table 4.12's note. It stops before writing on any other text and names the table and the cell. Read that text, convert the table's figures to these units if it states another, and add the text to the list. The importer also stops on any cell beside a line's - label and values. The list holds each EFO's footnotes as the OBR wrote - them, and the importer does not compare the products one EFO's line - covers with another's. Check that each alcohol line covers the same + label and values. When a sheet repeats a clearance table's heading, it + stops and names the rows. It checks no cell above a table's title or + below its end, so read any note there yourself. The list holds each + EFO's footnotes as the OBR wrote them, and the importer does not compare + the products one EFO's line covers with another's. Check that each + alcohol line covers the same products in both EFOs: in March 2026 the OBR added "Includes other fermented products" to the wine line. The importer writes `(1 + aggregate clearance growth) / (1 + ons.population growth) - 1`, From da296a9d0a6e7c584b43f6eff56a27f6ee788c0d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 1 Nov 2026 15:07:18 -0500 Subject: [PATCH 19/27] Give the round's docstrings active verbs; reflow the README step Co-Authored-By: Claude Opus 5.5 --- .../parameters/gov/economic_assumptions/README.md | 6 +++--- policyengine_uk/tests/test_import_obr_forecasts.py | 11 +++++------ policyengine_uk/utils/import_obr_forecasts.py | 5 +++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/policyengine_uk/parameters/gov/economic_assumptions/README.md b/policyengine_uk/parameters/gov/economic_assumptions/README.md index dbd0e40765..12618daa8e 100644 --- a/policyengine_uk/parameters/gov/economic_assumptions/README.md +++ b/policyengine_uk/parameters/gov/economic_assumptions/README.md @@ -527,9 +527,9 @@ the forecast levels, though not their growth. below its end, so read any note there yourself. The list holds each EFO's footnotes as the OBR wrote them, and the importer does not compare the products one EFO's line covers with another's. Check that each - alcohol line covers the same - products in both EFOs: in March 2026 the OBR added "Includes other - fermented products" to the wine line. The importer writes + alcohol line covers the same products in both EFOs: in March 2026 the + OBR added "Includes other fermented products" to the wine line. The + importer writes `(1 + aggregate clearance growth) / (1 + ons.population growth) - 1`, using the population in `yoy_growth.yaml`, so rerun it whenever that series changes. Until then the six diff --git a/policyengine_uk/tests/test_import_obr_forecasts.py b/policyengine_uk/tests/test_import_obr_forecasts.py index d3afe9ff10..a837e425bc 100644 --- a/policyengine_uk/tests/test_import_obr_forecasts.py +++ b/policyengine_uk/tests/test_import_obr_forecasts.py @@ -3479,9 +3479,8 @@ def cell(reference): def formatting_runs(tag, texts): - """Build a ``tag`` string element with a formatting run for each of - ``texts``, the later runs bold, as a spreadsheet writes a cell whose - formatting changes.""" + """Build a ``tag`` string element, ``is`` or ``si``, with a formatting + run for each of ``texts``, the later runs bold.""" string = ET.Element(f"{{{XLSX_MAIN}}}{tag}") for index, part in enumerate(texts): run = ET.SubElement(string, f"{{{XLSX_MAIN}}}r") @@ -4177,8 +4176,8 @@ def test_a_repeated_clearance_heading_stops_the_run(tmp_path): and the first keeps the table's listed note. Read in thousand hectolitres, the repeated figures give 2026-27 spirits growth of -90.1%, where the published figures give -0.8%. The importer refuses every - alcohol line and names both headings' rows, the renderer gets no growth, - and a run on the real March 2026 tables changed the same way stops + alcohol line and names both headings' rows, and the renderer gets no + growth. A run on the real March 2026 tables, changed the same way, stops before it writes anything. The importer reads the tobacco lines as before.""" changed, row = repeat_alcohol_heading(REAL_OCTOBER_2026_RECEIPTS, "4.12") @@ -4201,7 +4200,7 @@ def test_a_repeated_clearance_heading_stops_the_run(tmp_path): find_receipts_line(sheets, spec) assert str(refused.value) == repeated("4.12", 53), key # Rows built as plain dicts carry no number, so the refusal counts their - # places: row 53 is the 47th row the sheet stores. + # places: row 53 sits 47th among the rows the sheet stores. with pytest.raises(ValueError) as refused: find_receipts_line( {"4.12": [dict(line) for line in rows]}, CLEARANCE_SPECS["spirits"] diff --git a/policyengine_uk/utils/import_obr_forecasts.py b/policyengine_uk/utils/import_obr_forecasts.py index 363b2c6b10..71d96a3043 100644 --- a/policyengine_uk/utils/import_obr_forecasts.py +++ b/policyengine_uk/utils/import_obr_forecasts.py @@ -274,8 +274,9 @@ def _cell_value(cell: ET.Element, shared_strings: list[str]) -> str | None: class SheetRow(dict): - """A worksheet row as ``read_sheet_rows`` reads it: each cell's text by - column letter, and ``number``, the row's number on the worksheet.""" + """Hold a worksheet row as ``read_sheet_rows`` reads it: each cell's + text by column letter, and ``number``, the row's number on the + worksheet.""" def __init__(self, number: int): super().__init__() From 25fc73dcf0d0d1dd2337562be29d42f9a674a289 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 1 Nov 2026 15:12:31 -0500 Subject: [PATCH 20/27] Give the branch's docstrings real verbs Docstrings only: their ASTs match the previous commit's once each docstring drops out. Co-Authored-By: Claude Opus 5.5 --- .../tests/test_excise_quantity_projection.py | 19 ++++++----- .../tests/test_import_obr_forecasts.py | 34 +++++++++++-------- policyengine_uk/utils/import_obr_forecasts.py | 31 +++++++++-------- 3 files changed, 46 insertions(+), 38 deletions(-) diff --git a/policyengine_uk/tests/test_excise_quantity_projection.py b/policyengine_uk/tests/test_excise_quantity_projection.py index 92ab84a26d..8102e67c6c 100644 --- a/policyengine_uk/tests/test_excise_quantity_projection.py +++ b/policyengine_uk/tests/test_excise_quantity_projection.py @@ -91,8 +91,9 @@ def index(parameters, group, year): def household_frame(year, households): - """One adult in each household, with the given household inputs. Each - household is a dict of inputs; ``household_weight`` defaults to 1.""" + """Build a dataset with one adult in each household, from the given + household inputs. Pass each household as a dict of inputs; + ``household_weight`` defaults to 1.""" count = len(households) columns = sorted({name for household in households for name in household}) return UKSingleYearDataset( @@ -215,8 +216,8 @@ def test_indices_are_positive_from_the_2024_base_year(parameters): def test_each_series_is_its_clearances_per_head(parameters): - """Each year's growth, to six decimals, is the clearances' growth over - one plus that year's population growth, the growth of household + """Each year's growth equals, to six decimals, the clearances' growth + over one plus that year's population growth, the growth of household weights.""" for group, levels in CLEARANCES.items(): assert growth(parameters, f"{GROWTH}.{group}", 2024) == 0 @@ -295,7 +296,7 @@ def test_weighted_quantities_follow_the_clearances(parameters): def test_projected_quantities_are_the_base_times_the_index(quantities, weight): """For any quantities and weight: dataset projection gives each year's quantity as the 2024 quantity times its series' growth compounded, which - is its index to the index's five decimals; Core's uprating of a 2024 + matches its index to the index's five decimals; Core's uprating of a 2024 situation input (the other route to a later year) agrees; the weight grows by population alone.""" parameters = current_law() @@ -349,8 +350,8 @@ def test_a_later_year_charges_duty_on_the_uprated_quantity(parameters): def rate_leaves(parameters): - """Every alcohol and tobacco duty rate the duty formulas read, without - the wine strength easement's true-or-false switch.""" + """Return every alcohol and tobacco duty rate the duty formulas read, + without the wine strength easement's true-or-false switch.""" leaves = [ leaf for node in (ALCOHOL_RATES, TOBACCO_RATES) @@ -414,8 +415,8 @@ def test_a_duty_rate_reform_moves_receipts_by_the_rate_alone(factor, year, scale def test_the_change_touches_only_the_quantity_inputs(parameters): - """Differential against the projection without this change, which - leaves the quantity inputs out of the uprating map and out of Core's + """Compare with the projection without this change, which leaves the + quantity inputs out of the uprating map and out of Core's fallback: every other stored input keeps its value in every year, the quantity inputs differ by their index alone, and outputs differ only through the two duties.""" diff --git a/policyengine_uk/tests/test_import_obr_forecasts.py b/policyengine_uk/tests/test_import_obr_forecasts.py index a837e425bc..d87e32d60c 100644 --- a/policyengine_uk/tests/test_import_obr_forecasts.py +++ b/policyengine_uk/tests/test_import_obr_forecasts.py @@ -544,9 +544,10 @@ def clearance_table( start: int = 2, unit: str | None = None, ) -> dict[int, list[str]]: - """A table laid out as the OBR's Tables 4.11 and 4.12: title in column B, - the "Outturn" and "Forecast" markers on the next row, fiscal years on the - row after, then one row per line and any notes. A clearance table has no + """Build a table in the layout of the OBR's Tables 4.11 and 4.12: title + in column B, the "Outturn" and "Forecast" markers on the next row, fiscal + years on the row after, then one row per line and any notes. A clearance + table has no unit cell, since the OBR states the unit in a label or a note; ``unit`` adds one on a row of its own below the title, as a receipts table has.""" years = max(len(levels) for _, levels in rows) @@ -583,8 +584,8 @@ def clearance_table( def tobacco_table(chapter, first_year, levels, outturn_years=1): - """Table 4.11 (3.11 to March 2026): cigarettes in million sticks, then - non-cigarette clearances in thousand kg and their "of which" split.""" + """Build Table 4.11 (3.11 to March 2026): cigarettes in million sticks, + then non-cigarette clearances in thousand kg and their "of which" split.""" non_cigarette = tuple( float(hand_rolled) + float(other) for hand_rolled, other in zip( @@ -611,8 +612,9 @@ def tobacco_table(chapter, first_year, levels, outturn_years=1): def alcohol_table(chapter, first_year, levels, outturn_years=1, note=None): - """Table 4.12 (3.12 to March 2026): clearances in thousand hectolitres of - alcohol, which the OBR states in a note, then the duties' receipts in + """Build Table 4.12 (3.12 to March 2026): clearances in thousand + hectolitres of alcohol, which the OBR states in a note, then the duties' + receipts in £ billion under the same labels, which the importer must not read for the clearance lines.""" labels = { @@ -1783,7 +1785,7 @@ def test_rendered_growth_compounds_the_published_levels(years, data): def test_october_2026_tables_reproduce_the_committed_series(): - """yoy_growth.yaml holds exactly what the importer writes from the + """yoy_growth.yaml holds, byte for byte, what the importer writes from the October 2026 receipts tables (Tables 4.7, 4.8, 4.2, 4.10, 4.11 and 4.12) and the March 2026 ones (3.7, 3.8, 3.2, 3.11 and 3.12): rerunning it on the committed file changes nothing.""" @@ -3092,9 +3094,9 @@ def test_a_clearance_spec_needs_a_read_unit_cell_and_takes_no_rates_or_outturn() def test_real_october_2026_clearance_series(): - """The committed series, from the real October 2026 Tables 4.11 and 4.12 - over the real March 2026 Tables 3.11 and 3.12, per head of the committed - ons.population, by hand.""" + """Check the committed series by hand: the real October 2026 Tables + 4.11 and 4.12 over the real March 2026 Tables 3.11 and 3.12, per head of + the committed ons.population.""" population = population_growth(get_yoy_growth_path().read_text()) assert (population[2025], population[2026]) == (0.0033, 0.002) growth = compute_receipts_growth( @@ -3182,8 +3184,9 @@ def test_a_population_change_moves_the_per_head_series_once_the_importer_reruns( @given(years=st.integers(2, 7), data=st.data()) def test_rendered_clearances_grow_the_weighted_total_with_the_clearances(years, data): """For any clearances and population growth: in every year the household - quantity's growth times the household weight's growth is the clearances' - growth, to the series' six decimals, and after the forecast it is 1; the + quantity's growth times the household weight's growth equals the + clearances' growth, to the series' six decimals, and equals 1 after the + forecast; the series stays above -100%, so its index stays positive; a second run changes nothing.""" base, levels = {}, {} @@ -3364,7 +3367,7 @@ def test_real_tables_with_a_kg_cell_beside_a_line_are_refused(tmp_path): """The helper writes other tobacco in kg beside its line in the real October 2026 Table 4.11 and multiplies its values by 1,000. Reading them in the total's thousand kg gives growth of 862.8112. The renderer refuses - it, and a run on the real March 2026 tables changed the same way stops + it, and a run on the real March 2026 tables, changed the same way, stops before it writes anything.""" changed = put_unit_beside_real_line( REAL_OCTOBER_2026_RECEIPTS, "4.11", "Other tobacco products", "kg" @@ -3959,7 +3962,8 @@ def unread_text(text, title): @given(table=st.sampled_from(sorted(REAL_CLEARANCE_TABLES)), data=st.data()) def test_any_text_no_one_has_read_stops_a_real_clearance_table(table, data): """Put one text in a real table, in an empty cell of any row from its - title down, in any column, or on a new row. Any text outside the table's + title through the table's end, in any column, or on a new row. Any text + outside the table's list, from st.text() or shaped like a unit, stops every line of the table: the importer names the cell, or finds no line or table where the text took a label's or the title's place. A text from the list, other diff --git a/policyengine_uk/utils/import_obr_forecasts.py b/policyengine_uk/utils/import_obr_forecasts.py index 71d96a3043..7acb632726 100644 --- a/policyengine_uk/utils/import_obr_forecasts.py +++ b/policyengine_uk/utils/import_obr_forecasts.py @@ -1105,9 +1105,9 @@ def growth_ratio( @dataclass(frozen=True) class ReceiptsLine: - """A receipts line as the OBR publishes it in one EFO: £bn, or a + """Hold a receipts line as the OBR publishes it in one EFO: £bn, or a clearance line's own unit, by the calendar year in which each fiscal year - starts, and which of those years are outturn.""" + starts, and which of those years the OBR marks as outturn.""" table: str label: str @@ -1119,9 +1119,10 @@ class ReceiptsLine: class ReceiptsGrowth: """Growth in a receipts line, by year, rounded to six decimals. - The first year is outturn growth: this EFO's first (outturn) year over - the year before in the previous EFO, where it is outturn. Each later - year is the ratio of this EFO's consecutive levels. A line with its own + The first year holds outturn growth: this EFO's first (outturn) year + over the year before in the previous EFO, which marks that year as + outturn. Each later year holds the ratio of this EFO's consecutive + levels. A line with its own outturn starts at that outturn's second year and has no previous line; one with rates grows its base instead (see ``ReceiptsSeriesSpec``). A clearance line holds the clearances' growth, which @@ -1565,8 +1566,9 @@ def growth_in_effect(series: dict[int, float], year: int) -> float: def per_head_growth( growth: ReceiptsGrowth, population: dict[int, float] ) -> ReceiptsGrowth: - """A series' growth per head, to six decimals: one plus each year's - growth, over one plus that year's ``ons.population`` growth, minus one. + """Return a series' growth per head, to six decimals: one plus each + year's growth, over one plus that year's ``ons.population`` growth, + minus one. It divides the series' own growth, before rounding where ``growth.ratios`` holds it: a duty's base for a line with rates, whether @@ -1733,9 +1735,9 @@ def render_receipts_values( def receipts_basis(spec: ReceiptsSeriesSpec | None) -> str: - """How a line's growth is measured, for its comments: nothing for - receipts, the rates taken out for a base, and population for clearances - and for a base per head.""" + """Return what the importer takes out of a line's growth, for its + comments: nothing for receipts, the rates for a base, and population for + clearances and for a base per head.""" if isinstance(spec, ClearanceSeriesSpec): return "; per head: clearance growth over one plus ons.population growth" if spec is None or not spec.rates: @@ -1822,10 +1824,11 @@ def receipts_tail(content: str, growth: ReceiptsGrowth) -> tuple[dict[int, float def clearance_tail(content: str, last: int) -> tuple[dict[int, float], str]: - """A clearance series after the OBR's forecast, and its comment. The - clearances hold flat, so per head they grow by 1 / (1 + population) - 1, - to six decimals, in each year to ``ons.population``'s last, written where - the value changes.""" + """Return a clearance series after the OBR's forecast, and its + comment. The importer holds the clearances flat, so per head they grow + by 1 / (1 + population) - 1, to six decimals, in each year to + ``ons.population``'s last. It writes a year only where the value + changes.""" population = population_growth(content) yearly = { year: round(1 / (1 + growth_in_effect(population, year)) - 1, 6) + 0.0 From 510410e5ad5ad2261dd531ce30dfb04e734e268e Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Sun, 1 Nov 2026 15:14:57 -0500 Subject: [PATCH 21/27] Lead the README's clearance step with the comparison and its span Co-Authored-By: Claude Opus 5.5 --- .../gov/economic_assumptions/README.md | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/policyengine_uk/parameters/gov/economic_assumptions/README.md b/policyengine_uk/parameters/gov/economic_assumptions/README.md index 12618daa8e..e598bc7847 100644 --- a/policyengine_uk/parameters/gov/economic_assumptions/README.md +++ b/policyengine_uk/parameters/gov/economic_assumptions/README.md @@ -487,10 +487,10 @@ the forecast levels, though not their growth. renumbering does not matter, and reads only that table's rows. It takes each workbook's release from its contents sheet, or else its file name or URL, and refuses receipts tables from another EFO or of unknown release. - Every selected receipts table in both releases must declare `£ billion` - in its header (the importer also accepts `£bn` and `£ bn`, ignoring case - and whitespace); the OBR states each clearance line's unit in a label or - a note (below). + The importer requires `£ billion` in the header of every selected + receipts table in both releases, and also accepts `£bn` and `£ bn`, + ignoring case and whitespace. The OBR states each clearance line's unit + in a label or a note (below). Other units, including `£m`, and missing or ambiguous declarations stop the import before any file is written, so different cash magnitudes cannot be combined across releases. @@ -511,11 +511,11 @@ the forecast levels, though not their growth. The same run also refreshes the six alcohol and tobacco clearance series under `per_capita` from the new EFO's tobacco clearances and alcohol clearance forecasts, and the previous EFO's for the outturn year's base. - The importer interprets no words in these tables. From each table's - title through its configured end, the row above a title - `CLEARANCE_TABLE_ENDS` lists or the sheet's last row, the importer - compares text cells with `CLEARANCE_TABLE_TEXTS`, in both EFOs. It skips - the title cell, numbers and fiscal years. It also needs the cell that + From each table's title through its configured end, the importer + compares text cells with `CLEARANCE_TABLE_TEXTS`, in both EFOs. The end + falls on the row above a title `CLEARANCE_TABLE_ENDS` lists, or on the + sheet's last row. It skips the title cell, numbers and fiscal years. It + also needs the cell that states each line's unit: million sticks in the cigarette label, thousand kilograms in the non-cigarette total's label and thousand hectolitres of alcohol in Table 4.12's note. It stops before writing on @@ -532,9 +532,9 @@ the forecast levels, though not their growth. importer writes `(1 + aggregate clearance growth) / (1 + ons.population growth) - 1`, using the population in `yoy_growth.yaml`, so rerun it whenever that - series changes. Until then the six - clearance series and the seven gambling bases keep the values written - for the old population, and the byte-for-byte test fails. Then update + series changes. Until then the six clearance series and the seven + gambling bases keep the values the importer wrote for the old + population, and the byte-for-byte test fails. Then update the published levels in `policyengine_uk/tests/test_excise_quantity_projection.py` (`CLEARANCES`) and `policyengine_uk/tests/test_import_obr_forecasts.py` From 433b20fb72ba16896d7b41811e306dfa37131f50 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 2 Nov 2026 02:53:41 -0500 Subject: [PATCH 22/27] Restore the clearance layout checks; leave phonetic runs out of a cell The closed list of clearance texts does not settle a line's unit: two rearranged copies of the real October 2026 Table 4.11 keep every text on the list and still put hand-rolled and other tobacco under the cigarette label's million sticks. The importer again refuses any text beside the years in the fiscal-year row over a clearance line, and requires each part of a total (part_of_total) to sit under an "of which:" row directly below the total's label, with only the total's other parts between (validate_clearance_part). _string_text now joins a string's own and each run's , so the of a phonetic run () no longer joins a cell's text. An empty cell whose phonetic run spelt the alcohol receipts title could otherwise end the clearance table above an unread note. Tests: the two layouts and a second year row as refusal tests, the year row and a part's structure in the arbitrary-text property, phonetic runs in the formatting-run property, and a phonetic end title on the real Table 4.12. README and guide list both checks; four docstrings reflowed. Co-Authored-By: Claude Opus 5.5 --- docs/engineering/excise-duties.md | 21 +- .../gov/economic_assumptions/README.md | 50 +-- .../tests/test_import_obr_forecasts.py | 327 +++++++++++++++--- policyengine_uk/utils/import_obr_forecasts.py | 139 +++++++- 4 files changed, 437 insertions(+), 100 deletions(-) diff --git a/docs/engineering/excise-duties.md b/docs/engineering/excise-duties.md index 4cacde27d2..485cbe7bfe 100644 --- a/docs/engineering/excise-duties.md +++ b/docs/engineering/excise-duties.md @@ -293,16 +293,25 @@ numbers and fiscal years, and stops before writing anything when: - the table lacks the cell that states a line's unit: the cigarette label's "(million sticks)", the non-cigarette total's "('000s kg)" for hand-rolled and other tobacco, or Table 4.12's note for alcohol; +- a text sits beside the years in the fiscal-year row over a line, such as + a unit column's heading or a line's label moved there; +- hand-rolled or other tobacco does not sit under an "of which:" row + directly below the non-cigarette total's label, with only the total's + other parts between; - the OBR puts any cell beside a line's label and values; - the sheet repeats the table's heading, since the importer would check the text under one heading and could read a line under another. -The importer reads every formatting run of a cell's text. It names the -table and each unread cell, or the rows that repeat the heading. Whoever -runs it reads the text, converts the table's figures to the listed units if -it states another, and adds the text to the list. The importer checks no -cell above a table's title or below its end, so that person reads any note -there too. +The list alone does not settle a line's unit: a rearranged copy of the real +October 2026 Table 4.11 can put the two tobacco parts under the cigarette +label's million sticks with every text on the list. The two layout checks +stop it. The importer reads every formatting run of a cell's text, and +leaves out any phonetic run. It names the table and each unread cell, the +text in the fiscal-year row, the missing total or the rows that repeat the +heading. Whoever runs it reads the text, converts the table's figures to +the listed units if it states another, and adds the text to the list. The +importer checks no cell above a table's title or below its end, so that +person reads any note there too. ### Budget measures and reforms diff --git a/policyengine_uk/parameters/gov/economic_assumptions/README.md b/policyengine_uk/parameters/gov/economic_assumptions/README.md index e598bc7847..98246bdc15 100644 --- a/policyengine_uk/parameters/gov/economic_assumptions/README.md +++ b/policyengine_uk/parameters/gov/economic_assumptions/README.md @@ -511,33 +511,35 @@ the forecast levels, though not their growth. The same run also refreshes the six alcohol and tobacco clearance series under `per_capita` from the new EFO's tobacco clearances and alcohol clearance forecasts, and the previous EFO's for the outturn year's base. - From each table's title through its configured end, the importer - compares text cells with `CLEARANCE_TABLE_TEXTS`, in both EFOs. The end - falls on the row above a title `CLEARANCE_TABLE_ENDS` lists, or on the - sheet's last row. It skips the title cell, numbers and fiscal years. It - also needs the cell that - states each line's unit: million sticks in the cigarette label, - thousand kilograms in the non-cigarette total's label and thousand - hectolitres of alcohol in Table 4.12's note. It stops before writing on - any other text and names the table and the cell. Read that text, convert - the table's figures to these units if it states another, and add the - text to the list. The importer also stops on any cell beside a line's - label and values. When a sheet repeats a clearance table's heading, it - stops and names the rows. It checks no cell above a table's title or - below its end, so read any note there yourself. The list holds each - EFO's footnotes as the OBR wrote them, and the importer does not compare - the products one EFO's line covers with another's. Check that each - alcohol line covers the same products in both EFOs: in March 2026 the - OBR added "Includes other fermented products" to the wine line. The - importer writes + From each table's title through its configured end, the importer compares + text cells with `CLEARANCE_TABLE_TEXTS`, in both EFOs. The end falls on + the row above a title `CLEARANCE_TABLE_ENDS` lists, or on the sheet's + last row. It skips the title cell, numbers and fiscal years. It also + needs the cell that states each line's unit: million sticks in the + cigarette label, thousand kilograms in the non-cigarette total's label + and thousand hectolitres of alcohol in Table 4.12's note. It stops before + writing on any other text and names the table and the cell. Read that + text, convert the table's figures to these units if it states another, + and add the text to the list. A table of listed texts can still put a + unit over the wrong lines, so the importer also stops on any text beside + the years in the fiscal-year row over a line, and when hand-rolled or + other tobacco does not sit under an "of which:" row directly below the + non-cigarette total's label, with only the total's other parts between. + It stops on any cell beside a line's label and values too. When a sheet + repeats a clearance table's heading, the importer stops and names the + rows. It checks no cell above a table's title or below its end, so read + any note there yourself. The list holds each EFO's footnotes as the OBR + wrote them, and the importer does not compare the products one EFO's line + covers with another's. Check that each alcohol line covers the same + products in both EFOs: in March 2026 the OBR added "Includes other + fermented products" to the wine line. The importer writes `(1 + aggregate clearance growth) / (1 + ons.population growth) - 1`, using the population in `yoy_growth.yaml`, so rerun it whenever that series changes. Until then the six clearance series and the seven - gambling bases keep the values the importer wrote for the old - population, and the byte-for-byte test fails. Then update - the published levels in - `policyengine_uk/tests/test_excise_quantity_projection.py` - (`CLEARANCES`) and `policyengine_uk/tests/test_import_obr_forecasts.py` + gambling bases keep the values the importer wrote for the old population, + and the byte-for-byte test fails. Then update the published levels in + `policyengine_uk/tests/test_excise_quantity_projection.py` (`CLEARANCES`) + and `policyengine_uk/tests/test_import_obr_forecasts.py` (`OCTOBER_2026_CLEARANCES` and `MARCH_2026_CLEARANCES`, renamed for the new EFOs), the values those tests check by hand, the index table in `docs/engineering/excise-duties.md`, and the real-table fixtures in diff --git a/policyengine_uk/tests/test_import_obr_forecasts.py b/policyengine_uk/tests/test_import_obr_forecasts.py index d87e32d60c..315d08cbc6 100644 --- a/policyengine_uk/tests/test_import_obr_forecasts.py +++ b/policyengine_uk/tests/test_import_obr_forecasts.py @@ -545,11 +545,11 @@ def clearance_table( unit: str | None = None, ) -> dict[int, list[str]]: """Build a table in the layout of the OBR's Tables 4.11 and 4.12: title - in column B, the "Outturn" and "Forecast" markers on the next row, fiscal - years on the row after, then one row per line and any notes. A clearance - table has no - unit cell, since the OBR states the unit in a label or a note; ``unit`` - adds one on a row of its own below the title, as a receipts table has.""" + in column B, the "Outturn" and "Forecast" markers on the next row, + fiscal years on the row after, then one row per line and any notes. A + clearance table has no unit cell, since the OBR states the unit in a + label or a note; ``unit`` adds one on a row of its own below the title, + as a receipts table has.""" years = max(len(levels) for _, levels in rows) columns = [column_letter(3 + i) for i in range(years)] sheet = {start: [make_inline_cell(f"B{start}", title)]} @@ -614,9 +614,8 @@ def tobacco_table(chapter, first_year, levels, outturn_years=1): def alcohol_table(chapter, first_year, levels, outturn_years=1, note=None): """Build Table 4.12 (3.12 to March 2026): clearances in thousand hectolitres of alcohol, which the OBR states in a note, then the duties' - receipts in - £ billion under the same labels, which the importer must not read for - the clearance lines.""" + receipts in £ billion under the same labels, which the importer must not + read for the clearance lines.""" labels = { "beer_and_cider": " Beer and cider", "wine": " Wine1", @@ -3186,9 +3185,8 @@ def test_rendered_clearances_grow_the_weighted_total_with_the_clearances(years, """For any clearances and population growth: in every year the household quantity's growth times the household weight's growth equals the clearances' growth, to the series' six decimals, and equals 1 after the - forecast; the - series stays above -100%, so its index stays positive; a second run - changes nothing.""" + forecast; the series stays above -100%, so its index stays positive; a + second run changes nothing.""" base, levels = {}, {} for key in CLEARANCE_LEVELS: level = data.draw(CLEARANCE_LEVEL) @@ -3425,11 +3423,11 @@ def test_real_tables_with_a_kg_cell_beside_a_line_are_refused(tmp_path): def edit_real_table(workbook, table, cells=(), scale=(), notes=()): """Set each cell in ``cells`` to its text, number or, for None, nothing; - a tuple of texts gives one string with a formatting run for each. Add - the cell and its row where the sheet lacks them. Multiply the cells in - ``scale`` by their factors, then append ``notes`` in column B below the - sheet's last row. Preserve every other archive entry, including the - shared strings.""" + a tuple of texts gives one string with a formatting run for each, and an + element the inline string it holds. Add the cell and its row where the + sheet lacks them. Multiply the cells in ``scale`` by their factors, then + append ``notes`` in column B below the sheet's last row. Preserve every + other archive entry, including the shared strings.""" cells, scale = dict(cells), dict(scale) output = BytesIO() with ZipFile(BytesIO(workbook)) as source, ZipFile(output, "w") as dest: @@ -3471,6 +3469,9 @@ def cell(reference): elif isinstance(value, tuple): target.set("t", "inlineStr") target.append(formatting_runs("is", value)) + elif isinstance(value, ET.Element): + target.set("t", "inlineStr") + target.append(value) elif value is not None: ET.SubElement(target, f"{{{XLSX_MAIN}}}v").text = repr(float(value)) for entry in source.infolist(): @@ -3481,9 +3482,10 @@ def cell(reference): return output.getvalue() -def formatting_runs(tag, texts): +def formatting_runs(tag, texts, reading=None): """Build a ``tag`` string element, ``is`` or ``si``, with a formatting - run for each of ``texts``, the later runs bold.""" + run for each of ``texts``, the later runs bold, and with ``reading``, a + phonetic run (````) that holds it.""" string = ET.Element(f"{{{XLSX_MAIN}}}{tag}") for index, part in enumerate(texts): run = ET.SubElement(string, f"{{{XLSX_MAIN}}}r") @@ -3494,9 +3496,19 @@ def formatting_runs(tag, texts): ET.SubElement( run, f"{{{XLSX_MAIN}}}t", {f"{{{XML_NAMESPACE}}}space": "preserve"} ).text = part + if reading is not None: + string.append(phonetic_run(reading, len("".join(texts)))) return string +def phonetic_run(reading, length): + """Build a phonetic run (````) that holds ``reading`` over the + first ``length`` characters of its string.""" + run = ET.Element(f"{{{XLSX_MAIN}}}rPh", {"sb": "0", "eb": str(length)}) + ET.SubElement(run, f"{{{XLSX_MAIN}}}t").text = reading + return run + + def real_levels(workbook, key): """Read the levels of ``key`` from its table alone in ``workbook``, and from the real October 2026 table.""" @@ -3842,6 +3854,145 @@ def test_a_real_table_without_its_unit_cell_refuses_the_lines_it_describes(): ) +def in_year_row(table, heading, key): + """Write out the importer's refusal of the ``key`` line of real Table + ``table`` with ``heading`` in the fiscal-year row over the line.""" + return ( + f"Table {table} (sheet {table}) holds {heading} in the fiscal-year row " + f"over the {'.'.join(CLEARANCE_SPECS[key].path)} line. A text beside " + "the years could give the line another unit, so the importer refuses " + "any text there." + ) + + +def not_under_total(table, key): + """Write out the importer's refusal of the ``key`` line of real Table + ``table``, a part that does not sit under its total.""" + spec = CLEARANCE_SPECS[key] + return ( + f"Table {table} (sheet {table}) lacks {spec.unit_cell!r} on the row " + f'directly above the "of which:" row over the {".".join(spec.path)} ' + "line. The line takes its unit from that total's label, so the importer " + "needs the label there, with only the total's other parts between the " + '"of which:" row and the line.' + ) + + +# A table of listed texts can still put a unit over the wrong lines. Each +# layout maps rows of the real October 2026 Table 4.11, columns B to I, to +# the real rows they take: "heading" for the cigarette label beside row 4's +# fiscal years, None for nothing. The helper halves the hand-rolled and +# other tobacco figures (real rows 8 and 9) wherever they land, so the two +# parts sit under the cigarette label's million sticks. +REAL_TOBACCO_ROWS = {row.number: row for row in REAL_OCTOBER_2026_SHEETS["4.11"]} +TOBACCO_LAYOUTS = { + # The cigarette label heads a second row of years over the parts, in + # place of "of which:", and the non-cigarette total moves below them. + "cigarette-unit-year-header": ({6: None, 7: "heading", 10: 6}, "year row"), + # The cigarette line moves directly above "of which:", and the + # non-cigarette total below the parts. + "cigarette-unit-of-which": ({5: None, 6: 5, 10: 6}, "total"), + # A second row of years, headed by the cigarette label, sits over the + # non-cigarette total, "of which:" and the parts, each a row lower. + "cigarette-unit-second-year-row": ( + {6: "heading", 7: 6, 8: 7, 9: 8, 10: 9}, + "year row", + ), +} + + +def tobacco_layout(name): + """Rewrite the real October 2026 Table 4.11 in the layout ``name``.""" + layout, _ = TOBACCO_LAYOUTS[name] + cells = {} + for target, source in layout.items(): + for column in "BCDEFGHI": + if source == "heading": + value = REAL_TOBACCO_ROWS[5 if column == "B" else 4][column] + elif source is None or REAL_TOBACCO_ROWS[source].get(column) is None: + value = None + elif column == "B": + value = REAL_TOBACCO_ROWS[source][column] + else: + value = float(REAL_TOBACCO_ROWS[source][column]) + value /= 2 if source in (8, 9) else 1 + cells[f"{column}{target}"] = value + halved = { + f"{column}{row}": 0.5 + for row in (8, 9) + if row not in layout + for column in "CDEFGHI" + } + return edit_real_table( + REAL_OCTOBER_2026_RECEIPTS, "4.11", cells=cells, scale=halved + ) + + +@pytest.mark.parametrize("name", sorted(TOBACCO_LAYOUTS)) +def test_a_layout_that_puts_the_parts_under_another_unit_stops_the_run(name): + """Each layout keeps every text on the list and the non-cigarette + total's label in the table, but sets the hand-rolled and other tobacco + lines under the cigarette label's million sticks. In thousand kg, the + unit March 2026 states for them, the halved figures give 2025-26 growth + of -58.6% for hand-rolled tobacco and -56.8% for other tobacco, and the + published figures -17.1% and -13.6%. The importer reads cigarettes as + before and refuses both parts. It names the fiscal-year row when that + row holds the label, and the missing total otherwise. The run stops + before it renders anything.""" + _, check = TOBACCO_LAYOUTS[name] + changed = tobacco_layout(name) + rows = read_sheet_rows(changed, "4.11") + texts = { + receipts_label(value) + for row in rows[2:] + for value in row.values() + if receipts_label(value) + and not re.fullmatch(r"\d{4}-\d{2}|[\d.]+", receipts_label(value)) + } + assert texts <= CLEARANCE_TABLE_TEXTS[TOBACCO_CLEARANCES] + assert CLEARANCE_SPECS["hand_rolling_tobacco"].unit_cell in texts + for key, read, real in ( + ("hand_rolling_tobacco", -0.585697, -0.171394), + ("other_tobacco_products", -0.568094, -0.136189), + ): + (line,) = [ + row + for row in rows + if receipts_label(row.get("B")) == CLEARANCE_SPECS[key].row + ] + before = float(MARCH_2026_CLEARANCES[key][0]) + assert round(float(line["C"]) / before - 1, 6) == read, key + published = float(OCTOBER_2026_CLEARANCES[key][0]) + assert round(published / before - 1, 6) == real, key + sheets = {"4.11": rows, "4.12": REAL_OCTOBER_2026_SHEETS["4.12"]} + expected = {} + for key in TOBACCO_LINES: + if key == "cigarettes": + assert ( + find_receipts_line(sheets, CLEARANCE_SPECS[key]).levels + == find_receipts_line( + REAL_OCTOBER_2026_SHEETS, CLEARANCE_SPECS[key] + ).levels + ) + continue + expected[key] = ( + in_year_row("4.11", {"B": REAL_TOBACCO_ROWS[5]["B"]}, key) + if check == "year row" + else not_under_total("4.11", key) + ) + with pytest.raises(ValueError) as refused: + find_receipts_line(sheets, CLEARANCE_SPECS[key]) + assert str(refused.value) == expected[key], key + with pytest.raises(ValueError) as refused: + render( + get_yoy_growth_path().read_text(), + current=changed, + previous=REAL_RECEIPTS[2026], + **COMMITTED_RELEASES, + ) + assert str(refused.value) == expected["hand_rolling_tobacco"] + + def test_the_alcohol_receipts_table_stays_outside_the_clearance_table(): """On the real October 2026 sheet 4.12 the OBR follows the alcohol clearances with the alcohol receipts forecasts, which hold "£ billion" @@ -3963,12 +4114,14 @@ def unread_text(text, title): def test_any_text_no_one_has_read_stops_a_real_clearance_table(table, data): """Put one text in a real table, in an empty cell of any row from its title through the table's end, in any column, or on a new row. Any text - outside the table's - list, from st.text() or shaped like a unit, stops every line of the - table: the importer names the cell, or finds no line or table where the - text took a label's or the title's place. A text from the list, other - than a line's label or a marker, in a cell outside the lines and the - year row, changes nothing: the importer reads every line as before.""" + outside the table's list, from st.text() or shaped like a unit, stops + every line of the table: the importer names the cell, or finds no line + or table where the text took a label's or the title's place. A text from + the list, other than a line's label or a marker, in a cell outside the + lines, stops every line in the fiscal-year row, and a part of a total + between the total's row and the part; the importer names the year row or + the missing total. Anywhere else it changes nothing: the importer reads + every line as before.""" release, title = table _, sheet = REAL_CLEARANCE_TABLES[table] real, start, end, _ = real_clearance_rows(release, title) @@ -3996,27 +4149,48 @@ def test_any_text_no_one_has_read_stops_a_real_clearance_table(table, data): column = data.draw(st.sampled_from("ABCDEFGHIJKLM"), label="column") if data.draw(st.booleans(), label="new row"): lowest = year_row + 1 if listed else start + 1 - rows.insert(data.draw(st.integers(lowest, end), label="row"), {column: text}) + place = data.draw(st.integers(lowest, end), label="row") + rows.insert(place, {column: text}) else: empty = [ i for i in range(start, end) if not str(rows[i].get(column) or "").strip() ] if listed: empty = [ - i - for i in empty - if i not in line_rows and i != year_row and (i != start or column > "B") + i for i in empty if i not in line_rows and (i != start or column > "B") ] assume(empty) - rows[data.draw(st.sampled_from(empty), label="row")][column] = text + place = data.draw(st.sampled_from(empty), label="row") + rows[place][column] = text + # The real rows of each line and of the total that states a part's unit. + # A new row at ``place`` sits above the real row there. + real_row = { + receipts_label(real[i].get("B")): i + for i in range(start, end) + if real[i].get("B") + } for key in keys: spec = CLEARANCE_SPECS[key] if listed: - event("listed: read") - assert ( - find_receipts_line({sheet: rows}, spec).levels - == find_receipts_line({sheet: real}, spec).levels - ) + if place == year_row: + event("listed: fiscal-year row") + expected = in_year_row(sheet, {column: text}, key) + elif ( + spec.part_of_total + and real_row[spec.unit_cell] < place <= real_row[spec.row] + ): + event("listed: between a part and its total") + expected = not_under_total(sheet, key) + else: + event("listed: read") + assert ( + find_receipts_line({sheet: rows}, spec).levels + == find_receipts_line({sheet: real}, spec).levels + ) + continue + with pytest.raises(ValueError) as refused: + find_receipts_line({sheet: rows}, spec) + assert str(refused.value) == expected continue with pytest.raises(ValueError) as refused: find_receipts_line({sheet: rows}, spec) @@ -4078,15 +4252,16 @@ def test_the_check_reads_every_formatting_run_of_a_cell(name): assert str(refused.value) == expected -def runs_workbook(texts): +def runs_workbook(texts, reading=None): """Build a one-sheet workbook that holds ``texts`` in A1 as an inline string and in A2 as a shared string, each with a formatting run for each - text, and their join in A3 as an inline string of one ````.""" + text, and their join in A3 as an inline string of one ````. With + ``reading``, each string ends in a phonetic run that holds it.""" sheet = make_sheet( { 1: [ '' - + ET.tostring(formatting_runs("is", texts), encoding="unicode") + + ET.tostring(formatting_runs("is", texts, reading), encoding="unicode") + "" ], 2: ['0'], @@ -4096,8 +4271,12 @@ def runs_workbook(texts): sheet = ET.fromstring(sheet) joined = sheet.find("m:sheetData/m:row/m:c[@r='A3']/m:is/m:t", XLSX_NS) joined.text = "".join(texts) + if reading is not None: + sheet.find("m:sheetData/m:row/m:c[@r='A3']/m:is", XLSX_NS).append( + phonetic_run(reading, len(joined.text)) + ) strings = ET.Element(f"{{{XLSX_MAIN}}}sst") - strings.append(formatting_runs("si", texts)) + strings.append(formatting_runs("si", texts, reading)) output = BytesIO() with ZipFile(BytesIO(make_workbook({"1": ET.tostring(sheet)}))) as source: with ZipFile(output, "w") as dest: @@ -4107,32 +4286,69 @@ def runs_workbook(texts): return output.getvalue() +STRING_TEXT = st.text(st.characters(min_codepoint=0x20, max_codepoint=0xD7FF)) + + @settings(max_examples=100, deadline=None, derandomize=True) @given( - texts=st.lists( - st.text(st.characters(min_codepoint=0x20, max_codepoint=0xD7FF)), - min_size=1, - max_size=4, - ) + texts=st.lists(STRING_TEXT, min_size=1, max_size=4), + reading=st.one_of(st.none(), STRING_TEXT), ) -def test_a_string_reads_whole_in_any_formatting_runs(texts): +def test_a_string_reads_whole_in_any_formatting_runs(texts, reading): """However a spreadsheet splits a text into formatting runs, the reader returns the whole text, from an inline string and from a shared string - alike, as it does from one ````.""" - rows = read_sheet_rows(runs_workbook(texts), "1") + alike, as it does from one ````. It leaves out the text of any + phonetic run (````) the string holds.""" + rows = read_sheet_rows(runs_workbook(texts, reading), "1") assert [(row.number, row["A"]) for row in rows] == [ (number, "".join(texts)) for number in (1, 2, 3) ] -# A sheet that repeats a clearance table's heading must stop the run: the -# importer would check the text under one and could read a line under the -# other. TEN_THOUSAND_HECTOLITRES = ( "Note: Spirits forecasts shown in ten thousand hectolitres of alcohol." ) +def test_a_phonetic_run_cannot_end_a_clearance_table(): + """The helper writes B11 of the real October 2026 sheet 4.12, empty in + the published sheet, as an inline string with no text of its own and a + phonetic run that holds "4.12 Alcohol receipts forecasts". It puts a + note in ten thousand hectolitres in C11 and divides the spirits + forecasts by ten. A reader that kept the phonetic run would end the + clearance table at B11, above the note, and the importer would read + 2026-27 spirits growth of -90.1% in place of -0.8%. This reader returns + B11 empty, so the table runs to the real title in row 12, and the + importer refuses every alcohol line and names the note.""" + changed = edit_real_table( + REAL_OCTOBER_2026_RECEIPTS, + "4.12", + cells={ + "B11": formatting_runs("is", ("",), "4.12 Alcohol receipts forecasts"), + "C11": TEN_THOUSAND_HECTOLITRES, + }, + scale={f"{column}7": 0.1 for column in "DEFGHI"}, + ) + rows = read_sheet_rows(changed, "4.12") + (row,) = [row for row in rows if row.number == 11] + assert (row["B"], row["C"]) == ("", TEN_THOUSAND_HECTOLITRES) + (spirits,) = [line for line in rows if line.number == 7] + assert round(float(spirits["D"]) / float(spirits["C"]) - 1, 6) == -0.900793 + expected = unread("4.12", ALCOHOL_CLEARANCES, (TEN_THOUSAND_HECTOLITRES, "C")) + for key in ALCOHOL_LINES: + with pytest.raises(ValueError) as refused: + find_receipts_line({"4.12": rows}, CLEARANCE_SPECS[key]) + assert str(refused.value) == expected, key + with pytest.raises(ValueError) as refused: + compute_receipts_growth( + changed, REAL_RECEIPTS[2026], specs=CLEARANCE_SERIES_SPECS + ) + assert str(refused.value) == expected + + +# A sheet that repeats a clearance table's heading must stop the run: the +# importer would check the text under one and could read a line under the +# other. def repeat_alcohol_heading(workbook, sheet): """Clear the lines of the real alcohol clearance table on ``sheet``, rows 5 to 7, and repeat its rows 2 to 10 below the sheet's last row, so a @@ -4177,13 +4393,12 @@ def test_a_repeated_clearance_heading_stops_the_run(tmp_path): "4.12 Alcohol clearance forecasts" heading in row 53. It divides the repeated spirits forecasts by ten and states ten thousand hectolitres in a note below them. Each line then sits once, under the second heading, - and the first keeps the table's listed note. Read in thousand - hectolitres, the repeated figures give 2026-27 spirits growth of -90.1%, - where the published figures give -0.8%. The importer refuses every - alcohol line and names both headings' rows, and the renderer gets no - growth. A run on the real March 2026 tables, changed the same way, stops - before it writes anything. The importer reads the tobacco lines as - before.""" + and the first keeps the table's listed note. In thousand hectolitres, + the repeated figures give 2026-27 spirits growth of -90.1%, and the + published figures -0.8%. The importer refuses every alcohol line and + names both headings' rows, and the renderer gets no growth. A run on the + real March 2026 tables, changed the same way, stops before it writes + anything. The importer reads the tobacco lines as before.""" changed, row = repeat_alcohol_heading(REAL_OCTOBER_2026_RECEIPTS, "4.12") assert row == 53 rows = read_sheet_rows(changed, "4.12") diff --git a/policyengine_uk/utils/import_obr_forecasts.py b/policyengine_uk/utils/import_obr_forecasts.py index 7acb632726..1859195696 100644 --- a/policyengine_uk/utils/import_obr_forecasts.py +++ b/policyengine_uk/utils/import_obr_forecasts.py @@ -227,11 +227,25 @@ def extract_economy_workbook_bytes( return Path(workbook_name).name, archive.read(workbook_name) +STRING_TEXT = f"{{{WORKBOOK_NS['main']}}}t" +STRING_RUN = f"{{{WORKBOOK_NS['main']}}}r" + + def _string_text(item: ET.Element) -> str: - """Join the text of every ```` in a shared or inline string, so the - reader returns the whole text of a string split into formatting runs - (````).""" - return "".join(node.text or "" for node in item.iterfind(".//main:t", WORKBOOK_NS)) + """Join a shared or inline string's own ```` and the ```` of each + formatting run (````), in order, so the reader returns the whole text + of a string split into runs. It skips any other child, such as a + phonetic run (````), whose ```` holds a reading of the text, not + more of it.""" + parts = [] + for child in item: + if child.tag == STRING_TEXT: + parts.append(child.text or "") + elif child.tag == STRING_RUN: + parts.extend( + node.text or "" for node in child.findall("main:t", WORKBOOK_NS) + ) + return "".join(parts) def _load_shared_strings(archive: ZipFile) -> list[str]: @@ -713,6 +727,9 @@ class ClearanceSeriesSpec(ReceiptsSeriesSpec): in which the OBR states the line's unit, as ``CLEARANCE_TABLE_TEXTS`` lists it. In both EFOs the importer requires that cell and refuses any text in the table that the list lacks (``validate_clearance_table``). + For a part of a total (``part_of_total``), that cell is the total's + label, and the importer requires it on the row directly above the + "of which:" row over the line (``validate_clearance_part``). Dataset projection grows household weights by ``ons.population``, so ``render_receipts_growth_yaml`` writes (1 + aggregate clearance growth) / (1 + ons.population growth) - 1 @@ -722,6 +739,7 @@ class ClearanceSeriesSpec(ReceiptsSeriesSpec): """ unit_cell: str = "" + part_of_total: bool = False def __post_init__(self): name = ".".join(self.path) @@ -809,12 +827,14 @@ def __post_init__(self): title=TOBACCO_CLEARANCES, row="hand rolled tobacco clearances", unit_cell=TOBACCO_KILOGRAMS, + part_of_total=True, ), ClearanceSeriesSpec( path=("per_capita", "tobacco_clearances", "other_tobacco_products"), title=TOBACCO_CLEARANCES, row="other tobacco products", unit_cell=TOBACCO_KILOGRAMS, + part_of_total=True, ), ClearanceSeriesSpec( path=("per_capita", "alcohol_clearances", "beer_and_cider"), @@ -1122,11 +1142,11 @@ class ReceiptsGrowth: The first year holds outturn growth: this EFO's first (outturn) year over the year before in the previous EFO, which marks that year as outturn. Each later year holds the ratio of this EFO's consecutive - levels. A line with its own - outturn starts at that outturn's second year and has no previous line; - one with rates grows its base instead (see ``ReceiptsSeriesSpec``). A - clearance line holds the clearances' growth, which - ``render_receipts_growth_yaml`` writes per head (``per_head_growth``). + levels. A line with its own outturn starts at that outturn's second year + and has no previous line; one with rates grows its base instead (see + ``ReceiptsSeriesSpec``). A clearance line holds the clearances' growth, + which ``render_receipts_growth_yaml`` writes per head + (``per_head_growth``). ``ratios`` holds one plus each year's growth before rounding, as ``compute_receipts_growth`` finds it, so ``per_head_growth`` rounds the @@ -1340,6 +1360,57 @@ def validate_clearance_table( ) +def validate_clearance_part( + table_rows: list[dict[str, str | None]], + line_index: int, + spec: ClearanceSeriesSpec, + where: str, +) -> None: + """Refuse a part of a total that does not sit under the total. + + The OBR states the unit of hand-rolled and other tobacco only in the + label of the total they break down, "Non-cigarette clearances ('000s + kg)", on the row directly above an "of which:" row. Going up from the + line in ``table_rows`` (the rows below the table's title), past the + total's other parts, the importer needs an "of which:" row with no other + cell, then a row that holds ``spec.unit_cell`` as its first text and + only numbers after it. A table of listed texts could otherwise put the + parts under another line, such as "Cigarette clearances (million + sticks)", and the importer would read them in the wrong unit. + """ + + def texts(row: dict[str, str | None]) -> list[str]: + return [ + receipts_label(value) + for _, value in sorted(row.items(), key=lambda item: column_index(item[0])) + if receipts_label(value) + ] + + parts = { + other.row + for other in CLEARANCE_SERIES_SPECS + if other.part_of_total + and (other.title, other.unit_cell) == (spec.title, spec.unit_cell) + } + above = [texts(row) for row in table_rows[:line_index]] + while above and above[-1][:1] and above[-1][0] in parts: + above.pop() + # The two rows above the line and its fellow parts, empty past the top. + total, of_which = ([[], []] + above)[-2:] + if ( + of_which != ["of which:"] + or total[:1] != [spec.unit_cell] + or not all(NUMBER_RE.fullmatch(text) for text in total[1:]) + ): + raise ValueError( + f"{where} lacks {spec.unit_cell!r} on the row directly above the " + f'"of which:" row over the {".".join(spec.path)} line. The line ' + "takes its unit from that total's label, so the importer needs the " + "label there, with only the total's other parts between the \"of " + 'which:" row and the line.' + ) + + def find_receipts_line( sheets: dict[str, list[dict[str, str | None]]], spec: ReceiptsSeriesSpec ) -> ReceiptsLine: @@ -1350,7 +1421,10 @@ def find_receipts_line( headers and its "Outturn" and "Forecast" markers, which may sit on one row or several. Before it reads a clearance line's values, it checks the line's table, from its title through its end, against the texts someone - has read (``validate_clearance_table``). + has read (``validate_clearance_table``). It then refuses any text beside + the years in the fiscal-year row over the line, and a part of a total + that does not sit under the total (``validate_clearance_part``): a + table of listed texts can still put a unit over the wrong lines. """ matches = [] for sheet, rows in sheets.items(): @@ -1359,12 +1433,14 @@ def find_receipts_line( continue where = f"Table {table} (sheet {sheet})" years: dict[str, int] = {} + year_row: dict[str, str | None] = {} markers: dict[str, int] = {} header_rows = [] - for row in table_rows: + for index, row in enumerate(table_rows): row_years = _fiscal_year_columns(row) if row_years: years = row_years + year_row = row continue if not years: header_rows.append(row) @@ -1394,7 +1470,16 @@ def find_receipts_line( ] if labels and receipts_label(labels[0][1]) == spec.row: matches.append( - (table, sheet, labels[0], row, years, markers, header_rows) + ( + table, + sheet, + labels[0], + row, + years, + markers, + header_rows, + (year_row, table_rows, index), + ) ) name = ".".join(spec.path) @@ -1404,7 +1489,16 @@ def find_receipts_line( f'Expected one "{spec.row}" row in the "{spec.title}" table for ' f"{name}; found {found}" ) - table, sheet, (label_column, label), row, years, markers, header_rows = matches[0] + ( + table, + sheet, + (label_column, label), + row, + years, + markers, + header_rows, + (year_row, table_rows, line_index), + ) = matches[0] where = f"Table {table} (sheet {sheet})" if isinstance(spec, ClearanceSeriesSpec): validate_clearance_table(sheets[sheet], table, spec, where) @@ -1427,7 +1521,24 @@ def find_receipts_line( f"{where} holds {beside} beside the {name} row's label and values; " "a unit or note on the row needs deliberate importer support" ) - if not isinstance(spec, ClearanceSeriesSpec): + if isinstance(spec, ClearanceSeriesSpec): + # The OBR states each clearance unit in a label or a note. A text + # beside the years, such as a unit column's heading or a line's label + # moved there, could head the values below it with another unit. + heading = { + column: value + for column, value in year_row.items() + if normalise_label(value) and column not in years + } + if heading: + raise ValueError( + f"{where} holds {heading} in the fiscal-year row over the {name} " + "line. A text beside the years could give the line another " + "unit, so the importer refuses any text there." + ) + if spec.part_of_total: + validate_clearance_part(table_rows, line_index, spec, where) + else: validate_receipts_unit(header_rows, where) levels = {} for column, year in years.items(): From 7555d402d7295aec4fa0e1df23434319150a3d7a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 2 Nov 2026 05:13:07 -0500 Subject: [PATCH 23/27] Give the layout checks' comments and docs active verbs Comments, docstrings, the README step and the guide only: the importer and its tests parse to the same syntax trees as at 433b20fb7 with docstrings left out. Co-Authored-By: Claude Opus 5.5 --- docs/engineering/excise-duties.md | 16 ++++++++-------- .../gov/economic_assumptions/README.md | 16 ++++++++-------- .../tests/test_import_obr_forecasts.py | 10 +++++----- policyengine_uk/utils/import_obr_forecasts.py | 11 ++++++----- 4 files changed, 27 insertions(+), 26 deletions(-) diff --git a/docs/engineering/excise-duties.md b/docs/engineering/excise-duties.md index 485cbe7bfe..714bc36a7b 100644 --- a/docs/engineering/excise-duties.md +++ b/docs/engineering/excise-duties.md @@ -294,7 +294,7 @@ numbers and fiscal years, and stops before writing anything when: "(million sticks)", the non-cigarette total's "('000s kg)" for hand-rolled and other tobacco, or Table 4.12's note for alcohol; - a text sits beside the years in the fiscal-year row over a line, such as - a unit column's heading or a line's label moved there; + a unit column's heading or a line's label; - hand-rolled or other tobacco does not sit under an "of which:" row directly below the non-cigarette total's label, with only the total's other parts between; @@ -302,13 +302,13 @@ numbers and fiscal years, and stops before writing anything when: - the sheet repeats the table's heading, since the importer would check the text under one heading and could read a line under another. -The list alone does not settle a line's unit: a rearranged copy of the real -October 2026 Table 4.11 can put the two tobacco parts under the cigarette -label's million sticks with every text on the list. The two layout checks -stop it. The importer reads every formatting run of a cell's text, and -leaves out any phonetic run. It names the table and each unread cell, the -text in the fiscal-year row, the missing total or the rows that repeat the -heading. Whoever runs it reads the text, converts the table's figures to +A table can keep every text on the list and still put a line under another +unit: a rearranged copy of the real October 2026 Table 4.11 can put the two +tobacco parts under the cigarette label's million sticks. The two layout +checks stop it. The importer reads every formatting run of a cell's text, +and leaves out any phonetic run. It names the table and each unread cell, +the text in the fiscal-year row, the missing total or the rows that repeat +the heading. Whoever runs it reads the text, converts the table's figures to the listed units if it states another, and adds the text to the list. The importer checks no cell above a table's title or below its end, so that person reads any note there too. diff --git a/policyengine_uk/parameters/gov/economic_assumptions/README.md b/policyengine_uk/parameters/gov/economic_assumptions/README.md index 98246bdc15..dbafa58734 100644 --- a/policyengine_uk/parameters/gov/economic_assumptions/README.md +++ b/policyengine_uk/parameters/gov/economic_assumptions/README.md @@ -522,14 +522,14 @@ the forecast levels, though not their growth. text, convert the table's figures to these units if it states another, and add the text to the list. A table of listed texts can still put a unit over the wrong lines, so the importer also stops on any text beside - the years in the fiscal-year row over a line, and when hand-rolled or - other tobacco does not sit under an "of which:" row directly below the - non-cigarette total's label, with only the total's other parts between. - It stops on any cell beside a line's label and values too. When a sheet - repeats a clearance table's heading, the importer stops and names the - rows. It checks no cell above a table's title or below its end, so read - any note there yourself. The list holds each EFO's footnotes as the OBR - wrote them, and the importer does not compare the products one EFO's line + the years in the fiscal-year row over a line. It stops when hand-rolled + or other tobacco does not sit under an "of which:" row directly below the + non-cigarette total's label, with only the total's other parts between, + and on any cell beside a line's label and values. When a sheet repeats a + clearance table's heading, the importer stops and names the rows. It + checks no cell above a table's title or below its end, so read any note + there yourself. The list holds each EFO's footnotes as the OBR wrote + them, and the importer does not compare the products one EFO's line covers with another's. Check that each alcohol line covers the same products in both EFOs: in March 2026 the OBR added "Includes other fermented products" to the wine line. The importer writes diff --git a/policyengine_uk/tests/test_import_obr_forecasts.py b/policyengine_uk/tests/test_import_obr_forecasts.py index 315d08cbc6..3cb9f69727 100644 --- a/policyengine_uk/tests/test_import_obr_forecasts.py +++ b/policyengine_uk/tests/test_import_obr_forecasts.py @@ -3892,7 +3892,7 @@ def not_under_total(table, key): # The cigarette line moves directly above "of which:", and the # non-cigarette total below the parts. "cigarette-unit-of-which": ({5: None, 6: 5, 10: 6}, "total"), - # A second row of years, headed by the cigarette label, sits over the + # A second row of years, which the cigarette label heads, sits over the # non-cigarette total, "of which:" and the parts, each a row lower. "cigarette-unit-second-year-row": ( {6: "heading", 7: 6, 8: 7, 9: 8, 10: 9}, @@ -4118,10 +4118,10 @@ def test_any_text_no_one_has_read_stops_a_real_clearance_table(table, data): every line of the table: the importer names the cell, or finds no line or table where the text took a label's or the title's place. A text from the list, other than a line's label or a marker, in a cell outside the - lines, stops every line in the fiscal-year row, and a part of a total - between the total's row and the part; the importer names the year row or - the missing total. Anywhere else it changes nothing: the importer reads - every line as before.""" + lines, stops every line when it sits in the fiscal-year row, and a part + of a total when it sits between the part and the total's row; the + importer names the year row or the missing total. Anywhere else it + changes nothing: the importer reads every line as before.""" release, title = table _, sheet = REAL_CLEARANCE_TABLES[table] real, start, end, _ = real_clearance_rows(release, title) diff --git a/policyengine_uk/utils/import_obr_forecasts.py b/policyengine_uk/utils/import_obr_forecasts.py index 1859195696..3559c3fad1 100644 --- a/policyengine_uk/utils/import_obr_forecasts.py +++ b/policyengine_uk/utils/import_obr_forecasts.py @@ -727,9 +727,10 @@ class ClearanceSeriesSpec(ReceiptsSeriesSpec): in which the OBR states the line's unit, as ``CLEARANCE_TABLE_TEXTS`` lists it. In both EFOs the importer requires that cell and refuses any text in the table that the list lacks (``validate_clearance_table``). - For a part of a total (``part_of_total``), that cell is the total's - label, and the importer requires it on the row directly above the - "of which:" row over the line (``validate_clearance_part``). + For a part of a total (``part_of_total``), the OBR states the unit in + the total's label, and the importer requires that label on the row + directly above the "of which:" row over the line + (``validate_clearance_part``). Dataset projection grows household weights by ``ons.population``, so ``render_receipts_growth_yaml`` writes (1 + aggregate clearance growth) / (1 + ons.population growth) - 1 @@ -1523,8 +1524,8 @@ def find_receipts_line( ) if isinstance(spec, ClearanceSeriesSpec): # The OBR states each clearance unit in a label or a note. A text - # beside the years, such as a unit column's heading or a line's label - # moved there, could head the values below it with another unit. + # beside the years, such as a unit column's heading or a line's label, + # could head the values below it with another unit. heading = { column: value for column, value in year_row.items() From 45bd632e42d8dad980cc9c2c8a4ce8e86fd76da5 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 16:02:54 +0000 Subject: [PATCH 24/27] Require the non-cigarette total's figures on the row above "of which:" validate_clearance_part now needs the total's row to hold its label and one or more numbers after it, and nothing else, so a row that holds only the label no longer counts as the total. The refusal names the total's label as receipts_label gives it. A parametrized test inserts such a row into each real release's tobacco clearances and checks the refusal. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LwQq6L5XVCEvFyRi9rsktq --- docs/engineering/excise-duties.md | 4 +- .../gov/economic_assumptions/README.md | 4 +- .../tests/test_import_obr_forecasts.py | 38 ++++++++++++++++--- policyengine_uk/utils/import_obr_forecasts.py | 25 ++++++------ 4 files changed, 51 insertions(+), 20 deletions(-) diff --git a/docs/engineering/excise-duties.md b/docs/engineering/excise-duties.md index 714bc36a7b..cdb44df48e 100644 --- a/docs/engineering/excise-duties.md +++ b/docs/engineering/excise-duties.md @@ -296,8 +296,8 @@ numbers and fiscal years, and stops before writing anything when: - a text sits beside the years in the fiscal-year row over a line, such as a unit column's heading or a line's label; - hand-rolled or other tobacco does not sit under an "of which:" row - directly below the non-cigarette total's label, with only the total's - other parts between; + directly below the non-cigarette total's row, which holds its label and + figures, with only the total's other parts between; - the OBR puts any cell beside a line's label and values; - the sheet repeats the table's heading, since the importer would check the text under one heading and could read a line under another. diff --git a/policyengine_uk/parameters/gov/economic_assumptions/README.md b/policyengine_uk/parameters/gov/economic_assumptions/README.md index dbafa58734..32988cf35e 100644 --- a/policyengine_uk/parameters/gov/economic_assumptions/README.md +++ b/policyengine_uk/parameters/gov/economic_assumptions/README.md @@ -524,8 +524,8 @@ the forecast levels, though not their growth. unit over the wrong lines, so the importer also stops on any text beside the years in the fiscal-year row over a line. It stops when hand-rolled or other tobacco does not sit under an "of which:" row directly below the - non-cigarette total's label, with only the total's other parts between, - and on any cell beside a line's label and values. When a sheet repeats a + non-cigarette total's row, which holds its label and figures, with only + the total's other parts between, and on any cell beside a line's label and values. When a sheet repeats a clearance table's heading, the importer stops and names the rows. It checks no cell above a table's title or below its end, so read any note there yourself. The list holds each EFO's footnotes as the OBR wrote diff --git a/policyengine_uk/tests/test_import_obr_forecasts.py b/policyengine_uk/tests/test_import_obr_forecasts.py index 3cb9f69727..903127777c 100644 --- a/policyengine_uk/tests/test_import_obr_forecasts.py +++ b/policyengine_uk/tests/test_import_obr_forecasts.py @@ -3870,11 +3870,12 @@ def not_under_total(table, key): ``table``, a part that does not sit under its total.""" spec = CLEARANCE_SPECS[key] return ( - f"Table {table} (sheet {table}) lacks {spec.unit_cell!r} on the row " - f'directly above the "of which:" row over the {".".join(spec.path)} ' - "line. The line takes its unit from that total's label, so the importer " - "needs the label there, with only the total's other parts between the " - '"of which:" row and the line.' + f"Table {table} (sheet {table}) lacks the non-cigarette total's label " + f"({spec.unit_cell!r}, as receipts_label gives it) and its figures on " + 'the row directly above the "of which:" row over the ' + f"{'.'.join(spec.path)} line. The line takes its unit from that total's " + "label, so the importer needs the label and figures there, with only " + 'the total\'s other parts between the "of which:" row and the line.' ) @@ -3993,6 +3994,33 @@ def test_a_layout_that_puts_the_parts_under_another_unit_stops_the_run(name): assert str(refused.value) == expected["hand_rolling_tobacco"] +@pytest.mark.parametrize("release", ["March 2025", "March 2026", "October 2026"]) +def test_a_total_label_without_figures_does_not_count_as_the_total(release): + """Insert a row that holds only "Non-cigarette clearances ('000s kg)" + directly above the "of which:" row of the real tobacco clearances. The + row holds the total's label but none of its figures, so the importer + refuses hand-rolled and other tobacco with the part check's message. It + reads cigarettes as before.""" + _, sheet = REAL_CLEARANCE_TABLES[release, TOBACCO_CLEARANCES] + real, start, end, _ = real_clearance_rows(release, TOBACCO_CLEARANCES) + (of_which,) = [ + i for i in range(start, end) if receipts_label(real[i].get("B")) == "of which:" + ] + rows = [dict(row) for row in real] + rows.insert(of_which, {"B": "Non-cigarette clearances ('000s kg)"}) + for key in TOBACCO_LINES: + spec = CLEARANCE_SPECS[key] + if key == "cigarettes": + assert ( + find_receipts_line({sheet: rows}, spec).levels + == find_receipts_line({sheet: real}, spec).levels + ) + continue + with pytest.raises(ValueError) as refused: + find_receipts_line({sheet: rows}, spec) + assert str(refused.value) == not_under_total(sheet, key), key + + def test_the_alcohol_receipts_table_stays_outside_the_clearance_table(): """On the real October 2026 sheet 4.12 the OBR follows the alcohol clearances with the alcohol receipts forecasts, which hold "£ billion" diff --git a/policyengine_uk/utils/import_obr_forecasts.py b/policyengine_uk/utils/import_obr_forecasts.py index 3559c3fad1..e13c78ff1d 100644 --- a/policyengine_uk/utils/import_obr_forecasts.py +++ b/policyengine_uk/utils/import_obr_forecasts.py @@ -728,8 +728,8 @@ class ClearanceSeriesSpec(ReceiptsSeriesSpec): lists it. In both EFOs the importer requires that cell and refuses any text in the table that the list lacks (``validate_clearance_table``). For a part of a total (``part_of_total``), the OBR states the unit in - the total's label, and the importer requires that label on the row - directly above the "of which:" row over the line + the total's label, and the importer requires that label and the total's + figures on the row directly above the "of which:" row over the line (``validate_clearance_part``). Dataset projection grows household weights by ``ons.population``, so ``render_receipts_growth_yaml`` writes @@ -1374,10 +1374,11 @@ def validate_clearance_part( kg)", on the row directly above an "of which:" row. Going up from the line in ``table_rows`` (the rows below the table's title), past the total's other parts, the importer needs an "of which:" row with no other - cell, then a row that holds ``spec.unit_cell`` as its first text and - only numbers after it. A table of listed texts could otherwise put the - parts under another line, such as "Cigarette clearances (million - sticks)", and the importer would read them in the wrong unit. + cell, then a row that holds ``spec.unit_cell`` as its first text and one + or more numbers after it, and nothing else. A table of listed texts + could otherwise put the parts under another line, such as "Cigarette + clearances (million sticks)", and the importer would read them in the + wrong unit. """ def texts(row: dict[str, str | None]) -> list[str]: @@ -1401,14 +1402,16 @@ def texts(row: dict[str, str | None]) -> list[str]: if ( of_which != ["of which:"] or total[:1] != [spec.unit_cell] + or not total[1:] or not all(NUMBER_RE.fullmatch(text) for text in total[1:]) ): raise ValueError( - f"{where} lacks {spec.unit_cell!r} on the row directly above the " - f'"of which:" row over the {".".join(spec.path)} line. The line ' - "takes its unit from that total's label, so the importer needs the " - "label there, with only the total's other parts between the \"of " - 'which:" row and the line.' + f"{where} lacks the non-cigarette total's label ({spec.unit_cell!r}, " + "as receipts_label gives it) and its figures on the row directly " + f'above the "of which:" row over the {".".join(spec.path)} line. ' + "The line takes its unit from that total's label, so the importer " + "needs the label and figures there, with only the total's other " + 'parts between the "of which:" row and the line.' ) From 8d976334553863b0fcd7b0ee977d800395a22cb6 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 16:14:20 +0000 Subject: [PATCH 25/27] Release the branch's test caches when each module finishes test_import_obr_forecasts.py and test_excise_quantity_projection.py each gain a module-scoped autouse fixture that clears real_clearance_rows and current_law after the module's tests, as test_test_module_caches_released requires. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LwQq6L5XVCEvFyRi9rsktq --- policyengine_uk/tests/test_excise_quantity_projection.py | 8 ++++++++ policyengine_uk/tests/test_import_obr_forecasts.py | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/policyengine_uk/tests/test_excise_quantity_projection.py b/policyengine_uk/tests/test_excise_quantity_projection.py index 8102e67c6c..702fda11f7 100644 --- a/policyengine_uk/tests/test_excise_quantity_projection.py +++ b/policyengine_uk/tests/test_excise_quantity_projection.py @@ -82,6 +82,14 @@ def parameters(): return current_law() +@pytest.fixture(scope="module", autouse=True) +def release_current_law(): + # Drop the cached parameters once this module's tests are done, instead + # of keeping them for the rest of the test process. + yield + current_law.cache_clear() + + def growth(parameters, path, year): return parameters.get_child(path)(f"{year}-01-01") diff --git a/policyengine_uk/tests/test_import_obr_forecasts.py b/policyengine_uk/tests/test_import_obr_forecasts.py index 903127777c..db3f41536a 100644 --- a/policyengine_uk/tests/test_import_obr_forecasts.py +++ b/policyengine_uk/tests/test_import_obr_forecasts.py @@ -3553,6 +3553,14 @@ def real_clearance_rows(release, title): return rows, start, len(rows), None +@pytest.fixture(scope="module", autouse=True) +def release_real_clearance_rows(): + # Drop the cached sheets once this module's tests are done, instead of + # keeping them for the rest of the test process. + yield + real_clearance_rows.cache_clear() + + def unread(table, title, *cells): """Write out the importer's refusal of real Table ``table`` for the unread ``cells``, each as (text, column).""" From 8f3f0f396e9318de59032110aaa16351273f9ac3 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 16:31:01 +0000 Subject: [PATCH 26/27] Keep the README's clearance step within its line width Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01LwQq6L5XVCEvFyRi9rsktq --- policyengine_uk/parameters/gov/economic_assumptions/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/policyengine_uk/parameters/gov/economic_assumptions/README.md b/policyengine_uk/parameters/gov/economic_assumptions/README.md index 32988cf35e..86d6a29848 100644 --- a/policyengine_uk/parameters/gov/economic_assumptions/README.md +++ b/policyengine_uk/parameters/gov/economic_assumptions/README.md @@ -525,7 +525,8 @@ the forecast levels, though not their growth. the years in the fiscal-year row over a line. It stops when hand-rolled or other tobacco does not sit under an "of which:" row directly below the non-cigarette total's row, which holds its label and figures, with only - the total's other parts between, and on any cell beside a line's label and values. When a sheet repeats a + the total's other parts between, and on any cell beside a line's label + and values. When a sheet repeats a clearance table's heading, the importer stops and names the rows. It checks no cell above a table's title or below its end, so read any note there yourself. The list holds each EFO's footnotes as the OBR wrote From 0a68b3fcf37b9e56fb7547edac24fd79bd38bfd7 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 2 Nov 2026 17:20:19 +0000 Subject: [PATCH 27/27] Pin the part check's refusals of five layouts over the tobacco parts Replace the total's-label test with one parametrized over five layouts and the three releases: a label row without figures, a text after the total's figures, a text beside "of which:" and an empty row above or below it. Make the part check's message describe the whole layout it needs. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01N2tyHRTCRBaS4PzLNYo6tN --- .../tests/test_import_obr_forecasts.py | 48 ++++++++++++++----- policyengine_uk/utils/import_obr_forecasts.py | 13 ++--- 2 files changed, 42 insertions(+), 19 deletions(-) diff --git a/policyengine_uk/tests/test_import_obr_forecasts.py b/policyengine_uk/tests/test_import_obr_forecasts.py index db3f41536a..1a703cdb26 100644 --- a/policyengine_uk/tests/test_import_obr_forecasts.py +++ b/policyengine_uk/tests/test_import_obr_forecasts.py @@ -3878,12 +3878,13 @@ def not_under_total(table, key): ``table``, a part that does not sit under its total.""" spec = CLEARANCE_SPECS[key] return ( - f"Table {table} (sheet {table}) lacks the non-cigarette total's label " - f"({spec.unit_cell!r}, as receipts_label gives it) and its figures on " - 'the row directly above the "of which:" row over the ' - f"{'.'.join(spec.path)} line. The line takes its unit from that total's " - "label, so the importer needs the label and figures there, with only " - 'the total\'s other parts between the "of which:" row and the line.' + f"Table {table} (sheet {table}) lacks the layout that gives the " + f'{".".join(spec.path)} line its unit: an "of which:" row with no other ' + "cell directly above the line and the total's other parts, and directly " + "above that row the non-cigarette total's row, which holds its label " + f"({spec.unit_cell!r}, as receipts_label gives it), its figures and " + "nothing else. The line takes its unit from that total's label, so the " + "importer refuses any other layout." ) @@ -4002,20 +4003,41 @@ def test_a_layout_that_puts_the_parts_under_another_unit_stops_the_run(name): assert str(refused.value) == expected["hand_rolling_tobacco"] +# Ways to change the rows over the parts of a real tobacco clearances table, +# each given the rows and the place of the "of which:" row. In all three +# releases, column N of the total's row and column C of the "of which:" row +# are empty, so each layout adds a text and overwrites none. +TOTAL_LAYOUTS = { + # A row that holds the total's label and none of its figures. + "label-without-figures": lambda rows, of_which: rows.insert( + of_which, {"B": "Non-cigarette clearances ('000s kg)"} + ), + # The total's row holds another text after its figures. + "text-on-total-row": lambda rows, of_which: rows[of_which - 1].update( + {"N": "Cigarette clearances (million sticks)"} + ), + # The "of which:" row holds another cell. + "text-on-of-which-row": lambda rows, of_which: rows[of_which].update( + {"C": "Cigarette clearances (million sticks)"} + ), + "empty-row-above-of-which": lambda rows, of_which: rows.insert(of_which, {}), + "empty-row-below-of-which": lambda rows, of_which: rows.insert(of_which + 1, {}), +} + + +@pytest.mark.parametrize("layout", sorted(TOTAL_LAYOUTS)) @pytest.mark.parametrize("release", ["March 2025", "March 2026", "October 2026"]) -def test_a_total_label_without_figures_does_not_count_as_the_total(release): - """Insert a row that holds only "Non-cigarette clearances ('000s kg)" - directly above the "of which:" row of the real tobacco clearances. The - row holds the total's label but none of its figures, so the importer - refuses hand-rolled and other tobacco with the part check's message. It - reads cigarettes as before.""" +def test_a_part_needs_its_total_directly_above_of_which(release, layout): + """Change the rows over the parts of the real tobacco clearances in one + way. The importer refuses hand-rolled and other tobacco with the part + check's message and reads cigarettes as before.""" _, sheet = REAL_CLEARANCE_TABLES[release, TOBACCO_CLEARANCES] real, start, end, _ = real_clearance_rows(release, TOBACCO_CLEARANCES) (of_which,) = [ i for i in range(start, end) if receipts_label(real[i].get("B")) == "of which:" ] rows = [dict(row) for row in real] - rows.insert(of_which, {"B": "Non-cigarette clearances ('000s kg)"}) + TOTAL_LAYOUTS[layout](rows, of_which) for key in TOBACCO_LINES: spec = CLEARANCE_SPECS[key] if key == "cigarettes": diff --git a/policyengine_uk/utils/import_obr_forecasts.py b/policyengine_uk/utils/import_obr_forecasts.py index e13c78ff1d..7ee8887211 100644 --- a/policyengine_uk/utils/import_obr_forecasts.py +++ b/policyengine_uk/utils/import_obr_forecasts.py @@ -1406,12 +1406,13 @@ def texts(row: dict[str, str | None]) -> list[str]: or not all(NUMBER_RE.fullmatch(text) for text in total[1:]) ): raise ValueError( - f"{where} lacks the non-cigarette total's label ({spec.unit_cell!r}, " - "as receipts_label gives it) and its figures on the row directly " - f'above the "of which:" row over the {".".join(spec.path)} line. ' - "The line takes its unit from that total's label, so the importer " - "needs the label and figures there, with only the total's other " - 'parts between the "of which:" row and the line.' + f"{where} lacks the layout that gives the {'.'.join(spec.path)} line " + 'its unit: an "of which:" row with no other cell directly above the ' + "line and the total's other parts, and directly above that row the " + f"non-cigarette total's row, which holds its label ({spec.unit_cell!r}, " + "as receipts_label gives it), its figures and nothing else. The line " + "takes its unit from that total's label, so the importer refuses any " + "other layout." )