fix(control): a site meter that repeats one reading counts as stale

.changeset/c5-frozen-site-meter-stale.md

@@ -0,0 +1,5 @@
+---
+"ftw": patch
+---
+
+Treat a site meter that repeats one reading as stale. When FTW moves its own batteries and chargers by more than 500 W in sum, the devices' own readings follow, and the meter's total and phase values still stay the same for 30 s and 3 polls after that, FTW stops dispatch, returns devices to their own control and raises no EV, as for a silent meter. From the first such repeat, battery dispatch treats the reading as held, as it does for a reading with an old timestamp: it holds the battery instead of acting on the same error again. This also holds for a meter frozen since FTW started and for a car plugged in during the freeze. A quiet site is never marked stale, nor is a meter that sends a new reading only every 10 s and loses one, nor are moves that cancel out at the meter, and the first changed reading reopens dispatch. Before, such a meter counted as fresh because each poll brought a new timestamp, and in a rig run the battery swung from +5 kW to -5 kW on 40 s of repeated data. An EV order counts only once the charger reports the car drawing it, so at a site with no battery a frozen meter is found 30 s after that, not sooner.

go/cmd/ftw/main.go

@@ -2612,7 +2612,10 @@ func main() {
 			if timeout <= 0 {
 				timeout = 60 * time.Second
 			}
-			if gate := evaluateSiteDispatchFreshnessAt(tel, meter, candidate.Fuse.MaxAmps, candidate.Fuse.Phases, timeout, time.Now()); !gate.Allowed() {
+			ctrlMu.Lock()
+			gate := siteMeterGateLocked(tel, ctrl, meter, candidate.Fuse.MaxAmps, candidate.Fuse.Phases, timeout, time.Now())
+			ctrlMu.Unlock()
+			if !gate.Allowed() {
 				return fmt.Errorf("measure before enabling control: %s", gate.Reason)
 			}
 			return nil
@@ -3050,15 +3053,11 @@ func main() {
 			// standdown/default commands remain permitted.
 			ctrlMu.Lock()
 			ctrl.ObserveSiteImport(tel)
+			freshness := siteDispatchGateLocked(tel, ctrl, watchdogTimeout, tickNow)
 			siteMeterDriver := ctrl.SiteMeterDriver
-			siteFuseAmps := ctrl.SiteFuseAmps
-			siteFusePhases := ctrl.SiteFusePhases
 			forecastIntent := forecastCurtailmentActive(ctrl, tickNow)
 			ctrlMu.Unlock()
 			forecastCurtail.ObserveIntent(forecastIntent)
-			freshness := evaluateSiteDispatchFreshnessAt(
-				tel, siteMeterDriver, siteFuseAmps, siteFusePhases, watchdogTimeout, tickNow,
-			)
 			if reason := setupRuntimeBlockReason(); reason != "" {
 				freshness.Reason = reason
 			}

go/cmd/ftw/site_dispatch_safety.go

@@ -24,6 +24,16 @@ type siteDispatchDecision struct {
 
 func (d siteDispatchDecision) Allowed() bool { return d.Reason == siteDispatchReady }
 
+// frozenMeter closes the gate when the site meter repeats one reading while
+// FTW's orders moved (control.State.ObserveSiteMeterFreeze). Such a meter
+// answers every poll, so its UpdatedAt alone looks fresh.
+func (d siteDispatchDecision) frozenMeter(frozen bool) siteDispatchDecision {
+	if frozen && d.Allowed() {
+		d.Reason = siteDispatchMeterStale
+	}
+	return d
+}
+
 // evaluateSiteDispatchFreshnessAt rejects physical dispatch when the site
 // meter itself is stale. Per-phase currents are optional until the configured
 // meter has emitted at least one of them; once available, every configured
@@ -71,11 +81,38 @@ func siteDispatchNow(
 		timeout = 60 * time.Second
 	}
 	ctrlMu.Lock()
-	siteMeter := ctrl.SiteMeterDriver
-	fuseAmps := ctrl.SiteFuseAmps
-	phases := ctrl.SiteFusePhases
-	ctrlMu.Unlock()
-	return evaluateSiteDispatchFreshnessAt(tel, siteMeter, fuseAmps, phases, timeout, now)
+	defer ctrlMu.Unlock()
+	return siteDispatchGateLocked(tel, ctrl, timeout, now)
+}
+
+// siteDispatchGateLocked is the pre-dispatch verdict the tick and
+// siteDispatchNow share: the site meter's age and phase currents, and
+// whether it repeats one reading while FTW's devices move. Call it with
+// ctrlMu held.
+func siteDispatchGateLocked(
+	tel *telemetry.Store,
+	ctrl *control.State,
+	timeout time.Duration,
+	now time.Time,
+) siteDispatchDecision {
+	return siteMeterGateLocked(tel, ctrl, ctrl.SiteMeterDriver, ctrl.SiteFuseAmps, ctrl.SiteFusePhases, timeout, now)
+}
+
+// siteMeterGateLocked is siteDispatchGateLocked for a given meter and fuse.
+// Setup uses it to judge a candidate's fuse before it enables control. Only
+// the running site meter can be frozen. Call it with ctrlMu held.
+func siteMeterGateLocked(
+	tel *telemetry.Store,
+	ctrl *control.State,
+	meter string,
+	fuseAmps float64,
+	phases int,
+	timeout time.Duration,
+	now time.Time,
+) siteDispatchDecision {
+	ctrl.ObserveSiteMeterFreeze(tel, now)
+	frozen := meter == ctrl.SiteMeterDriver && ctrl.SiteMeterFrozen()
+	return evaluateSiteDispatchFreshnessAt(tel, meter, fuseAmps, phases, timeout, now).frozenMeter(frozen)
 }
 
 // sitePhaseCurrentsAt returns the configured phases in L1/L2/L3 order.

go/cmd/ftw/site_meter_stale_test.go

@@ -92,6 +92,78 @@ func TestEvaluateSiteDispatchFreshnessAllowsMeterWithoutPhaseTelemetry(t *testin
 	}
 }
 
+// A frozen site meter (control.State.SiteMeterFrozen) closes the gate as a
+// stale meter, and leaves an already closed gate's reason alone.
+func TestFrozenSiteMeterClosesDispatchGate(t *testing.T) {
+	tel := telemetry.NewStore()
+	tel.Update("meter", telemetry.DerMeter, 2667, nil, nil)
+	open := evaluateSiteDispatchFreshnessAt(tel, "meter", 0, 3, time.Minute, time.Now())
+	if !open.Allowed() {
+		t.Fatalf("fresh meter blocked: %q", open.Reason)
+	}
+	if got := open.frozenMeter(false); !got.Allowed() {
+		t.Fatalf("meter that is not frozen blocked: %q", got.Reason)
+	}
+	if got := open.frozenMeter(true); got.Reason != siteDispatchMeterStale {
+		t.Fatalf("frozen meter reason = %q, want %q", got.Reason, siteDispatchMeterStale)
+	}
+	phases := siteDispatchDecision{Reason: siteDispatchPhaseCurrentsStale}
+	if got := phases.frozenMeter(true); got.Reason != siteDispatchPhaseCurrentsStale {
+		t.Fatalf("frozen meter replaced reason %q with %q", siteDispatchPhaseCurrentsStale, got.Reason)
+	}
+}
+
+// The gate the tick and operator setpoints share watches the meter itself:
+// one repeated sample (seed 207537) while FTW winds the battery 3 kW closes
+// it as site_meter_stale, which sends drivers to default and chargers to
+// 0 W, before the rig's EV rise at 38 s. A changed value reopens it.
+func TestSiteDispatchGateClosesOnFrozenMeter(t *testing.T) {
+	tel := telemetry.NewStore()
+	cfg := &config.Config{Site: config.Site{WatchdogTimeoutS: 120}}
+	ctrl := control.NewState(0, 50, "meter")
+	cfgMu, ctrlMu := &sync.RWMutex{}, &sync.Mutex{}
+	frozen := []byte(`{"w":2667,"l1_a":2.4,"l2_a":5.5,"l3_a":-0.2,"l1_v":230,"l2_v":230,"l3_v":230}`)
+	soc := 0.5
+	tel.Update("bat", telemetry.DerBattery, 0, &soc, nil)
+	tel.EnsureDriverHealth("meter")
+	tel.EnsureDriverHealth("bat")
+
+	start := time.Now()
+	staleAt := -1
+	for s := 0; s <= 40; s++ {
+		if s == 2 { // FTW's order, and the battery follows it
+			ctrl.PrevTargets["bat"] = -3000
+			tel.Update("bat", telemetry.DerBattery, -3000, &soc, nil)
+		}
+		tel.Update("meter", telemetry.DerMeter, 2667, nil, frozen)
+		got := siteDispatchNow(tel, cfg, cfgMu, ctrl, ctrlMu, start.Add(time.Duration(s)*time.Second))
+		if !got.Allowed() {
+			if got.Reason != siteDispatchMeterStale {
+				t.Fatalf("frozen meter reason = %q, want %q", got.Reason, siteDispatchMeterStale)
+			}
+			staleAt = s
+			break
+		}
+	}
+	if staleAt < 0 || staleAt >= 38 {
+		t.Fatalf("meter repeated one sample for 40 s while the battery moved 3 kW: stale at %d s, want before 38 s", staleAt)
+	}
+
+	// Setup's check before it enables control, with the candidate's own
+	// fuse, sees the same verdict.
+	ctrlMu.Lock()
+	setup := siteMeterGateLocked(tel, ctrl, "meter", 25, 3, 2*time.Minute, start.Add(40*time.Second))
+	ctrlMu.Unlock()
+	if setup.Reason != siteDispatchMeterStale {
+		t.Fatalf("setup gate on a frozen meter = %q, want %q", setup.Reason, siteDispatchMeterStale)
+	}
+
+	tel.Update("meter", telemetry.DerMeter, 2654, nil, nil)
+	if got := siteDispatchNow(tel, cfg, cfgMu, ctrl, ctrlMu, start.Add(41*time.Second)); !got.Allowed() {
+		t.Fatalf("meter moving again stayed blocked: %q", got.Reason)
+	}
+}
+
 func TestStaleSiteDefaultTrackerIncludesOwnerOnceAndRearmsOnRecovery(t *testing.T) {
 	var tracker staleSiteDefaultTracker
 	names := []string{"ferroamp", "sungrow", "easee"}

go/internal/control/dispatch.go

@@ -407,6 +407,9 @@ type State struct {
 	// dispatchMeterAt is the UpdatedAt of the last site-meter sample the
 	// battery dispatch acted on. See siteMeterHeld.
 	dispatchMeterAt time.Time
+	// meterFreeze finds a site meter that repeats one reading while FTW's
+	// orders move. See ObserveSiteMeterFreeze.
+	meterFreeze meterFreeze
 
 	// Slew + holdoff
 	SlewRateW float64
@@ -1459,6 +1462,8 @@ func ComputeDispatch(
 		rawGridW = siteMeter.SmoothedW
 	}
 	meterHeld := state.siteMeterHeld(siteMeter)
+	// A repeat that ignores FTW's own move is no new sample either.
+	meterHeld = meterHeld || state.siteMeterRepeatsAfterMove(siteMeter)
 	// Live EV/V2X charger readings override the manual slider on each tick —
 	// hardware truth beats guesses. Positive V2X is a vehicle charging load;
 	// negative V2X is kept separate so the default controller does not