diff --git a/.changeset/c5-frozen-site-meter-stale.md b/.changeset/c5-frozen-site-meter-stale.md new file mode 100644 index 000000000..709e96376 --- /dev/null +++ b/.changeset/c5-frozen-site-meter-stale.md @@ -0,0 +1,5 @@ +--- +"ftw": patch +--- + +Treat a site meter that repeats one reading as stale. When FTW moves its own batteries and chargers by more than 500 W in sum, the devices' own readings follow, and the meter's total and phase values still stay the same for 30 s and 3 polls after that, FTW stops dispatch, returns devices to their own control and raises no EV, as for a silent meter. From the first such repeat, battery dispatch treats the reading as held, as it does for a reading with an old timestamp: it holds the battery instead of acting on the same error again. This also holds for a meter frozen since FTW started and for a car plugged in during the freeze. A quiet site is never marked stale, nor is a meter that sends a new reading only every 10 s and loses one, nor are moves that cancel out at the meter, and the first changed reading reopens dispatch. Before, such a meter counted as fresh because each poll brought a new timestamp, and in a rig run the battery swung from +5 kW to -5 kW on 40 s of repeated data. An EV order counts only once the charger reports the car drawing it, so at a site with no battery a frozen meter is found 30 s after that, not sooner. diff --git a/go/cmd/ftw/main.go b/go/cmd/ftw/main.go index 4b23e1d63..5da8d034d 100644 --- a/go/cmd/ftw/main.go +++ b/go/cmd/ftw/main.go @@ -2612,7 +2612,10 @@ func main() { if timeout <= 0 { timeout = 60 * time.Second } - if gate := evaluateSiteDispatchFreshnessAt(tel, meter, candidate.Fuse.MaxAmps, candidate.Fuse.Phases, timeout, time.Now()); !gate.Allowed() { + ctrlMu.Lock() + gate := siteMeterGateLocked(tel, ctrl, meter, candidate.Fuse.MaxAmps, candidate.Fuse.Phases, timeout, time.Now()) + ctrlMu.Unlock() + if !gate.Allowed() { return fmt.Errorf("measure before enabling control: %s", gate.Reason) } return nil @@ -3050,15 +3053,11 @@ func main() { // standdown/default commands remain permitted. ctrlMu.Lock() ctrl.ObserveSiteImport(tel) + freshness := siteDispatchGateLocked(tel, ctrl, watchdogTimeout, tickNow) siteMeterDriver := ctrl.SiteMeterDriver - siteFuseAmps := ctrl.SiteFuseAmps - siteFusePhases := ctrl.SiteFusePhases forecastIntent := forecastCurtailmentActive(ctrl, tickNow) ctrlMu.Unlock() forecastCurtail.ObserveIntent(forecastIntent) - freshness := evaluateSiteDispatchFreshnessAt( - tel, siteMeterDriver, siteFuseAmps, siteFusePhases, watchdogTimeout, tickNow, - ) if reason := setupRuntimeBlockReason(); reason != "" { freshness.Reason = reason } diff --git a/go/cmd/ftw/site_dispatch_safety.go b/go/cmd/ftw/site_dispatch_safety.go index b65583d31..d1d815270 100644 --- a/go/cmd/ftw/site_dispatch_safety.go +++ b/go/cmd/ftw/site_dispatch_safety.go @@ -24,6 +24,16 @@ type siteDispatchDecision struct { func (d siteDispatchDecision) Allowed() bool { return d.Reason == siteDispatchReady } +// frozenMeter closes the gate when the site meter repeats one reading while +// FTW's orders moved (control.State.ObserveSiteMeterFreeze). Such a meter +// answers every poll, so its UpdatedAt alone looks fresh. +func (d siteDispatchDecision) frozenMeter(frozen bool) siteDispatchDecision { + if frozen && d.Allowed() { + d.Reason = siteDispatchMeterStale + } + return d +} + // evaluateSiteDispatchFreshnessAt rejects physical dispatch when the site // meter itself is stale. Per-phase currents are optional until the configured // meter has emitted at least one of them; once available, every configured @@ -71,11 +81,38 @@ func siteDispatchNow( timeout = 60 * time.Second } ctrlMu.Lock() - siteMeter := ctrl.SiteMeterDriver - fuseAmps := ctrl.SiteFuseAmps - phases := ctrl.SiteFusePhases - ctrlMu.Unlock() - return evaluateSiteDispatchFreshnessAt(tel, siteMeter, fuseAmps, phases, timeout, now) + defer ctrlMu.Unlock() + return siteDispatchGateLocked(tel, ctrl, timeout, now) +} + +// siteDispatchGateLocked is the pre-dispatch verdict the tick and +// siteDispatchNow share: the site meter's age and phase currents, and +// whether it repeats one reading while FTW's devices move. Call it with +// ctrlMu held. +func siteDispatchGateLocked( + tel *telemetry.Store, + ctrl *control.State, + timeout time.Duration, + now time.Time, +) siteDispatchDecision { + return siteMeterGateLocked(tel, ctrl, ctrl.SiteMeterDriver, ctrl.SiteFuseAmps, ctrl.SiteFusePhases, timeout, now) +} + +// siteMeterGateLocked is siteDispatchGateLocked for a given meter and fuse. +// Setup uses it to judge a candidate's fuse before it enables control. Only +// the running site meter can be frozen. Call it with ctrlMu held. +func siteMeterGateLocked( + tel *telemetry.Store, + ctrl *control.State, + meter string, + fuseAmps float64, + phases int, + timeout time.Duration, + now time.Time, +) siteDispatchDecision { + ctrl.ObserveSiteMeterFreeze(tel, now) + frozen := meter == ctrl.SiteMeterDriver && ctrl.SiteMeterFrozen() + return evaluateSiteDispatchFreshnessAt(tel, meter, fuseAmps, phases, timeout, now).frozenMeter(frozen) } // sitePhaseCurrentsAt returns the configured phases in L1/L2/L3 order. diff --git a/go/cmd/ftw/site_meter_stale_test.go b/go/cmd/ftw/site_meter_stale_test.go index 74e8f42a4..318b7dc6e 100644 --- a/go/cmd/ftw/site_meter_stale_test.go +++ b/go/cmd/ftw/site_meter_stale_test.go @@ -92,6 +92,78 @@ func TestEvaluateSiteDispatchFreshnessAllowsMeterWithoutPhaseTelemetry(t *testin } } +// A frozen site meter (control.State.SiteMeterFrozen) closes the gate as a +// stale meter, and leaves an already closed gate's reason alone. +func TestFrozenSiteMeterClosesDispatchGate(t *testing.T) { + tel := telemetry.NewStore() + tel.Update("meter", telemetry.DerMeter, 2667, nil, nil) + open := evaluateSiteDispatchFreshnessAt(tel, "meter", 0, 3, time.Minute, time.Now()) + if !open.Allowed() { + t.Fatalf("fresh meter blocked: %q", open.Reason) + } + if got := open.frozenMeter(false); !got.Allowed() { + t.Fatalf("meter that is not frozen blocked: %q", got.Reason) + } + if got := open.frozenMeter(true); got.Reason != siteDispatchMeterStale { + t.Fatalf("frozen meter reason = %q, want %q", got.Reason, siteDispatchMeterStale) + } + phases := siteDispatchDecision{Reason: siteDispatchPhaseCurrentsStale} + if got := phases.frozenMeter(true); got.Reason != siteDispatchPhaseCurrentsStale { + t.Fatalf("frozen meter replaced reason %q with %q", siteDispatchPhaseCurrentsStale, got.Reason) + } +} + +// The gate the tick and operator setpoints share watches the meter itself: +// one repeated sample (seed 207537) while FTW winds the battery 3 kW closes +// it as site_meter_stale, which sends drivers to default and chargers to +// 0 W, before the rig's EV rise at 38 s. A changed value reopens it. +func TestSiteDispatchGateClosesOnFrozenMeter(t *testing.T) { + tel := telemetry.NewStore() + cfg := &config.Config{Site: config.Site{WatchdogTimeoutS: 120}} + ctrl := control.NewState(0, 50, "meter") + cfgMu, ctrlMu := &sync.RWMutex{}, &sync.Mutex{} + frozen := []byte(`{"w":2667,"l1_a":2.4,"l2_a":5.5,"l3_a":-0.2,"l1_v":230,"l2_v":230,"l3_v":230}`) + soc := 0.5 + tel.Update("bat", telemetry.DerBattery, 0, &soc, nil) + tel.EnsureDriverHealth("meter") + tel.EnsureDriverHealth("bat") + + start := time.Now() + staleAt := -1 + for s := 0; s <= 40; s++ { + if s == 2 { // FTW's order, and the battery follows it + ctrl.PrevTargets["bat"] = -3000 + tel.Update("bat", telemetry.DerBattery, -3000, &soc, nil) + } + tel.Update("meter", telemetry.DerMeter, 2667, nil, frozen) + got := siteDispatchNow(tel, cfg, cfgMu, ctrl, ctrlMu, start.Add(time.Duration(s)*time.Second)) + if !got.Allowed() { + if got.Reason != siteDispatchMeterStale { + t.Fatalf("frozen meter reason = %q, want %q", got.Reason, siteDispatchMeterStale) + } + staleAt = s + break + } + } + if staleAt < 0 || staleAt >= 38 { + t.Fatalf("meter repeated one sample for 40 s while the battery moved 3 kW: stale at %d s, want before 38 s", staleAt) + } + + // Setup's check before it enables control, with the candidate's own + // fuse, sees the same verdict. + ctrlMu.Lock() + setup := siteMeterGateLocked(tel, ctrl, "meter", 25, 3, 2*time.Minute, start.Add(40*time.Second)) + ctrlMu.Unlock() + if setup.Reason != siteDispatchMeterStale { + t.Fatalf("setup gate on a frozen meter = %q, want %q", setup.Reason, siteDispatchMeterStale) + } + + tel.Update("meter", telemetry.DerMeter, 2654, nil, nil) + if got := siteDispatchNow(tel, cfg, cfgMu, ctrl, ctrlMu, start.Add(41*time.Second)); !got.Allowed() { + t.Fatalf("meter moving again stayed blocked: %q", got.Reason) + } +} + func TestStaleSiteDefaultTrackerIncludesOwnerOnceAndRearmsOnRecovery(t *testing.T) { var tracker staleSiteDefaultTracker names := []string{"ferroamp", "sungrow", "easee"} diff --git a/go/internal/control/dispatch.go b/go/internal/control/dispatch.go index f68f54885..5c3fe5b1d 100644 --- a/go/internal/control/dispatch.go +++ b/go/internal/control/dispatch.go @@ -407,6 +407,9 @@ type State struct { // dispatchMeterAt is the UpdatedAt of the last site-meter sample the // battery dispatch acted on. See siteMeterHeld. dispatchMeterAt time.Time + // meterFreeze finds a site meter that repeats one reading while FTW's + // orders move. See ObserveSiteMeterFreeze. + meterFreeze meterFreeze // Slew + holdoff SlewRateW float64 @@ -1459,6 +1462,8 @@ func ComputeDispatch( rawGridW = siteMeter.SmoothedW } meterHeld := state.siteMeterHeld(siteMeter) + // A repeat that ignores FTW's own move is no new sample either. + meterHeld = meterHeld || state.siteMeterRepeatsAfterMove(siteMeter) // Live EV/V2X charger readings override the manual slider on each tick — // hardware truth beats guesses. Positive V2X is a vehicle charging load; // negative V2X is kept separate so the default controller does not diff --git a/go/internal/control/meter_freeze.go b/go/internal/control/meter_freeze.go new file mode 100644 index 000000000..913f7a5da --- /dev/null +++ b/go/internal/control/meter_freeze.go @@ -0,0 +1,211 @@ +package control + +import ( + "encoding/json" + "log/slog" + "math" + "time" + + "github.com/srcfl/ftw/go/internal/telemetry" +) + +// A site meter can answer every poll with the same numbers. Each answer is +// a new sample, so every check that keys on UpdatedAt calls it fresh. In rig +// seed 207537 a meter repeated one sample for 39.9 s; dispatch wound the +// battery from +4 kW to -5 kW on it and raised an EV. +// +// A live meter must show a large move of FTW's own devices. When FTW's +// orders move, the devices' own readings follow, and the meter still repeats +// the same total and phase values long after that, the meter is frozen and +// counts as stale. An order a device does not follow (a full battery, a car +// that does not draw, a charger that has not applied it yet) does not count, +// and moves that cancel each other out do not count, since the meter sees +// only their sum. +const ( + // meterFreezeMoveW is how far FTW's devices must move in sum, by order + // and by their own readings, from where they stood when the meter last + // changed. Site meters resolve the total to 1-10 W and phase currents + // to 0.01-1 A (1 A is 230 W). Batteries follow an order to within tens + // of watts, and inverters ignore steps below about 100-200 W. 500 W is + // 50 times the coarsest total step and above every inverter dead band. + // Each EV step is at least 1380 W. + meterFreezeMoveW = 500.0 + // meterFreezeAfter is how long a live meter may take to show that move. + // Swedish HAN and DSMR 4 meters send a reading every 10 s, so a driver + // polling every second sees up to 10 identical samples by design. A + // telegram lost to a CRC error leaves the last values in place for 20 s, + // and FTW makes its large moves right after a telegram that shows a load + // step. 30 s is three telegram periods, so one lost telegram cannot trip + // it. Seed 207537's freeze is still caught before its EV rise at 38 s. + meterFreezeAfter = 30 * time.Second + // meterFreezeSamples is how many identical samples after the move the + // verdict needs, so a slow poller is judged on several answers. + meterFreezeSamples = 3 +) + +// meterSample is the total and per-phase values a site meter reports. +// Absent fields compare as absent. +type meterSample struct { + vals [10]float64 + has [10]bool +} + +type meterFreeze struct { + driver string + at time.Time // UpdatedAt of the last sample seen + sample meterSample + // devices are FTW's devices when the sample last changed. + devices map[ownDeviceKey]ownDevice + movedAt time.Time // when the devices first moved past meterFreezeMoveW + repeats int // identical samples since movedAt + // repeatAfterMove: the sample at `at` repeats the values from before + // the devices moved past meterFreezeMoveW. See siteMeterRepeatsAfterMove. + repeatAfterMove bool + frozen bool +} + +// ObserveSiteMeterFreeze compares the site meter's newest sample with the +// one before it, and FTW's own devices with where they stood when the meter +// last changed. Call it every control tick under the control lock, before +// the dispatch gate reads SiteMeterFrozen. A quiet site whose devices do not +// move is never frozen, and any change in the meter's values clears it. +func (s *State) ObserveSiteMeterFreeze(store *telemetry.Store, now time.Time) { + if s == nil || store == nil { + return + } + f := &s.meterFreeze + w, data, at, ok := store.OnlineMeterSample(s.SiteMeterDriver) + if !ok || f.driver != s.SiteMeterDriver { + // An offline meter is stale on its own; start over once it returns. + *f = meterFreeze{driver: s.SiteMeterDriver} + if !ok { + return + } + } + // No new sample: siteMeterHeld and the watchdog handle that. + if !at.After(f.at) { + return + } + sample := readMeterSample(w, data) + devices := s.ownDevices(store) + if f.at.IsZero() || sample != f.sample { + if f.frozen { + slog.Info("site meter values move again", "site_meter", f.driver) + } + *f = meterFreeze{driver: f.driver, at: at, sample: sample, devices: devices} + return + } + f.at = at + if f.frozen { + return // only a change in the meter's values clears it + } + moved := devicesMovedW(f.devices, devices) + if math.Abs(moved) <= meterFreezeMoveW { + // Not moved, or moved back: a live meter may repeat itself here. + f.movedAt, f.repeats, f.repeatAfterMove = time.Time{}, 0, false + return + } + f.repeatAfterMove = true + if f.movedAt.IsZero() { + f.movedAt = now + return + } + f.repeats++ + if f.repeats >= meterFreezeSamples && now.Sub(f.movedAt) >= meterFreezeAfter { + f.frozen = true + slog.Warn("site meter repeats one reading while FTW's devices moved; treating it as stale", + "site_meter", f.driver, "devices_moved_w", math.Round(moved), + "since", now.Sub(f.movedAt).Round(time.Second), "samples", f.repeats) + } +} + +// SiteMeterFrozen reports whether ObserveSiteMeterFreeze found the site +// meter repeating one reading while FTW's devices moved. +func (s *State) SiteMeterFrozen() bool { + return s != nil && s.meterFreeze.frozen +} + +// siteMeterRepeatsAfterMove reports whether meter is the sample +// ObserveSiteMeterFreeze judged last, and it repeats the values from before +// FTW's devices moved by more than meterFreezeMoveW. A live meter shows +// such a move, so the repeat carries no new error. Battery dispatch treats +// it as a held sample (siteMeterHeld) from the first repeat, not only once +// the meter counts as stale. On a meter that sends a reading every 10 s, +// this waits for the next reading instead of acting on the old one again. +func (s *State) siteMeterRepeatsAfterMove(meter *telemetry.DerReading) bool { + f := &s.meterFreeze + return meter != nil && f.driver == s.SiteMeterDriver && f.repeatAfterMove && meter.UpdatedAt.Equal(f.at) +} + +type ownDeviceKey struct { + t telemetry.DerType + name string +} + +// ownDevice is one battery or charger: FTW's last order to it and its own +// latest reading. +type ownDevice struct { + orderW, readW float64 +} + +// ownDevices are every battery and charger that reports a reading. One FTW +// has no order for counts as ordered to 0 W: a battery before FTW's first +// order after a start, a charger with no offer (unplugged, or its offer not +// yet known). So a meter frozen from the start, or a car plugged in during +// a freeze, is judged like any other move. +func (s *State) ownDevices(store *telemetry.Store) map[ownDeviceKey]ownDevice { + out := make(map[ownDeviceKey]ownDevice) + for driver, w := range store.OnlineRawWByType(telemetry.DerBattery) { + out[ownDeviceKey{telemetry.DerBattery, driver}] = ownDevice{orderW: s.PrevTargets[driver], readW: w} + } + for driver, w := range store.OnlineRawWByType(telemetry.DerEV) { + out[ownDeviceKey{telemetry.DerEV, driver}] = ownDevice{orderW: s.evCommanded[driver], readW: w} + } + return out +} + +// devicesMovedW is the net power FTW moved its devices by between a and b, +// in the site's sign, which is what the meter should show. Each device +// counts the smaller of its order's move and its reading's move, in the +// reading's direction, so only an order the device followed counts. A +// device that moved against its order, or was absent from a, counts 0. +func devicesMovedW(a, b map[ownDeviceKey]ownDevice) float64 { + moved := 0.0 + for k, now := range b { + then, ok := a[k] + if !ok { + continue + } + order, read := now.orderW-then.orderW, now.readW-then.readW + if order*read <= 0 { + continue + } + moved += math.Copysign(min(math.Abs(order), math.Abs(read)), read) + } + return moved +} + +func readMeterSample(w float64, data json.RawMessage) meterSample { + var d struct { + L1A *float64 `json:"l1_a"` + L2A *float64 `json:"l2_a"` + L3A *float64 `json:"l3_a"` + L1W *float64 `json:"l1_w"` + L2W *float64 `json:"l2_w"` + L3W *float64 `json:"l3_w"` + L1V *float64 `json:"l1_v"` + L2V *float64 `json:"l2_v"` + L3V *float64 `json:"l3_v"` + } + if len(data) > 0 { + _ = json.Unmarshal(data, &d) + } + var out meterSample + out.vals[0], out.has[0] = w, true + for i, p := range [...]*float64{d.L1A, d.L2A, d.L3A, d.L1W, d.L2W, d.L3W, d.L1V, d.L2V, d.L3V} { + if p != nil { + out.vals[i+1], out.has[i+1] = *p, true + } + } + return out +} diff --git a/go/internal/control/meter_freeze_test.go b/go/internal/control/meter_freeze_test.go new file mode 100644 index 000000000..6ed8a0ca4 --- /dev/null +++ b/go/internal/control/meter_freeze_test.go @@ -0,0 +1,448 @@ +package control + +import ( + "encoding/json" + "math" + "testing" + "time" + + "github.com/srcfl/ftw/go/internal/telemetry" +) + +// frozenRigMeter is the sample the rig's site meter repeated in seed 207537. +var frozenRigMeter = json.RawMessage(`{"w":2667,"l1_a":2.4,"l2_a":5.5,"l3_a":-0.2,` + + `"l1_w":552,"l2_w":1265,"l3_w":-46,"l1_v":230,"l2_v":230,"l3_v":230}`) + +func freezeState(clk *time.Time) *State { + st := NewState(0, 50, "meter") + st.clock = func() time.Time { return *clk } + return st +} + +// setBattery sends an order and lets the battery follow it. +func setBattery(st *State, store *telemetry.Store, w float64) { + st.PrevTargets["bat"] = w + store.Update("bat", telemetry.DerBattery, w, ptrF64(0.5), nil) +} + +// freezeStore is a store whose test drivers all report: the check reads +// telemetry through health-gated accessors. +func freezeStore() *telemetry.Store { + store := telemetry.NewStore() + for _, name := range []string{"meter", "bat", "own", "charger", "a", "b"} { + store.EnsureDriverHealth(name) + } + return store +} + +// Seed 207537: the meter answers every poll with one sample while the +// battery PI winds on it. The freeze must count as stale well before the +// rig's EV rise went out (38 s into the freeze), and the battery must stop +// moving once it does. +func TestFrozenMeterWithMovingOrdersIsStale(t *testing.T) { + store := freezeStore() + soc := 0.5 + store.Update("bat", telemetry.DerBattery, 4075, &soc, nil) + store.DriverHealthMut("bat").RecordSuccess() + + clk := time.Now() + st := freezeState(&clk) + st.Mode = ModeSelfConsumption + st.MinDispatchIntervalS = 0 + st.DriverLimits = map[string]PowerLimits{"bat": {MaxChargeW: 5000, MaxDischargeW: 5000}} + st.PrevTargets["bat"] = 4075 + capsMap := caps(map[string]float64{"bat": 10000}) + + frozenAfter := time.Duration(-1) + minW, maxW := 4075.0, 4075.0 + ordered := 4075.0 + start := clk + for clk.Sub(start) < 38*time.Second { + // The rig's battery follows an order about 2 s later: one tick. + store.Update("bat", telemetry.DerBattery, ordered, &soc, nil) + store.Update("meter", telemetry.DerMeter, 2667, nil, frozenRigMeter) + store.DriverHealthMut("meter").RecordSuccess() + st.ObserveSiteMeterFreeze(store, clk) + if st.SiteMeterFrozen() { + if frozenAfter < 0 { + frozenAfter = clk.Sub(start) + } + } else { + // The gate is open: dispatch acts on the frozen meter. + for _, tg := range ComputeDispatch(store, st, capsMap, 3*16*230) { + if tg.Driver == "bat" { + ordered = tg.TargetW + minW, maxW = min(minW, tg.TargetW), max(maxW, tg.TargetW) + } + } + } + clk = clk.Add(2 * time.Second) + } + if frozenAfter < 0 { + t.Fatalf("meter repeated one sample for 38 s while battery orders moved %.0f..%.0f W; not stale", minW, maxW) + } + if frozenAfter >= 38*time.Second { + t.Fatalf("stale only after %v; the rig's EV rise went out at 38 s", frozenAfter) + } + t.Logf("stale after %v; battery orders %.0f..%.0f W", frozenAfter, minW, maxW) + // In the rig the battery swung from +5 kW to -5 kW before the verdict. + // Once the battery has moved past meterFreezeMoveW and the meter still + // repeats itself, dispatch holds it. + if maxW-minW > 2500 { + t.Fatalf("battery orders swung %.0f..%.0f W on a repeated sample before the verdict", minW, maxW) + } + + // Any change in the meter's values ends the freeze. + store.Update("meter", telemetry.DerMeter, 2654, nil, nil) + st.ObserveSiteMeterFreeze(store, clk) + if st.SiteMeterFrozen() { + t.Fatal("a moving meter stayed stale") + } +} + +// Holding the battery on a repeat must not stall a live meter's loop. A +// 10 s HAN meter, polled on every 2 s tick, and the battery following an +// order one tick later: dispatch reaches the import target, does not +// overshoot into export by acting on one reading five times, and the meter +// never counts as stale. A noise-free meter that sends a new reading on +// every tick does the same. +func TestFrozenMeterHoldLetsLiveMeterLoopSettle(t *testing.T) { + // The 2 s case overshoots about 1.4 kW with or without the hold: the PI + // acting on a meter one tick late. Only the 10 s case is bounded here. + for _, tc := range []struct { + telegram time.Duration + maxExport float64 + }{{2 * time.Second, math.Inf(1)}, {10 * time.Second, 1000}} { + telegram := tc.telegram + store := freezeStore() + soc := 0.5 + clk := time.Now() + st := freezeState(&clk) + st.Mode = ModeSelfConsumption + st.MinDispatchIntervalS = 0 + st.DriverLimits = map[string]PowerLimits{"bat": {MaxChargeW: 5000, MaxDischargeW: 5000}} + capsMap := caps(map[string]float64{"bat": 10000}) + start := clk + houseW, ordered, batW, gridW := 3000.0, 0.0, 0.0, 3000.0 + for clk.Sub(start) < 3*time.Minute { + if clk.Sub(start)%telegram == 0 { + gridW = houseW + batW // the meter shows the battery as it is now + if -gridW > tc.maxExport { + t.Fatalf("%v telegrams: the battery overshot to %.0f W export at %v", telegram, -gridW, clk.Sub(start)) + } + } + batW = ordered // the battery takes the last order + store.Update("bat", telemetry.DerBattery, batW, &soc, nil) + store.Update("meter", telemetry.DerMeter, gridW, nil, nil) + st.ObserveSiteMeterFreeze(store, clk) + if st.SiteMeterFrozen() { + t.Fatalf("%v telegrams: live meter marked stale at %v", telegram, clk.Sub(start)) + } + for _, tg := range ComputeDispatch(store, st, capsMap, 3*16*230) { + if tg.Driver == "bat" { + ordered = tg.TargetW + } + } + clk = clk.Add(2 * time.Second) + } + if err := houseW + batW; err > 300 || err < -300 { + t.Errorf("%v telegrams: the site still imports %.0f W after 3 min (battery %.0f W)", telegram, err, batW) + } + } +} + +// A quiet site whose devices do not move keeps a meter that repeats itself. +func TestFrozenMeterQuietSiteStaysFresh(t *testing.T) { + store := freezeStore() + clk := time.Now() + st := freezeState(&clk) + for i := 0; i < 150; i++ { // 5 min at 2 s + setBattery(st, store, -300+float64(i%2)*400) // small PI moves under the threshold + store.Update("meter", telemetry.DerMeter, 12, nil, frozenRigMeter) + st.ObserveSiteMeterFreeze(store, clk) + if st.SiteMeterFrozen() { + t.Fatalf("quiet site marked stale after %d identical samples", i+1) + } + clk = clk.Add(2 * time.Second) + } +} + +// A HAN or DSMR 4 meter sends a new reading every 10 s; a driver polling it +// every second repeats each reading ten times. Large battery moves inside +// that window must not mark it stale, or it would flap. +func TestFrozenMeterTenSecondTelegramsDoNotFlap(t *testing.T) { + store := freezeStore() + clk := time.Now() + st := freezeState(&clk) + for s := 0; s < 300; s++ { + setBattery(st, store, float64((s%4)-2)*2000) // the battery swings +-4 kW + gridW := 800 + float64((s/10)%7) // one new value per 10 s telegram + store.Update("meter", telemetry.DerMeter, gridW, nil, nil) + st.ObserveSiteMeterFreeze(store, clk) + if st.SiteMeterFrozen() { + t.Fatalf("10 s telegram meter marked stale at %d s", s) + } + clk = clk.Add(time.Second) + } +} + +// Orders a device does not follow do not count: a car that does not draw a +// raised offer (a slow charger, or a full car), a full battery that stays +// at 0 W, and a battery that charges on its own against an order to +// discharge. The rig's meter has no noise, so it repeats itself here. +func TestFrozenMeterOrdersNotFollowedDoNotCount(t *testing.T) { + store := freezeStore() + clk := time.Now() + st := freezeState(&clk) + st.PrevTargets["bat"] = 0 + store.Update("bat", telemetry.DerBattery, 0, ptrF64(1), nil) + store.Update("own", telemetry.DerBattery, 0, ptrF64(0.1), nil) + store.Update("charger", telemetry.DerEV, 0, nil, nil) + for i := 0; i < 60; i++ { // 2 min at 2 s + st.PrevTargets["bat"] = 3000 * float64(i%2) + st.evCommanded = map[string]float64{"charger": 11040} + if i == 1 { + st.PrevTargets["own"] = -3000 + store.Update("own", telemetry.DerBattery, 1000, ptrF64(0.1), nil) + } + store.Update("meter", telemetry.DerMeter, -2332, nil, nil) + st.ObserveSiteMeterFreeze(store, clk) + if st.SiteMeterFrozen() { + t.Fatalf("orders the devices ignored marked the meter stale after %d samples", i+1) + } + clk = clk.Add(2 * time.Second) + } +} + +// The verdict waits for the slowest legitimate refresh after the move, and +// for several samples, before it calls the meter stale. +func TestFrozenMeterWaitsForTheMoveToLand(t *testing.T) { + store := freezeStore() + clk := time.Now() + st := freezeState(&clk) + poll := func() bool { + store.Update("meter", telemetry.DerMeter, 500, nil, nil) + st.ObserveSiteMeterFreeze(store, clk) + return st.SiteMeterFrozen() + } + setBattery(st, store, 0) + poll() + setBattery(st, store, -2000) + poll() // the move is seen here + moved := clk + for clk.Sub(moved) < meterFreezeAfter-time.Second { + clk = clk.Add(time.Second) + if poll() { + t.Fatalf("stale %v after the move", clk.Sub(moved)) + } + } + clk = clk.Add(time.Second) + if !poll() { + t.Fatalf("not stale %v after a 2 kW move", clk.Sub(moved)) + } +} + +// A 10 s HAN or DSMR 4 meter behind a bridge polled every second that loses +// one telegram (a CRC error) serves the old values for 20 s. A battery move +// right after the last good telegram must not mark this live meter stale. +func TestFrozenMeterLostTelegramDoesNotTrip(t *testing.T) { + store := freezeStore() + clk := time.Now() + st := freezeState(&clk) + tripped := -1 + gridW := 800.0 + for s := 0; s < 60; s++ { + // the battery steps 1 kW further one second after each telegram + if s%10 == 1 { + setBattery(st, store, -float64(s/10)*1000) + } + // telegrams every 10 s, except the one at s=20 is lost + if s%10 == 0 && s != 20 { + gridW++ + } + store.Update("meter", telemetry.DerMeter, gridW, nil, nil) + st.ObserveSiteMeterFreeze(store, clk) + if st.SiteMeterFrozen() && tripped < 0 { + tripped = s + } + clk = clk.Add(time.Second) + } + if tripped >= 0 { + t.Errorf("live 10 s meter with one lost telegram marked stale at %d s", tripped) + } +} + +// Two batteries trade 1.5 kW each way. The meter sees only the sum, which +// does not change, so the repeat is no evidence. +func TestFrozenMeterOppositeMovesCancel(t *testing.T) { + store := freezeStore() + clk := time.Now() + st := freezeState(&clk) + set := func(name string, w float64) { + st.PrevTargets[name] = w + store.Update(name, telemetry.DerBattery, w, ptrF64(0.5), nil) + } + set("a", 2000) + set("b", 0) + for s := 0; s < 60; s++ { + if s == 2 { + set("a", 500) + set("b", 1500) + } + store.Update("meter", telemetry.DerMeter, 300, nil, nil) // net unchanged + st.ObserveSiteMeterFreeze(store, clk) + if st.SiteMeterFrozen() { + t.Errorf("opposite moves that cancel at the meter marked it stale at %d s", s) + return + } + clk = clk.Add(time.Second) + } +} + +// A car plugged in during a freeze, at a site with no battery: its charger +// had no offer when the meter last changed. FTW's raise, which the car +// draws, must still count, or an EV rise on a frozen meter goes unseen. +func TestFrozenMeterNewlyPluggedEVCounts(t *testing.T) { + store := freezeStore() + clk := time.Now() + st := freezeState(&clk) + store.Update("charger", telemetry.DerEV, 0, nil, nil) + for s := 0; s < 60; s++ { + switch { + case s == 5: // car plugged: first offer, 0 W + st.evCommanded = map[string]float64{"charger": 0} + case s >= 10: // FTW raises to 6 A x 3 and the car draws it + st.evCommanded = map[string]float64{"charger": 4140} + store.Update("charger", telemetry.DerEV, 4140, nil, nil) + } + store.Update("meter", telemetry.DerMeter, 2667, nil, frozenRigMeter) + st.ObserveSiteMeterFreeze(store, clk) + if st.SiteMeterFrozen() { + return + } + clk = clk.Add(time.Second) + } + t.Error("frozen meter while FTW raised a newly plugged EV 4.1 kW for 50 s: never stale") +} + +// A battery that moves 2 kW and back before the meter's next telegram +// leaves nothing for the meter to show, so a repeat is no evidence. +func TestFrozenMeterMoveAndBackDoesNotCount(t *testing.T) { + store := freezeStore() + clk := time.Now() + st := freezeState(&clk) + setBattery(st, store, 0) + for s := 0; s < 90; s++ { + switch s { + case 1: + setBattery(st, store, 2000) + case 5: + setBattery(st, store, 0) + } + store.Update("meter", telemetry.DerMeter, 300, nil, nil) + st.ObserveSiteMeterFreeze(store, clk) + if st.SiteMeterFrozen() { + t.Fatalf("a move that came back marked the meter stale at %d s", s) + } + clk = clk.Add(time.Second) + } +} + +// A meter already frozen when FTW starts: FTW has no battery orders yet +// when the meter's only sample is taken. Winding the battery 3 kW on it must +// still count. +func TestFrozenMeterFromStartIsCaught(t *testing.T) { + store := freezeStore() + clk := time.Now() + st := freezeState(&clk) // PrevTargets empty, as after a restart + store.Update("bat", telemetry.DerBattery, 0, ptrF64(0.5), nil) + for s := 0; s < 60; s++ { + if s >= 3 { + setBattery(st, store, -3000) + } + store.Update("meter", telemetry.DerMeter, 2667, nil, frozenRigMeter) + st.ObserveSiteMeterFreeze(store, clk) + if st.SiteMeterFrozen() { + return + } + clk = clk.Add(time.Second) + } + t.Error("meter frozen from boot while FTW wound the battery to -3 kW: never stale") +} + +// After a start, FTW's first order to a battery that already runs on its +// own counts only by how far the battery's reading moves. Ordering what it +// already does is no move. +func TestFrozenMeterFirstOrderMatchingAutonomousBatteryDoesNotCount(t *testing.T) { + store := freezeStore() + clk := time.Now() + st := freezeState(&clk) + store.Update("bat", telemetry.DerBattery, 1500, ptrF64(0.5), nil) + for s := 0; s < 60; s++ { + if s >= 3 { + setBattery(st, store, 1500) + } + store.Update("meter", telemetry.DerMeter, 2667, nil, frozenRigMeter) + st.ObserveSiteMeterFreeze(store, clk) + if st.SiteMeterFrozen() { + t.Fatalf("first order matching the battery's own 1500 W marked the meter stale at %d s", s) + } + clk = clk.Add(time.Second) + } +} + +// A slow poller (one answer every 20 s) after a 3 kW move: the verdict +// needs three repeated answers after the move, not just 30 s. +func TestFrozenMeterSlowPollerNeedsThreeRepeats(t *testing.T) { + store := freezeStore() + clk := time.Now() + st := freezeState(&clk) + start := clk + poll := func(s int) bool { + store.Update("meter", telemetry.DerMeter, 900, nil, nil) + st.ObserveSiteMeterFreeze(store, start.Add(time.Duration(s)*time.Second)) + return st.SiteMeterFrozen() + } + setBattery(st, store, 0) + poll(0) + setBattery(st, store, -3000) + poll(20) // the move is seen + if poll(40) { + t.Fatal("stale after one repeat") + } + if poll(60) { + t.Fatal("stale after two repeats, 40 s after the move") + } + if !poll(80) { + t.Fatal("not stale after three repeats, 60 s after a 3 kW move") + } +} + +// The rig's OCPP charger accepts a new limit at once, and the car draws it +// about 30 s later (seed 207537: offer at 157 s, draw at 188 s). A live +// meter with no noise repeats itself until then. Counting the accepted +// offer as a move would mark this meter stale, so an EV order counts only +// once the charger's reading shows the car drawing it. +func TestFrozenMeterAcceptedOfferWaitsForTheCar(t *testing.T) { + store := freezeStore() + clk := time.Now() + st := freezeState(&clk) + store.Update("charger", telemetry.DerEV, 0, nil, nil) + st.evCommanded = map[string]float64{"charger": 0} + gridW := 208.0 + for s := 0; s < 120; s++ { + if s == 1 { // FTW raises to 6 A x 3; the charger accepts at once + st.evCommanded = map[string]float64{"charger": 4140} + } + if s == 32 { // the car draws; the meter and the charger show it + gridW += 4140 + store.Update("charger", telemetry.DerEV, 4140, nil, nil) + } + store.Update("meter", telemetry.DerMeter, gridW, nil, nil) + st.ObserveSiteMeterFreeze(store, clk) + if st.SiteMeterFrozen() { + t.Fatalf("live meter marked stale at %d s; the car started 31 s after the offer", s) + } + clk = clk.Add(time.Second) + } +} diff --git a/go/internal/telemetry/store.go b/go/internal/telemetry/store.go index 32e668501..8fce890b6 100644 --- a/go/internal/telemetry/store.go +++ b/go/internal/telemetry/store.go @@ -711,6 +711,38 @@ func (s *Store) OnlineRawW(driver string, t DerType) (w float64, at time.Time, o return r.RawW, r.UpdatedAt, true } +// OnlineMeterSample is OnlineMeterW with the reading's data, which holds +// the phase values. +func (s *Store) OnlineMeterSample(driver string) (w float64, data json.RawMessage, at time.Time, ok bool) { + s.mu.RLock() + defer s.mu.RUnlock() + r, found := s.readings[key(driver, DerMeter)] + if !found || math.IsNaN(r.RawW) || math.IsInf(r.RawW, 0) { + return 0, nil, time.Time{}, false + } + if h, known := s.health[driver]; !known || !h.TelemetryLive() { + return 0, nil, time.Time{}, false + } + return r.RawW, r.Data, r.UpdatedAt, true +} + +// OnlineRawWByType returns the latest raw power of every driver of one type +// that is still reporting. A watchdog-offline driver is left out. +func (s *Store) OnlineRawWByType(t DerType) map[string]float64 { + s.mu.RLock() + defer s.mu.RUnlock() + out := make(map[string]float64) + for _, r := range s.readings { + if r.DerType != t || math.IsNaN(r.RawW) || math.IsInf(r.RawW, 0) { + continue + } + if h, known := s.health[r.Driver]; known && h.TelemetryLive() { + out[r.Driver] = r.RawW + } + } + return out +} + // ReadingsByDriver returns all readings from one driver. func (s *Store) ReadingsByDriver(driver string) []*DerReading { s.mu.RLock() diff --git a/go/internal/telemetry/telemetry_test.go b/go/internal/telemetry/telemetry_test.go index 2ac3d4412..f00586da3 100644 --- a/go/internal/telemetry/telemetry_test.go +++ b/go/internal/telemetry/telemetry_test.go @@ -723,3 +723,39 @@ func TestOnlineRawWSkipsOfflineBattery(t *testing.T) { t.Fatal("an offline battery's last reading was returned") } } + +// The meter's sample comes with its phase data, and both stop once the +// meter goes offline. +func TestOnlineMeterSampleSkipsOfflineMeter(t *testing.T) { + s := NewStore() + data := json.RawMessage(`{"l1_a":2.4}`) + s.Update("meter", DerMeter, 2667, nil, data) + if _, _, _, ok := s.OnlineMeterSample("meter"); ok { + t.Fatal("a meter with no health record was returned") + } + s.DriverHealthMut("meter").RecordSuccess() + w, got, at, ok := s.OnlineMeterSample("meter") + if !ok || w != 2667 || string(got) != string(data) || at.IsZero() { + t.Fatalf("online meter: %v %s %v %v", w, got, at, ok) + } + s.DriverHealthMut("meter").SetOffline() + if _, _, _, ok := s.OnlineMeterSample("meter"); ok { + t.Fatal("an offline meter's last sample was returned") + } +} + +// Every reporting driver of one type, and none that went offline. +func TestOnlineRawWByTypeSkipsOfflineDrivers(t *testing.T) { + s := NewStore() + for name, w := range map[string]float64{"a": -3000, "b": 1500} { + s.Update(name, DerBattery, w, nil, nil) + s.DriverHealthMut(name).RecordSuccess() + } + s.Update("charger", DerEV, 4140, nil, nil) + s.DriverHealthMut("charger").RecordSuccess() + s.DriverHealthMut("b").SetOffline() + got := s.OnlineRawWByType(DerBattery) + if len(got) != 1 || got["a"] != -3000 { + t.Fatalf("online batteries = %v, want only a at -3000 W", got) + } +}