From a2fea1067828006b3b4374a62700236ede389a12 Mon Sep 17 00:00:00 2001 From: Michael Holzmann Date: Sat, 14 Nov 2026 14:54:35 +0000 Subject: [PATCH] chore: harden shared state, parser bounds and build pinning - Guard the AES output buffer against oversized advertisements - Protect the cross-core MpptData hand-off with a critical section and publish from a consistent snapshot - Scan in 30s blocks and clear results between them: NimBLE 1.4.x only frees its result list when a scan completes, and one entry is kept per unique BLE address (including MAC-randomising phones) - Do not let the forced-reconnect watchdog reset the reboot watchdog - Publish discovery immediately after a reconnect instead of waiting for the retry timer - Pin espressif32 to 6.13.0 (Arduino core 2.0.17), the last platform compatible with NimBLE-Arduino 1.4.x - Document diagnostics, watchdogs and the data_age_s field --- README.md | 49 ++++++++++++++++++++++-- include/example.config.h | 4 ++ platformio.ini | 4 +- src/main.cpp | 82 ++++++++++++++++++++++++++-------------- src/victron_ble.cpp | 5 +++ 5 files changed, 111 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 30cfa4e..0f1e942 100644 --- a/README.md +++ b/README.md @@ -37,9 +37,10 @@ better in Home Assistant while it's there. |---|---| | ๐Ÿ“ถ **Reads the advertisement** | Passive BLE scan of the SmartSolar MPPT's broadcasts โ€” no pairing, no connection, lower draw. | | ๐Ÿ” **Real decryption** | AES-128-CTR exactly as the Victron BLE protocol specifies, matching the reference Python implementation bit for bit. | -| ๐Ÿ“ก **MQTT every 10s** | One JSON payload with every value, plus a retained online/offline LWT topic. | +| ๐Ÿ“ก **MQTT every 10s** | One JSON payload with every value, plus a retained online/offline LWT topic and a `diag` topic with uptime, RSSI, heap and reconnect counters. | | ๐Ÿ  **Auto-discovery** | Appears in Home Assistant as a **Victron SmartSolar MPPT** device with entities grouped. No YAML. | -| ๐Ÿงต **Two cores, no crashes** | BLE scanning pinned to core 1, WiFi/MQTT to core 0 via FreeRTOS โ€” sidesteps the radio coexistence crashes that come from sharing a core. | +| ๐Ÿงต **Two cores** | BLE scanning pinned to core 1, WiFi/MQTT to core 0 via FreeRTOS. Coexistence is handled by a 50% BLE scan duty cycle, not by the core split. | +| ๐Ÿ” **Self-healing** | Non-blocking reconnects, forced MQTT reconnect after 5 minutes without a successful publish, and a reboot after 60 minutes | | ๐Ÿ”Œ **External load too** | The MPPT's load output current is published as amps, and multiplied by battery voltage to give watts โ€” so the dashboard shows draw as well as charge. | | ๐Ÿ“Š **Dashboard included** | A ready-made Mushroom + Mini Graph dashboard card and optional template sensors. | | ๐Ÿชซ **0.15W idle** | The entire point. | @@ -154,7 +155,28 @@ cp include/example.config.h include/config.h "charge_state_raw": 3, "error_code": 0, "external_load_a": "0.35", - "external_load_w": "4.5" + "external_load_w": "4.5", + "data_age_s": 2 +} +``` + +`data_age_s` is the age of the last decoded BLE packet. If it keeps growing while +`solar/mppt/diag` still updates, MQTT and WiFi are fine and the problem is BLE reception. + +```json +{ + "uptime_s": 86400, + "wifi_status": 3, + "wifi_rssi": -61, + "free_heap": 182344, + "min_free_heap": 140112, + "wifi_disconnects": 0, + "mqtt_connects": 1, + "publish_failures": 0, + "ble_valid": 14321, + "ble_ignored": 0, + "ble_rssi": -72, + "ble_age_s": 2 } ``` @@ -173,6 +195,27 @@ cp include/example.config.h include/config.h | 252 | External control | +## Troubleshooting + +Start by subscribing to the `solar/mppt/diag` topic: + +| Observation | Likely cause | +|---|---| +| `diag` updates, `ble_age_s`/`data_age_s` growing | MQTT and WiFi are fine; the MPPT advertisement is not being received (range, MPPT asleep, VictronConnect connected to it, BLE scan failed) | +| `diag` stops, `status` still `online` | WiFi or broker path; watch the serial log for `[WiFi] Disconnected, reason=` and `[MQTT] Connection lost` | +| `publish_failures` climbing | Socket/broker rejects; check broker logs for duplicate `MQTT_CLIENT_ID` | +| `free_heap` low or `min_free_heap` dropping each boot | Memory pressure; shorten the diag interval or reduce scanning | + +Note that retained `online` on `solar/mppt/status` only means the broker has not yet +noticed the TCP session is gone; it does not prove the ESP32 is still publishing. + +The firmware includes two safety nets (both defined in `src/main.cpp`): + +- **5 minutes** without a successful publish โ†’ drop and rebuild the MQTT connection. +- **60 minutes** without any successful publish โ†’ `ESP.restart()`. + +--- + ## Home Assistant Once the ESP32 is running, go to **Settings โ†’ Devices & Services โ†’ MQTT**. A **Victron diff --git a/include/example.config.h b/include/example.config.h index f6273dc..8094ce6 100644 --- a/include/example.config.h +++ b/include/example.config.h @@ -36,5 +36,9 @@ #define BLE_SCAN_INTERVAL_MS 100 #define BLE_SCAN_WINDOW_MS 50 +// Scan in blocks; NimBLE 1.4.x frees its result list only when a scan ends. +#define BLE_SCAN_PERIOD_S 30 +#define BLE_SCAN_PAUSE_MS 1000 + // How often to publish the diagnostics payload (milliseconds) #define DIAG_INTERVAL_MS 60000 diff --git a/platformio.ini b/platformio.ini index b9a5111..2c24d0f 100644 --- a/platformio.ini +++ b/platformio.ini @@ -1,5 +1,7 @@ [env:esp32dev] -platform = espressif32 +; Pinned: NimBLE-Arduino 1.4.x targets Arduino core 2.x (IDF 4.4). +; espressif32 6.13.0 is the last platform release on Arduino core 2.0.17. +platform = espressif32@6.13.0 board = esp32dev framework = arduino diff --git a/src/main.cpp b/src/main.cpp index 3205de1..dca2ba8 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -12,7 +12,7 @@ static uint8_t s_encKey[16]; // Shared between BLE core (1) and MQTT core (0) -static volatile bool s_newData = false; +static portMUX_TYPE s_dataMux = portMUX_INITIALIZER_UNLOCKED; static MpptData s_latest = {}; // MQTT + WiFi โ€” only touched by core 0 task @@ -33,6 +33,8 @@ static volatile int s_lastBleRssi = 0; static volatile bool s_bleSeen = false; static volatile uint32_t s_lastPublishOkMs = 0; +static uint32_t s_lastDiscoveryAttemptMs = 0; + #ifndef MQTT_TOPIC_DIAG #define MQTT_TOPIC_DIAG "solar/mppt/diag" #endif @@ -52,6 +54,16 @@ static volatile uint32_t s_lastPublishOkMs = 0; #define BLE_SCAN_WINDOW_MS 50 #endif +// NimBLE-Arduino 1.4.x only frees its result vector when a scan completes, so +// scan in long blocks and clear in between instead of scanning forever. +#ifndef BLE_SCAN_PERIOD_S +#define BLE_SCAN_PERIOD_S 30 +#endif + +#ifndef BLE_SCAN_PAUSE_MS +#define BLE_SCAN_PAUSE_MS 1000 +#endif + // Reconnect strategy #ifndef WIFI_RETRY_MS #define WIFI_RETRY_MS 10000UL @@ -151,6 +163,7 @@ static bool mqttEnsureConnected(uint32_t now) { s_mqttConnects++; s_mqtt.publish(MQTT_TOPIC_STATUS, "online", true); s_discoveryPublished = false; + s_lastDiscoveryAttemptMs = 0; // publish discovery immediately return true; } @@ -165,10 +178,10 @@ static bool mqttEnsureConnected(uint32_t now) { static bool publishDiscovery() { - static uint32_t lastAttemptMs = 0; uint32_t now = millis(); - if (lastAttemptMs != 0 && now - lastAttemptMs < DISCOVERY_RETRY_MS) return false; - lastAttemptMs = now; + if (s_lastDiscoveryAttemptMs != 0 && + now - s_lastDiscoveryAttemptMs < DISCOVERY_RETRY_MS) return false; + s_lastDiscoveryAttemptMs = now; // Common device suffix appended to every payload // identifiers must match across all sensors for HA to group them @@ -329,7 +342,9 @@ void mqttTask(void* param) { delay(1000); // Buffer must be set BEFORE connect โ€” largest discovery packet is ~570 bytes - s_mqtt.setBufferSize(1024); + if (!s_mqtt.setBufferSize(1024)) { + Serial.println("[MQTT] setBufferSize(1024) FAILED"); + } s_mqtt.setServer(MQTT_HOST, MQTT_PORT); s_mqtt.setKeepAlive(30); s_mqtt.setSocketTimeout(5); @@ -337,6 +352,7 @@ void mqttTask(void* param) { s_lastPublishOkMs = millis(); uint32_t lastDiagMs = millis(); uint32_t lastDataMs = s_lastPublishOkMs; + uint32_t lastForceReconnectMs = 0; for (;;) { uint32_t now = millis(); @@ -353,13 +369,18 @@ void mqttTask(void* param) { // Publish on a fixed cadence, not only when a BLE packet arrives: this // keeps the session exercised and makes an outage visible instead of - // leaving a silently stale broker. + // leaving a silently stale broker. New BLE packets simply refresh the + // snapshot that the next scheduled publish sends. if (mqttUp && s_bleSeen) { - if (s_newData) { - s_newData = false; - if (publishData(s_latest, now)) lastDataMs = now; - } else if (now - lastDataMs >= PUBLISH_INTERVAL_MS) { - if (publishData(s_latest, now)) lastDataMs = now; + MpptData snapshot; + + // Take a consistent copy; the BLE callback runs on the other core. + portENTER_CRITICAL(&s_dataMux); + snapshot = s_latest; + portEXIT_CRITICAL(&s_dataMux); + + if (now - lastDataMs >= PUBLISH_INTERVAL_MS) { + if (publishData(snapshot, now)) lastDataMs = now; } } @@ -369,12 +390,15 @@ void mqttTask(void* param) { } // MQTT thinks it is connected but nothing gets out -> half-open socket. - if (mqttUp && now - s_lastPublishOkMs >= MQTT_WATCHDOG_MS) { + // Keep the actual last-success timestamp untouched so the reboot + // watchdog below still fires if reconnecting does not help. + if (mqttUp && now - s_lastPublishOkMs >= MQTT_WATCHDOG_MS && + now - lastForceReconnectMs >= MQTT_WATCHDOG_MS) { Serial.printf("[MQTT] No successful publish for %lus, forcing reconnect\n", (unsigned long)((now - s_lastPublishOkMs) / 1000UL)); s_mqtt.disconnect(); - s_lastPublishOkMs = now; // give the new connection a full window - lastDataMs = now; + lastForceReconnectMs = now; + lastDataMs = 0; // publish the newest snapshot right after reconnect } // Total comms failure for a long time -> the ESP32 itself is wedged. @@ -405,11 +429,14 @@ class VictronAdvertisedDeviceCallbacks : public NimBLEAdvertisedDeviceCallbacks MpptData parsed = parseVictronMppt(s_encKey, payload, payloadLen); if (parsed.valid) { - s_latest = parsed; - s_lastBleMs = millis(); - s_bleSeen = true; + portENTER_CRITICAL(&s_dataMux); + s_latest = parsed; + portEXIT_CRITICAL(&s_dataMux); + + s_lastBleRssi = device->getRSSI(); + s_lastBleMs = millis(); + s_bleSeen = true; s_bleValid++; - s_newData = true; Serial.println("[BLE] Valid Victron packet decoded"); } else { s_bleIgnored++; @@ -426,11 +453,10 @@ static void bleInit() { s_scan = NimBLEDevice::getScan(); s_scan->setAdvertisedDeviceCallbacks(new VictronAdvertisedDeviceCallbacks(), true); s_scan->setActiveScan(false); - s_scan->setMaxResults(0); // callbacks only, don't store devices s_scan->setInterval(BLE_SCAN_INTERVAL_MS); s_scan->setWindow(BLE_SCAN_WINDOW_MS); - Serial.printf("[BLE] Scanner initialised (interval %d ms, window %d ms)\n", - BLE_SCAN_INTERVAL_MS, BLE_SCAN_WINDOW_MS); + Serial.printf("[BLE] Scanner initialised (interval %d ms, window %d ms, %d s blocks)\n", + BLE_SCAN_INTERVAL_MS, BLE_SCAN_WINDOW_MS, BLE_SCAN_PERIOD_S); } // โ”€โ”€ Setup & Loop โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ @@ -460,12 +486,10 @@ void setup() { } void loop() { - // Continuous passive scan. Restart only if the controller/host stopped it. - if (!s_scan->isScanning()) { - Serial.println("[BLE] Starting continuous scan"); - if (!s_scan->start(0, nullptr, true)) { - Serial.println("[BLE] Scan start FAILED"); - } - } - delay(5000); + // Long passive scan, then clear: NimBLE 1.4.x keeps one result object per + // unique BLE address (including phones with rotating MACs) until a scan + // completes, so clearing after each block keeps the heap flat. + s_scan->start(BLE_SCAN_PERIOD_S, false); + s_scan->clearResults(); + delay(BLE_SCAN_PAUSE_MS); } \ No newline at end of file diff --git a/src/victron_ble.cpp b/src/victron_ble.cpp index 8803171..e0c0f90 100644 --- a/src/victron_ble.cpp +++ b/src/victron_ble.cpp @@ -103,6 +103,11 @@ MpptData parseVictronMppt(const uint8_t* encKey, const uint8_t* data, size_t len } uint8_t decrypted[32] = {0}; + if (encLen > sizeof(decrypted)) { + Serial.printf("[BLE] Encrypted payload too long (%u bytes)\n", + (unsigned)encLen); + return result; + } if (!aes128ctrDecrypt(encKey, nonce, encrypted, decrypted, encLen)) { Serial.println("[BLE] AES decryption failed"); return result;