fix: recover from WiFi/MQTT dead states

- Replace the blocking WiFi and MQTT connect loops with non-blocking
  retry-on-timer functions, so the MQTT task never parks forever on an
  unreachable broker
- Publish data on a fixed cadence as well as on new BLE packets, adding
  data_age_s so stale values are visible
- Track successful publishes; force an MQTT reconnect after 5 minutes
  without one (half-open TCP socket) and reboot after 60 minutes with no
  success at all
- Retry Home Assistant discovery every 30s until every config topic has
  been accepted instead of silently giving up
- MQTT keepalive 30s, socket timeout 5s, publish buffer 1024 bytes
This commit is contained in:
Michael Holzmann 2026-11-14 14:49:36 +00:00
parent 98ac623423
commit 4cd4fbe41a
1 changed files with 142 additions and 60 deletions

View File

@ -29,7 +29,9 @@ static volatile uint32_t s_pubFailures = 0;
static volatile uint32_t s_bleValid = 0; static volatile uint32_t s_bleValid = 0;
static volatile uint32_t s_bleIgnored = 0; static volatile uint32_t s_bleIgnored = 0;
static volatile uint32_t s_lastBleMs = 0; static volatile uint32_t s_lastBleMs = 0;
static volatile int s_lastBleRssi = 0;
static volatile bool s_bleSeen = false; static volatile bool s_bleSeen = false;
static volatile uint32_t s_lastPublishOkMs = 0;
#ifndef MQTT_TOPIC_DIAG #ifndef MQTT_TOPIC_DIAG
#define MQTT_TOPIC_DIAG "solar/mppt/diag" #define MQTT_TOPIC_DIAG "solar/mppt/diag"
@ -50,6 +52,25 @@ static volatile bool s_bleSeen = false;
#define BLE_SCAN_WINDOW_MS 50 #define BLE_SCAN_WINDOW_MS 50
#endif #endif
// Reconnect strategy
#ifndef WIFI_RETRY_MS
#define WIFI_RETRY_MS 10000UL
#endif
#ifndef MQTT_RETRY_MS
#define MQTT_RETRY_MS 5000UL
#endif
#ifndef DISCOVERY_RETRY_MS
#define DISCOVERY_RETRY_MS 30000UL
#endif
// MQTT says "connected" but nothing publishes for this long -> force reconnect
#ifndef MQTT_WATCHDOG_MS
#define MQTT_WATCHDOG_MS 300000UL
#endif
// Nothing has published at all for this long -> reboot
#ifndef REBOOT_WATCHDOG_MS
#define REBOOT_WATCHDOG_MS 3600000UL
#endif
// ── Helpers ────────────────────────────────────────────────────────────────── // ── Helpers ──────────────────────────────────────────────────────────────────
static bool hexToBin(const char* hex, uint8_t* out, size_t outLen) { static bool hexToBin(const char* hex, uint8_t* out, size_t outLen) {
@ -90,26 +111,31 @@ static void onWiFiEvent(WiFiEvent_t event, WiFiEventInfo_t info) {
} }
} }
static void wifiConnect() { // Non-blocking: returns immediately and retries on a timer. Never parks the
Serial.printf("[WiFi] Connecting to %s", WIFI_SSID); // MQTT task, so BLE data keeps flowing while the network is down.
static void wifiEnsureConnected(uint32_t now) {
static uint32_t lastAttemptMs = 0;
if (WiFi.status() == WL_CONNECTED) return;
if (lastAttemptMs != 0 && now - lastAttemptMs < WIFI_RETRY_MS) return;
lastAttemptMs = now;
Serial.printf("[WiFi] Connecting to %s (status=%d)\n",
WIFI_SSID, (int)WiFi.status());
WiFi.disconnect();
WiFi.begin(WIFI_SSID, WIFI_PASSWORD); WiFi.begin(WIFI_SSID, WIFI_PASSWORD);
int attempts = 0;
while (WiFi.status() != WL_CONNECTED && attempts < 20) {
delay(500);
Serial.print(".");
attempts++;
}
if (WiFi.status() == WL_CONNECTED) {
Serial.printf("\n[WiFi] Connected, IP: %s\n", WiFi.localIP().toString().c_str());
} else {
Serial.println("\n[WiFi] Failed, will retry");
}
} }
// ── MQTT ───────────────────────────────────────────────────────────────────── // ── MQTT ─────────────────────────────────────────────────────────────────────
static void mqttConnect() { static bool mqttEnsureConnected(uint32_t now) {
while (!s_mqtt.connected()) { static uint32_t lastAttemptMs = 0;
if (s_mqtt.connected()) return true;
if (WiFi.status() != WL_CONNECTED) return false;
if (lastAttemptMs != 0 && now - lastAttemptMs < MQTT_RETRY_MS) return false;
lastAttemptMs = now;
Serial.print("[MQTT] Connecting..."); Serial.print("[MQTT] Connecting...");
bool ok; bool ok;
if (strlen(MQTT_USER) > 0) { if (strlen(MQTT_USER) > 0) {
@ -119,23 +145,30 @@ static void mqttConnect() {
ok = s_mqtt.connect(MQTT_CLIENT_ID, nullptr, nullptr, ok = s_mqtt.connect(MQTT_CLIENT_ID, nullptr, nullptr,
MQTT_TOPIC_STATUS, 0, true, "offline"); MQTT_TOPIC_STATUS, 0, true, "offline");
} }
if (ok) { if (ok) {
Serial.println(" connected"); Serial.println(" connected");
s_mqttConnects++; s_mqttConnects++;
s_mqtt.publish(MQTT_TOPIC_STATUS, "online", true); s_mqtt.publish(MQTT_TOPIC_STATUS, "online", true);
s_discoveryPublished = false; s_discoveryPublished = false;
} else { return true;
Serial.printf(" failed (rc=%d), retry in 5s\n", s_mqtt.state());
delay(5000);
}
} }
Serial.printf(" failed (rc=%d), retry in %lums\n",
s_mqtt.state(), (unsigned long)MQTT_RETRY_MS);
return false;
} }
// ── Home Assistant Discovery ────────────────────────────────────────────────── // ── Home Assistant Discovery ──────────────────────────────────────────────────
// Payloads are hardcoded strings in flash — no runtime String building, // Payloads are hardcoded strings in flash — no runtime String building,
// no buffer size surprises. Each is well under 600 bytes total. // no buffer size surprises. Each is well under 600 bytes total.
static void publishDiscovery() { static bool publishDiscovery() {
static uint32_t lastAttemptMs = 0;
uint32_t now = millis();
if (lastAttemptMs != 0 && now - lastAttemptMs < DISCOVERY_RETRY_MS) return false;
lastAttemptMs = now;
// Common device suffix appended to every payload // Common device suffix appended to every payload
// identifiers must match across all sensors for HA to group them // identifiers must match across all sensors for HA to group them
@ -207,23 +240,35 @@ static void publishDiscovery() {
}, },
}; };
bool allOk = true;
for (auto& s : sensors) { for (auto& s : sensors) {
bool ok = s_mqtt.publish(s.topic, s.payload, true); bool ok = s_mqtt.publish(s.topic, s.payload, true);
if (!ok) {
allOk = false;
s_pubFailures++;
}
Serial.printf("[HA] %s: %s (%d bytes)\n", Serial.printf("[HA] %s: %s (%d bytes)\n",
ok ? "OK" : "FAIL", s.topic, strlen(s.payload)); ok ? "OK" : "FAIL", s.topic, strlen(s.payload));
delay(50); delay(50);
} }
if (allOk) {
s_discoveryPublished = true; s_discoveryPublished = true;
s_lastPublishOkMs = millis();
Serial.println("[HA] Discovery complete"); Serial.println("[HA] Discovery complete");
} else {
Serial.println("[HA] Discovery incomplete, will retry");
}
#undef DEV #undef DEV
#undef AVAIL #undef AVAIL
return allOk;
} }
// ── Data Publisher ──────────────────────────────────────────────────────────── // ── Data Publisher ────────────────────────────────────────────────────────────
static void publishData(const MpptData& d) { static bool publishData(const MpptData& d, uint32_t now) {
StaticJsonDocument<512> doc; StaticJsonDocument<512> doc;
doc["battery_voltage"] = serialized(String(d.batteryVoltage, 2)); doc["battery_voltage"] = serialized(String(d.batteryVoltage, 2));
doc["battery_current"] = serialized(String(d.batteryCurrent, 2)); doc["battery_current"] = serialized(String(d.batteryCurrent, 2));
@ -234,15 +279,20 @@ static void publishData(const MpptData& d) {
doc["error_code"] = d.errorCode; doc["error_code"] = d.errorCode;
doc["external_load_a"] = serialized(String(d.externalLoad, 2)); doc["external_load_a"] = serialized(String(d.externalLoad, 2));
doc["external_load_w"] = serialized(String(d.externalLoad * d.batteryVoltage, 1)); doc["external_load_w"] = serialized(String(d.externalLoad * d.batteryVoltage, 1));
// Seconds since the last decoded BLE packet. Grows when BLE stops
// delivering, which distinguishes a BLE problem from an MQTT problem.
doc["data_age_s"] = s_bleSeen ? (now - s_lastBleMs) / 1000UL : 0;
char buf[256]; char buf[320];
serializeJson(doc, buf); serializeJson(doc, buf);
if (!s_mqtt.publish(MQTT_TOPIC_DATA, buf, false)) { if (!s_mqtt.publish(MQTT_TOPIC_DATA, buf, false)) {
s_pubFailures++; s_pubFailures++;
Serial.printf("[MQTT] Publish FAILED: %s\n", buf); Serial.printf("[MQTT] Publish FAILED: %s\n", buf);
return; return false;
} }
s_lastPublishOkMs = millis();
Serial.printf("[MQTT] Published: %s\n", buf); Serial.printf("[MQTT] Published: %s\n", buf);
return true;
} }
// ── Diagnostics ────────────────────────────────────────────────────────────── // ── Diagnostics ──────────────────────────────────────────────────────────────
@ -250,21 +300,26 @@ static void publishData(const MpptData& d) {
static void publishDiagnostics(uint32_t now) { static void publishDiagnostics(uint32_t now) {
uint32_t bleAgeS = s_bleSeen ? (now - s_lastBleMs) / 1000UL : 999999UL; uint32_t bleAgeS = s_bleSeen ? (now - s_lastBleMs) / 1000UL : 999999UL;
char buf[256]; char buf[320];
snprintf(buf, sizeof(buf), snprintf(buf, sizeof(buf),
"{\"uptime_s\":%lu,\"wifi_status\":%d,\"wifi_rssi\":%d," "{\"uptime_s\":%lu,\"wifi_status\":%d,\"wifi_rssi\":%d,"
"\"free_heap\":%u,\"min_free_heap\":%u," "\"free_heap\":%u,\"min_free_heap\":%u,"
"\"wifi_disconnects\":%lu,\"mqtt_connects\":%lu," "\"wifi_disconnects\":%lu,\"mqtt_connects\":%lu,"
"\"publish_failures\":%lu,\"ble_valid\":%lu," "\"publish_failures\":%lu,\"ble_valid\":%lu,"
"\"ble_ignored\":%lu,\"ble_age_s\":%lu}", "\"ble_ignored\":%lu,\"ble_rssi\":%d,\"ble_age_s\":%lu}",
(unsigned long)(now / 1000UL), (int)WiFi.status(), (int)WiFi.RSSI(), (unsigned long)(now / 1000UL), (int)WiFi.status(), (int)WiFi.RSSI(),
(unsigned)ESP.getFreeHeap(), (unsigned)ESP.getMinFreeHeap(), (unsigned)ESP.getFreeHeap(), (unsigned)ESP.getMinFreeHeap(),
(unsigned long)s_wifiDisconnects, (unsigned long)s_mqttConnects, (unsigned long)s_wifiDisconnects, (unsigned long)s_mqttConnects,
(unsigned long)s_pubFailures, (unsigned long)s_bleValid, (unsigned long)s_pubFailures, (unsigned long)s_bleValid,
(unsigned long)s_bleIgnored, (unsigned long)bleAgeS); (unsigned long)s_bleIgnored, (int)s_lastBleRssi,
(unsigned long)bleAgeS);
bool ok = s_mqtt.publish(MQTT_TOPIC_DIAG, buf, false); bool ok = s_mqtt.publish(MQTT_TOPIC_DIAG, buf, false);
if (!ok) s_pubFailures++; if (ok) {
s_lastPublishOkMs = millis();
} else {
s_pubFailures++;
}
Serial.printf("[DIAG] %s %s\n", ok ? "OK" : "FAIL", buf); Serial.printf("[DIAG] %s %s\n", ok ? "OK" : "FAIL", buf);
} }
@ -273,34 +328,61 @@ static void publishDiagnostics(uint32_t now) {
void mqttTask(void* param) { void mqttTask(void* param) {
delay(1000); delay(1000);
// Buffer must be set BEFORE connect — 620 bytes covers our largest payload // Buffer must be set BEFORE connect — largest discovery packet is ~570 bytes
s_mqtt.setBufferSize(620); s_mqtt.setBufferSize(1024);
s_mqtt.setServer(MQTT_HOST, MQTT_PORT); s_mqtt.setServer(MQTT_HOST, MQTT_PORT);
s_mqtt.setKeepAlive(60); s_mqtt.setKeepAlive(30);
s_mqtt.setSocketTimeout(5);
wifiConnect();
mqttConnect();
s_lastPublishOkMs = millis();
uint32_t lastDiagMs = millis(); uint32_t lastDiagMs = millis();
uint32_t lastDataMs = s_lastPublishOkMs;
for (;;) { for (;;) {
uint32_t now = millis(); uint32_t now = millis();
if (WiFi.status() != WL_CONNECTED) wifiConnect(); wifiEnsureConnected(now);
if (!s_mqtt.connected()) mqttConnect(); bool mqttUp = mqttEnsureConnected(now);
s_mqtt.loop();
if (mqttUp) {
if (!s_mqtt.loop()) {
Serial.printf("[MQTT] Connection lost (rc=%d)\n", s_mqtt.state());
}
if (!s_discoveryPublished) publishDiscovery(); if (!s_discoveryPublished) publishDiscovery();
}
// Publish on a fixed cadence, not only when a BLE packet arrives: this
// keeps the session exercised and makes an outage visible instead of
// leaving a silently stale broker.
if (mqttUp && s_bleSeen) {
if (s_newData) { if (s_newData) {
s_newData = false; s_newData = false;
publishData(s_latest); if (publishData(s_latest, now)) lastDataMs = now;
} else if (now - lastDataMs >= PUBLISH_INTERVAL_MS) {
if (publishData(s_latest, now)) lastDataMs = now;
}
} }
if (now - lastDiagMs >= DIAG_INTERVAL_MS) { if (now - lastDiagMs >= DIAG_INTERVAL_MS) {
lastDiagMs = now; lastDiagMs = now;
publishDiagnostics(now); if (mqttUp) publishDiagnostics(now);
}
// MQTT thinks it is connected but nothing gets out -> half-open socket.
if (mqttUp && now - s_lastPublishOkMs >= MQTT_WATCHDOG_MS) {
Serial.printf("[MQTT] No successful publish for %lus, forcing reconnect\n",
(unsigned long)((now - s_lastPublishOkMs) / 1000UL));
s_mqtt.disconnect();
s_lastPublishOkMs = now; // give the new connection a full window
lastDataMs = now;
}
// Total comms failure for a long time -> the ESP32 itself is wedged.
if (now - s_lastPublishOkMs >= REBOOT_WATCHDOG_MS) {
Serial.printf("[WD] No successful publish for %lus, rebooting\n",
(unsigned long)((now - s_lastPublishOkMs) / 1000UL));
Serial.flush();
ESP.restart();
} }
delay(100); delay(100);