[SYS/BT] Semaphores and new BT Controller (#1819)

Add semaphore to the enqueue calls to avoid concurrency issues
Add new BT controller to avoid scan evt timeout bug

main/User_config.h

@@ -180,6 +180,12 @@ const byte mac[] = {0xDE, 0xED, 0xBA, 0xFE, 0x54, 0x95}; //W5100 ethernet shield
 #ifndef GeneralTimeOut
 #  define GeneralTimeOut 20 // time out if a task is stuck in seconds (should be more than TimeBetweenReadingRN8209/1000) and more than 3 seconds, the WDT will reset the ESP, used also for MQTT connection
 #endif
+#ifndef QueueSemaphoreTimeOutTask
+#  define QueueSemaphoreTimeOutTask 3000 // time out for semaphore retrieval from a task
+#endif
+#ifndef QueueSemaphoreTimeOutLoop
+#  define QueueSemaphoreTimeOutLoop 100 // time out for semaphore retrieval from the loop
+#endif
 
 #if defined(ESP8266) || defined(ESP32)
 // Uncomment to use a device running TheengsGateway to decode BLE data. (https://github.com/theengs/gateway)
@@ -767,6 +773,10 @@ unsigned long lastDiscovery = 0; // Time of the last discovery to trigger automa
 #  define isBlack(device)       device->isBlkL
 #  define isDiscovered(device)  device->isDisc
 
+/*--------------------Minimum freeHeap--------------------*/
+// Below this parameter we trigger a restart, this avoid stuck boards like seen in https://github.com/1technophile/OpenMQTTGateway/issues/1693
+#  define MinimumMemory 40000
+
 /*----------------CONFIGURABLE PARAMETERS-----------------*/
 struct SYSConfig_s {
   bool discovery; // HA discovery convention

main/ZactuatorONOFF.ino

@@ -304,6 +304,6 @@ void stateONOFFMeasures() {
   JsonObject ONOFFdata = jsonBuffer.to<JsonObject>();
   ONOFFdata["cmd"] = (int)digitalRead(ACTUATOR_ONOFF_GPIO);
   ONOFFdata["origin"] = subjectGTWONOFFtoMQTT;
-  enqueueJsonObject(ONOFFdata);
+  handleJsonEnqueue(ONOFFdata, QueueSemaphoreTimeOutTask);
 }
 #endif

main/ZdisplaySSD1306.ino

@@ -467,7 +467,8 @@ String stateSSD1306Display() {
   DISPLAYdata["log-oled"] = (bool)logToOLEDDisplay;
   DISPLAYdata["json-oled"] = (bool)jsonDisplay;
   DISPLAYdata["origin"] = subjectSSD1306toMQTT;
-  enqueueJsonObject(DISPLAYdata);
+  handleJsonEnqueue(DISPLAYdata);
+
   // apply
   Oled.display->setBrightness(round(displayBrightness * 2.55));
 

main/Zgateway2G.ino

@@ -91,8 +91,7 @@ bool _2GtoMQTT() {
     A6l.deleteSMS(unreadSMSLocs[i]); // we delete the SMS received
     Log.trace(F("Adv data 2GtoMQTT" CR));
     SMSdata["origin"] = subject2GtoMQTT;
-    enqueueJsonObject(SMSdata);
-    return true;
+    return handleJsonEnqueue(SMSdata);
   }
   return false;
 }

main/ZgatewayBLEConnect.ino

@@ -121,7 +121,7 @@ bool zBLEConnect::processActions(std::vector<BLEAction>& actions) {
         BLEdata["success"] = result;
         if (result || it.ttl <= 1) {
           buildTopicFromId(BLEdata, subjectBTtoMQTT);
-          enqueueJsonObject(BLEdata);
+          handleJsonEnqueue(BLEdata, QueueSemaphoreTimeOutTask);
         }
       }
     }
@@ -161,7 +161,7 @@ void LYWSD03MMC_connect::notifyCB(NimBLERemoteCharacteristic* pChar, uint8_t* pD
       BLEdata["volt"] = (float)(((pData[4] * 256) + pData[3]) / 1000.0);
       BLEdata["batt"] = (float)(((((pData[4] * 256) + pData[3]) / 1000.0) - 2.1) * 100);
       buildTopicFromId(BLEdata, subjectBTtoMQTT);
-      enqueueJsonObject(BLEdata);
+      handleJsonEnqueue(BLEdata, QueueSemaphoreTimeOutTask);
     } else {
       Log.notice(F("Invalid notification data" CR));
       return;
@@ -226,7 +226,7 @@ void DT24_connect::notifyCB(NimBLERemoteCharacteristic* pChar, uint8_t* pData, s
       BLEdata["tempc"] = (float)(m_data[24] * 256) + m_data[25];
       BLEdata["tempf"] = (float)(convertTemp_CtoF((m_data[24] * 256) + m_data[25]));
       buildTopicFromId(BLEdata, subjectBTtoMQTT);
-      enqueueJsonObject(BLEdata);
+      handleJsonEnqueue(BLEdata, QueueSemaphoreTimeOutTask);
     } else {
       Log.notice(F("Invalid notification data" CR));
       return;
@@ -303,7 +303,7 @@ void BM2_connect::notifyCB(NimBLERemoteCharacteristic* pChar, uint8_t* pData, si
       BLEdata["volt"] = volt;
       Log.trace(F("volt: %F" CR), volt);
       buildTopicFromId(BLEdata, subjectBTtoMQTT);
-      enqueueJsonObject(BLEdata);
+      handleJsonEnqueue(BLEdata, QueueSemaphoreTimeOutTask);
     } else {
       Log.notice(F("Invalid notification data" CR));
       return;
@@ -361,7 +361,7 @@ void HHCCJCY01HHCC_connect::publishData() {
       BLEdata["id"] = (char*)mac_address.c_str();
       BLEdata["batt"] = (int)batteryValue;
       buildTopicFromId(BLEdata, subjectBTtoMQTT);
-      enqueueJsonObject(BLEdata);
+      handleJsonEnqueue(BLEdata, QueueSemaphoreTimeOutTask);
     } else {
       Log.notice(F("Failed getting characteristic" CR));
     }
@@ -393,7 +393,7 @@ void XMWSDJ04MMC_connect::notifyCB(NimBLERemoteCharacteristic* pChar, uint8_t* p
       BLEdata["volt"] = (float)((pData[4] | (pData[5] << 8)) / 1000.0);
       BLEdata["batt"] = (float)((((pData[4] | (pData[5] << 8)) / 1000.0) - 2.1) * 100);
       buildTopicFromId(BLEdata, subjectBTtoMQTT);
-      enqueueJsonObject(BLEdata);
+      handleJsonEnqueue(BLEdata, QueueSemaphoreTimeOutTask);
     } else {
       Log.notice(F("Invalid notification data" CR));
       return;
@@ -490,7 +490,7 @@ bool SBS1_connect::processActions(std::vector<BLEAction>& actions) {
           BLEdata["id"] = it.addr;
           BLEdata["state"] = result ? it.value : it.value == "on" ? "off" : "on";
           buildTopicFromId(BLEdata, subjectBTtoMQTT);
-          enqueueJsonObject(BLEdata);
+          handleJsonEnqueue(BLEdata, QueueSemaphoreTimeOutTask);
         }
       }
     }

main/ZgatewayBT.ino

@@ -103,23 +103,7 @@ void BTConfig_init() {
   BTConfig.ignoreWBlist = false;
   BTConfig.presenceAwayTimer = PresenceAwayTimer;
   BTConfig.movingTimer = MovingTimer;
-}
-
-// Watchdog, if there was no change of BLE scanCount, restart the ESP
-void btScanWDG() {
-  static unsigned long previousBtScanCount = 0;
-  static unsigned long lastBtScan = 0;
-  unsigned long now = millis();
-  if (!ProcessLock &&
-      previousBtScanCount == scanCount &&
-      scanCount != 0 &&
-      (now - lastBtScan > ((BTConfig.BLEinterval + BTConfig.scanDuration) < GeneralTimeOut ? GeneralTimeOut + 1000 : (BTConfig.BLEinterval + BTConfig.scanDuration)))) {
-    Log.error(F("BLE Scan watchdog triggered at : %ds" CR), lastBtScan / 1000);
-    ESPRestart(4);
-  } else {
-    previousBtScanCount = scanCount;
-    lastBtScan = now;
-  }
+  BTConfig.forcePassiveScan = false;
 }
 
 unsigned long timeBetweenConnect = 0;
@@ -148,6 +132,7 @@ String stateBTMeasures(bool start) {
   jo["ignoreWBlist"] = BTConfig.ignoreWBlist;
   jo["presenceawaytimer"] = BTConfig.presenceAwayTimer;
   jo["movingtimer"] = BTConfig.movingTimer;
+  jo["forcepassivescan"] = BTConfig.forcePassiveScan;
   jo["bletaskstack"] = uxTaskGetStackHighWaterMark(xProcBLETaskHandle);
   jo["blecoretaskstack"] = uxTaskGetStackHighWaterMark(xCoreTaskHandle);
 
@@ -160,7 +145,7 @@ String stateBTMeasures(bool start) {
   String output;
   serializeJson(jo, output);
   jo["origin"] = subjectBTtoMQTT;
-  enqueueJsonObject(jo);
+  handleJsonEnqueue(jo, QueueSemaphoreTimeOutTask);
   return (output);
 }
 
@@ -224,6 +209,8 @@ void BTConfig_fromJson(JsonObject& BTdata, bool startup = false) {
   Config_update(BTdata, "presenceawaytimer", BTConfig.presenceAwayTimer);
   // Timer to trigger a device state as offline if not seen
   Config_update(BTdata, "movingtimer", BTConfig.movingTimer);
+  // Force passive scan
+  Config_update(BTdata, "forcepassivescan", BTConfig.forcePassiveScan);
   // MinRSSI set
   Config_update(BTdata, "minrssi", BTConfig.minRssi);
   // Send undecoded device data
@@ -278,6 +265,7 @@ void BTConfig_fromJson(JsonObject& BTdata, bool startup = false) {
     jo["ignoreWBlist"] = BTConfig.ignoreWBlist;
     jo["presenceawaytimer"] = BTConfig.presenceAwayTimer;
     jo["movingtimer"] = BTConfig.movingTimer;
+    jo["forcepassivescan"] = BTConfig.forcePassiveScan;
     // Save config into NVS (non-volatile storage)
     String conf = "";
     serializeJson(jsonBuffer, conf);
@@ -418,7 +406,7 @@ void updateDevicesStatus() {
         BLEdata["id"] = p->macAdr;
         BLEdata["state"] = "offline";
         buildTopicFromId(BLEdata, subjectBTtoMQTT);
-        enqueueJsonObject(BLEdata);
+        handleJsonEnqueue(BLEdata, QueueSemaphoreTimeOutTask);
         // We set the lastUpdate to 0 to avoid replublishing the offline state
         p->lastUpdate = 0;
       }
@@ -432,7 +420,7 @@ void updateDevicesStatus() {
         BLEdata["id"] = p->macAdr;
         BLEdata["state"] = "offline";
         buildTopicFromId(BLEdata, subjectBTtoMQTT);
-        enqueueJsonObject(BLEdata);
+        handleJsonEnqueue(BLEdata, QueueSemaphoreTimeOutTask);
         // We set the lastUpdate to 0 to avoid replublishing the offline state
         p->lastUpdate = 0;
       }
@@ -668,7 +656,7 @@ void BLEscan() {
   BLEScan* pBLEScan = BLEDevice::getScan();
   MyAdvertisedDeviceCallbacks myCallbacks;
   pBLEScan->setAdvertisedDeviceCallbacks(&myCallbacks);
-  if (millis() > (timeBetweenActive + BTConfig.intervalActiveScan) || BTConfig.intervalActiveScan == BTConfig.BLEinterval) {
+  if ((millis() > (timeBetweenActive + BTConfig.intervalActiveScan) || BTConfig.intervalActiveScan == BTConfig.BLEinterval) && !BTConfig.forcePassiveScan) {
     pBLEScan->setActiveScan(true);
     timeBetweenActive = millis();
   } else {
@@ -756,17 +744,18 @@ void BLEconnect() {
 void stopProcessing() {
   ProcessLock = true;
   // We stop the scan
+  Log.notice(F("Stopping BLE scan" CR));
   BLEScan* pBLEScan = BLEDevice::getScan();
   if (pBLEScan->isScanning()) {
     pBLEScan->stop();
   }
+  Log.notice(F("Stopping BLE tasks" CR));
   //Suspending, deleting tasks and stopping BT to free memory
   vTaskSuspend(xCoreTaskHandle);
   vTaskDelete(xCoreTaskHandle);
   vTaskSuspend(xProcBLETaskHandle);
   vTaskDelete(xProcBLETaskHandle);
-  if (BLEDevice::getInitialized()) BLEDevice::deinit(true);
-  Log.notice(F("BLE gateway stopped, free heap: %d" CR), ESP.getFreeHeap());
+  Log.notice(F("BLE gateway stopped %T, free heap: %d" CR), ESP.getFreeHeap());
 }
 
 void coreTask(void* pvParameters) {
@@ -787,7 +776,7 @@ void coreTask(void* pvParameters) {
             timeBetweenConnect = millis();
             BLEconnect();
           }
-          dumpDevices();
+          //dumpDevices();
           Log.trace(F("CoreTask stack free: %u" CR), uxTaskGetStackHighWaterMark(xCoreTaskHandle));
           xSemaphoreGive(semaphoreBLEOperation);
         } else {
@@ -901,6 +890,7 @@ void setupBT() {
   Log.notice(F("Low Power Mode: %d" CR), lowpowermode);
   Log.notice(F("Presence Away Timer: %d" CR), BTConfig.presenceAwayTimer);
   Log.notice(F("Moving Timer: %d" CR), BTConfig.movingTimer);
+  Log.notice(F("Force passive scan: %T" CR), BTConfig.forcePassiveScan);
 
   atomic_init(&forceBTScan, 0); // in theory, we don't need this
 
@@ -936,7 +926,7 @@ boolean valid_service_data(const char* data, int size) {
 void launchBTDiscovery(bool overrideDiscovery) {
   if (!overrideDiscovery && newDevices == 0)
     return;
-  if (xSemaphoreTake(semaphoreCreateOrUpdateDevice, pdMS_TO_TICKS(1000)) == pdFALSE) {
+  if (xSemaphoreTake(semaphoreCreateOrUpdateDevice, pdMS_TO_TICKS(QueueSemaphoreTimeOutTask)) == pdFALSE) {
     Log.error(F("Semaphore NOT taken" CR));
     return;
   }
@@ -1128,7 +1118,7 @@ void PublishDeviceData(JsonObject& BLEdata) {
     // If the device is not a sensor and pubOnlySensors is true we don't publish this payload
     if (!BTConfig.pubOnlySensors || BLEdata.containsKey("model") || BLEdata.containsKey("distance")) { // Identified device
       buildTopicFromId(BLEdata, subjectBTtoMQTT);
-      enqueueJsonObject(BLEdata);
+      handleJsonEnqueue(BLEdata, QueueSemaphoreTimeOutTask);
     } else {
       Log.notice(F("Not a sensor device filtered" CR));
       return;
@@ -1144,7 +1134,7 @@ void PublishDeviceData(JsonObject& BLEdata) {
       BLEdata["mac"] = BLEdata["id"];
       BLEdata["id"] = BLEdata["uuid"];
     }
-    enqueueJsonObject(BLEdata);
+    handleJsonEnqueue(BLEdata, QueueSemaphoreTimeOutTask);
   } else {
     Log.notice(F("Low rssi, device filtered" CR));
     return;
@@ -1290,7 +1280,7 @@ void immediateBTAction(void* pvParameters) {
       BLEdata["id"] = BLEactions.back().addr;
       BLEdata["success"] = false;
       buildTopicFromId(BLEdata, subjectBTtoMQTT);
-      enqueueJsonObject(BLEdata);
+      handleJsonEnqueue(BLEdata, QueueSemaphoreTimeOutTask);
       BLEactions.pop_back();
       ProcessLock = false;
     }
@@ -1389,8 +1379,8 @@ void MQTTtoBT(char* topicOri, JsonObject& BTdata) { // json object decoding
     WorBupdated |= updateWorB(BTdata, false);
 
     if (WorBupdated) {
-      if (xSemaphoreTake(semaphoreCreateOrUpdateDevice, pdMS_TO_TICKS(1000)) == pdTRUE) {
-        dumpDevices();
+      if (xSemaphoreTake(semaphoreCreateOrUpdateDevice, pdMS_TO_TICKS(QueueSemaphoreTimeOutTask)) == pdTRUE) {
+        //dumpDevices();
         xSemaphoreGive(semaphoreCreateOrUpdateDevice);
       }
     }

main/ZgatewayGFSunInverter.ino

@@ -59,7 +59,7 @@ void GFSunInverterDataHandler(GfSun2000Data data) {
   jdata["register"] = jregister;
 #  endif
   jdata["origin"] = subjectRFtoMQTT;
-  enqueueJsonObject(jdata);
+  handleJsonEnqueue(jdata);
 }
 
 void GFSunInverterErrorHandler(int errorId, char* errorMessage) {
@@ -72,7 +72,7 @@ void GFSunInverterErrorHandler(int errorId, char* errorMessage) {
   jdata["msg"] = errorMessage;
   jdata["id"] = errorId;
   jdata["origin"] = subjectRFtoMQTT;
-  enqueueJsonObject(jdata);
+  handleJsonEnqueue(jdata);
 }
 
 void setupGFSunInverter() {

main/ZgatewayIR.ino

@@ -169,7 +169,7 @@ void IRtoMQTT() {
     } else if (!isAduplicateSignal(MQTTvalue) && MQTTvalue != 0) { // conditions to avoid duplications of IR -->MQTT
       Log.trace(F("Adv data IRtoMQTT" CR));
       IRdata["origin"] = subjectIRtoMQTT;
-      enqueueJsonObject(IRdata);
+      handleJsonEnqueue(IRdata);
       Log.trace(F("Store val: %D" CR), MQTTvalue);
       storeSignalValue(MQTTvalue);
       if (repeatIRwMQTT) {

main/ZgatewayLORA.ino

@@ -105,7 +105,7 @@ void createOrUpdateDeviceLORA(const char* id, const char* model, uint8_t flags)
 void launchLORADiscovery(bool overrideDiscovery) {
   if (!overrideDiscovery && newLORADevices == 0)
     return;
-  if (xSemaphoreTake(semaphorecreateOrUpdateDeviceLORA, pdMS_TO_TICKS(1000)) == pdFALSE) {
+  if (xSemaphoreTake(semaphorecreateOrUpdateDeviceLORA, pdMS_TO_TICKS(QueueSemaphoreTimeOutLoop)) == pdFALSE) {
     Log.error(F("[LORA] semaphorecreateOrUpdateDeviceLORA Semaphore NOT taken" CR));
     return;
   }
@@ -453,12 +453,11 @@ void LORAtoMQTT() {
     } else {
       LORAdataBuffer["origin"] = subjectLORAtoMQTT;
     }
-    enqueueJsonObject(LORAdata);
-
+    handleJsonEnqueue(LORAdata);
     if (repeatLORAwMQTT) {
       Log.trace(F("Pub LORA for rpt" CR));
       LORAdata["origin"] = subjectMQTTtoLORA;
-      enqueueJsonObject(LORAdata);
+      handleJsonEnqueue(LORAdata);
     }
   }
 }

main/ZgatewayPilight.ino

@@ -81,11 +81,11 @@ void pilightCallback(const String& protocol, const String& message, int status,
     RFPiLightdata["repeats"] = (int)repeats;
     RFPiLightdata["status"] = (int)status;
     RFPiLightdata["origin"] = subjectPilighttoMQTT;
-    enqueueJsonObject(RFPiLightdata);
+    handleJsonEnqueue(RFPiLightdata);
     if (repeatPilightwMQTT) {
       Log.trace(F("Pub Pilight for rpt" CR));
       RFPiLightdata["origin"] = subjectMQTTtoPilight;
-      enqueueJsonObject(RFPiLightdata);
+      handleJsonEnqueue(RFPiLightdata);
     }
   }
 }
@@ -108,7 +108,7 @@ void pilightRawCallback(const uint16_t* pulses, size_t length) {
 
   // Enqueue data
   RFPiLightdata["origin"] = subjectPilighttoMQTT;
-  enqueueJsonObject(RFPiLightdata);
+  handleJsonEnqueue(RFPiLightdata);
 }
 #  endif
 
@@ -273,7 +273,7 @@ void MQTTtoPilight(char* topicOri, JsonObject& Pilightdata) {
       if (msgLength > 0) {
         Log.trace(F("Adv data MQTTtoPilight push state via PilighttoMQTT" CR));
         Pilightdata["origin"] = subjectGTWPilighttoMQTT;
-        enqueueJsonObject(Pilightdata);
+        handleJsonEnqueue(Pilightdata);
         success = true;
       } else {
         switch (msgLength) {

main/ZgatewayRF.ino

@@ -171,14 +171,14 @@ void RFtoMQTT() {
         RFtoMQTTdiscovery(MQTTvalue);
 #  endif
       RFdata["origin"] = subjectRFtoMQTT;
-      enqueueJsonObject(RFdata);
+      handleJsonEnqueue(RFdata);
       // Casting "receivedSignal[o].value" to (unsigned long) because ArduinoLog doesn't support uint64_t for ESP's
       Log.trace(F("Store val: %u" CR), (unsigned long)MQTTvalue);
       storeSignalValue(MQTTvalue);
       if (repeatRFwMQTT) {
         Log.trace(F("Pub RF for rpt" CR));
         RFdata["origin"] = subjectMQTTtoRF;
-        enqueueJsonObject(RFdata);
+        handleJsonEnqueue(RFdata);
       }
     }
   }

main/ZgatewayRF2.ino

@@ -133,7 +133,7 @@ void RF2toMQTT() {
       RF2toMQTTdiscovery(RF2data);
 #  endif
     RF2data["origin"] = subjectRF2toMQTT;
-    enqueueJsonObject(RF2data);
+    handleJsonEnqueue(RF2data);
   }
 }
 

main/ZgatewayRFM69.ino

@@ -216,7 +216,7 @@ void MQTTtoRFM69(char* topicOri, JsonObject& RFM69data) {
         Log.notice(F(" OK " CR));
         // Acknowledgement to the GTWRF topic
         RFM69data["origin"] = subjectGTWRFM69toMQTT;
-        enqueueJsonObject(RFM69data);
+        handleJsonEnqueue(RFM69data);
       } else {
         Log.error(F("MQTTtoRFM69 sending failed" CR));
       }

main/ZgatewayRTL_433.ino

@@ -108,7 +108,7 @@ void createOrUpdateDeviceRTL_433(const char* id, const char* model, uint8_t flag
 void launchRTL_433Discovery(bool overrideDiscovery) {
   if (!overrideDiscovery && newRTL_433Devices == 0)
     return;
-  if (xSemaphoreTake(semaphorecreateOrUpdateDeviceRTL_433, pdMS_TO_TICKS(1000)) == pdFALSE) {
+  if (xSemaphoreTake(semaphorecreateOrUpdateDeviceRTL_433, pdMS_TO_TICKS(QueueSemaphoreTimeOutLoop)) == pdFALSE) {
     Log.error(F("[rtl_433] semaphorecreateOrUpdateDeviceRTL_433 Semaphore NOT taken" CR));
     return;
   }
@@ -258,7 +258,7 @@ void rtl_433_Callback(char* message) {
       storeRTL_433Discovery(RFrtl_433_ESPdata, (char*)model.c_str(), (char*)uniqueid.c_str());
 #  endif
     RFrtl_433_ESPdata["origin"] = (char*)topic.c_str();
-    enqueueJsonObject(RFrtl_433_ESPdata);
+    handleJsonEnqueue(RFrtl_433_ESPdata);
     storeSignalValue(MQTTvalue);
     pubOled((char*)topic.c_str(), RFrtl_433_ESPdata);
   }

main/ZgatewaySRFB.ino

@@ -121,13 +121,13 @@ void _rfbDecode() {
     if (!isAduplicateSignal(MQTTvalue) && MQTTvalue != 0) { // conditions to avoid duplications of RF -->MQTT
       Log.trace(F("Adv data SRFBtoMQTT" CR));
       SRFBdata["origin"] = subjectSRFBtoMQTT;
-      enqueueJsonObject(SRFBdata);
+      handleJsonEnqueue(SRFBdata);
       Log.trace(F("Store val: %lu" CR), MQTTvalue);
       storeSignalValue(MQTTvalue);
       if (repeatSRFBwMQTT) {
         Log.trace(F("Publish SRFB for rpt" CR));
         SRFBdata["origin"] = subjectMQTTtoSRFB;
-        enqueueJsonObject(SRFBdata);
+        handleJsonEnqueue(SRFBdata);
       }
     }
     _rfbAck();

main/ZgatewayWeatherStation.ino

@@ -41,7 +41,7 @@ void PairedDeviceAdded(byte newID) {
   RFdata["sensor"] = newID;
   RFdata["action"] = "paired";
   RFdata["origin"] = subjectRFtoMQTT;
-  enqueueJsonObject(RFdata);
+  handleJsonEnqueue(RFdata);
   wsdr.pair(NULL, PairedDeviceAdded);
 }
 
@@ -61,7 +61,7 @@ void sendWindSpeedData(byte id, float wind_speed, byte battery_status) {
     RFdata["wind_speed"] = wind_speed;
     RFdata["battery"] = bitRead(battery_status, 0) == 0 ? "OK" : "Low";
     RFdata["origin"] = subjectRFtoMQTT;
-    enqueueJsonObject(RFdata);
+    handleJsonEnqueue(RFdata);
     Log.trace(F("Store wind speed val: %lu" CR), MQTTvalue);
     storeSignalValue(MQTTvalue);
   }
@@ -76,7 +76,7 @@ void sendRainData(byte id, float rain_volume, byte battery_status) {
     RFdata["rain_volume"] = rain_volume;
     RFdata["battery"] = bitRead(battery_status, 1) == 0 ? "OK" : "Low";
     RFdata["origin"] = subjectRFtoMQTT;
-    enqueueJsonObject(RFdata);
+    handleJsonEnqueue(RFdata);
     Log.trace(F("Store rain_volume: %lu" CR), MQTTvalue);
     storeSignalValue(MQTTvalue);
   }
@@ -92,7 +92,7 @@ void sendWindData(byte id, int wind_direction, float wind_gust, byte battery_sta
     RFdata["wind_gust"] = wind_gust;
     RFdata["battery"] = bitRead(battery_status, 0) == 0 ? "OK" : "Low";
     RFdata["origin"] = subjectRFtoMQTT;
-    enqueueJsonObject(RFdata);
+    handleJsonEnqueue(RFdata);
     Log.trace(F("Store wind data val: %lu" CR), MQTTvalue);
     storeSignalValue(MQTTvalue);
   }
@@ -109,7 +109,7 @@ void sendTemperatureData(byte id, float temperature, int humidity, byte battery_
     RFdata["humidity"] = humidity;
     RFdata["battery"] = bitRead(battery_status, 0) == 0 ? "OK" : "Low";
     RFdata["origin"] = subjectRFtoMQTT;
-    enqueueJsonObject(RFdata);
+    handleJsonEnqueue(RFdata);
     Log.trace(F("Store temp val: %lu" CR), MQTTvalue);
     storeSignalValue(MQTTvalue);
   }

main/ZsensorADC.ino

@@ -94,7 +94,7 @@ void MeasureADC() {
         ADCdata["mvolt_scaled"] = (int)voltage; // real scaled/calibrated value in mV
 #  endif
         ADCdata["origin"] = ADCTOPIC;
-        enqueueJsonObject(ADCdata);
+        handleJsonEnqueue(ADCdata);
         persistedadc = val;
       }
     }

main/ZsensorAHTx0.ino

@@ -104,7 +104,7 @@ void MeasureAHTTempHum() {
         Log.notice(F("Same Humidity. Don't send it" CR));
       }
       AHTx0data["origin"] = AHTTOPIC;
-      enqueueJsonObject(AHTx0data);
+      handleJsonEnqueue(AHTx0data);
     }
     persisted_aht_tempc = ahtTempC.temperature;
     persisted_aht_hum = ahtHum.relative_humidity;

main/ZsensorBH1750.ino

@@ -101,7 +101,7 @@ void MeasureLightIntensity() {
         Log.trace(F("Same wattsm2 don't send it" CR));
       }
       BH1750data["origin"] = subjectBH1750toMQTT;
-      enqueueJsonObject(BH1750data);
+      handleJsonEnqueue(BH1750data);
     }
     persistedll = Lux;
     persistedlf = ftcd;

main/ZsensorBME280.ino

@@ -192,7 +192,7 @@ void MeasureTempHumAndPressure() {
         Log.trace(F("Same Altitude Feet don't send it" CR));
       }
       BME280data["origin"] = BMETOPIC;
-      enqueueJsonObject(BME280data);
+      handleJsonEnqueue(BME280data);
     }
 
     persisted_bme_tempc = BmeTempC;

main/ZsensorC37_YL83_HMRD.ino

@@ -67,7 +67,7 @@ void MeasureC37_YL83_HMRDWater() {
     }
     if (C37_YL83_HMRDdata.size() > 0) {
       C37_YL83_HMRDdata["origin"] = C37_YL83_HMRD_TOPIC;
-      enqueueJsonObject(C37_YL83_HMRDdata);
+      handleJsonEnqueue(C37_YL83_HMRDdata);
       delay(10);
 #  if defined(DEEP_SLEEP_IN_US) || defined(ESP32_EXT0_WAKE_PIN)
       if (sensorDigitalValue == 1) //No water detected, all good we can sleep

main/ZsensorDHT.ino

@@ -69,7 +69,7 @@ void MeasureTempAndHum() {
         Log.trace(F("Same temp don't send it" CR));
       }
       DHTdata["origin"] = DHTTOPIC;
-      enqueueJsonObject(DHTdata);
+      handleJsonEnqueue(DHTdata);
     }
     persistedh = h;
     persistedt = t;

main/ZsensorDS1820.ino

@@ -158,7 +158,7 @@ void MeasureDS1820Temp() {
           }
           String origin = String(OW_TOPIC) + "/" + ds1820_addr[i];
           DS1820data["origin"] = origin;
-          enqueueJsonObject(DS1820data);
+          handleJsonEnqueue(DS1820data);
           delay(10);
 #  if defined(DEEP_SLEEP_IN_US) || defined(ESP32_EXT0_WAKE_PIN)
           ready_to_sleep = true;

main/ZsensorGPIOInput.ino

@@ -94,7 +94,7 @@ void MeasureGPIOInput() {
         GPIOdata["gpio"] = "LOW";
       }
       GPIOdata["origin"] = subjectGPIOInputtoMQTT;
-      enqueueJsonObject(GPIOdata);
+      handleJsonEnqueue(GPIOdata);
 
 #  if defined(ZactuatorONOFF) && defined(ACTUATOR_TRIGGER)
       //Trigger the actuator if we are not at startup

main/ZsensorHCSR04.ino

@@ -69,8 +69,7 @@ void MeasureDistance() {
         Log.trace(F("HC SR04 Distance hasn't changed" CR));
       }
       distance = d;
-      HCSR04data["origin"] = subjectHCSR04;
-      enqueueJsonObject(HCSR04data);
+      handleJsonEnqueue(HCSR04data);
     }
   }
 }

main/ZsensorHCSR501.ino

@@ -65,7 +65,7 @@ void MeasureHCSR501() {
     digitalWrite(HCSR501_LED_NOTIFY_GPIO, pirState == HCSR501_LED_ON);
 #  endif
     HCSR501data["origin"] = subjectHCSR501toMQTT;
-    enqueueJsonObject(HCSR501data);
+    handleJsonEnqueue(HCSR501data);
   }
 }
 #endif

main/ZsensorHTU21.ino

@@ -101,7 +101,7 @@ void MeasureTempHum() {
         Log.notice(F("Same Humidity. Don't send it" CR));
       }
       HTU21data["origin"] = HTUTOPIC;
-      enqueueJsonObject(HTU21data);
+      handleJsonEnqueue(HTU21data);
     }
     persisted_htu_tempc = HtuTempC;
     persisted_htu_hum = HtuHum;

main/ZsensorINA226.ino

@@ -73,7 +73,7 @@ void MeasureINA226() {
     INA226data["current"] = current;
     INA226data["power"] = power;
     INA226data["origin"] = subjectINA226toMQTT;
-    enqueueJsonObject(INA226data);
+    handleJsonEnqueue(INA226data);
   }
 }
 

main/ZsensorLM75.ino

@@ -88,7 +88,7 @@ void MeasureTemp() {
         LM75data["tempc"] = (float)lm75TempC;
         LM75data["tempf"] = (float)lm75TempF;
         LM75data["origin"] = LM75TOPIC;
-        enqueueJsonObject(LM75data);
+        handleJsonEnqueue(LM75data);
       } else {
         Log.notice(F("Same Temp. Don't send it" CR));
       }

main/ZsensorMQ2.ino

@@ -67,7 +67,7 @@ void MeasureGasMQ2() {
     MQ2data["gas"] = analogRead(MQ2SENSORADCPIN);
     MQ2data["detected"] = digitalRead(MQ2SENSORDETECTPIN) == HIGH ? "false" : "true";
     MQ2data["origin"] = subjectMQ2toMQTT;
-    enqueueJsonObject(MQ2data);
+    handleJsonEnqueue(MQ2data);
   }
 }
 #endif

main/ZsensorRN8209.ino

@@ -91,7 +91,7 @@ void rn8209_loop(void* mode) {
       previousCurrent = current;
       if (RN8209data) {
         RN8209data["origin"] = subjectRN8209toMQTT;
-        enqueueJsonObject(RN8209data);
+        handleJsonEnqueue(RN8209data, QueueSemaphoreTimeOutTask);
       }
     }
     //esp_task_wdt_reset();

main/ZsensorSHTC3.ino

@@ -61,7 +61,7 @@ void MeasureTempAndHum() {
           Log.trace(F("Same temp don't send it" CR));
         }
         SHTC3data["origin"] = SHTC3TOPIC;
-        enqueueJsonObject(SHTC3data);
+        handleJsonEnqueue(SHTC3data);
       }
       persistedh = h;
       persistedt = t;

main/ZsensorTEMT6000.ino

@@ -68,7 +68,7 @@ void MeasureLightIntensityTEMT6000() {
       TEMT6000data["ftcd"] = (float)(lux) / 10.764;
       TEMT6000data["wattsm2"] = (float)(lux) / 683.0;
       TEMT6000data["origin"] = subjectTEMT6000toMQTT;
-      enqueueJsonObject(TEMT6000data);
+      handleJsonEnqueue(TEMT6000data);
     } else {
       Log.trace(F("Same lux value, do not send" CR));
     }

main/ZsensorTSL2561.ino

@@ -105,7 +105,7 @@ void MeasureLightIntensityTSL2561() {
         TSL2561data["ftcd"] = (float)(event.light) / 10.764;
         TSL2561data["wattsm2"] = (float)(event.light) / 683.0;
         TSL2561data["origin"] = subjectTSL12561toMQTT;
-        enqueueJsonObject(TSL2561data);
+        handleJsonEnqueue(TSL2561data);
       } else {
         Log.trace(F("Same lux value, do not send" CR));
       }

main/ZwebUI.ino

@@ -1589,7 +1589,7 @@ String stateWebUIStatus() {
 
   // WebUIdata["currentMessage"] = currentWebUIMessage;
   WebUIdata["origin"] = subjectWebUItoMQTT;
-  enqueueJsonObject(WebUIdata);
+  handleJsonEnqueue(WebUIdata);
   return output;
 }
 

main/config_BT.h

@@ -165,6 +165,7 @@ struct BTConfig_s {
   bool ignoreWBlist; // Disable Whitelist & Blacklist
   unsigned long presenceAwayTimer; //Timer that trigger a tracker/PIR state as offline/off if not seen
   unsigned long movingTimer; //Timer that trigger a moving sensor state as offline if not seen
+  bool forcePassiveScan; //Force passive scan
 };
 
 // Global struct to store live BT configuration data

main/main.ino

@@ -92,6 +92,12 @@ struct JsonBundle {
 };
 
 std::queue<JsonBundle> jsonQueue;
+
+#  ifdef ESP32
+// Mutex  to protect the queue
+SemaphoreHandle_t xQueueMutex;
+#  endif
+
 #endif
 
 StaticJsonDocument<JSON_MSG_BUFFER> modulesBuffer;
@@ -450,6 +456,30 @@ void enqueueJsonObject(const StaticJsonDocument<JSON_MSG_BUFFER>& jsonDoc) {
 #endif
 }
 
+#ifdef ESP32
+// Semaphore check before enqueueing a document
+bool handleJsonEnqueue(const StaticJsonDocument<JSON_MSG_BUFFER>& jsonDoc, int timeout) {
+  if (xSemaphoreTake(xQueueMutex, pdMS_TO_TICKS(timeout))) {
+    enqueueJsonObject(jsonDoc);
+    xSemaphoreGive(xQueueMutex);
+    return true;
+  } else {
+    Log.error(F("xQueueMutex not taken" CR));
+    blockedMessages++;
+    return false;
+  }
+}
+#else
+bool handleJsonEnqueue(const StaticJsonDocument<JSON_MSG_BUFFER>& jsonDoc, int timeout) {
+  enqueueJsonObject(jsonDoc);
+}
+#endif
+
+// Semaphore check before enqueueing a document with default timeout QueueSemaphoreTimeOutLoop
+bool handleJsonEnqueue(const StaticJsonDocument<JSON_MSG_BUFFER>& jsonDoc) {
+  return handleJsonEnqueue(jsonDoc, QueueSemaphoreTimeOutLoop);
+}
+
 #if defined(ESP8266) || defined(ESP32) || defined(__AVR_ATmega2560__) || defined(__AVR_ATmega1280__)
 
 /*
@@ -1029,6 +1059,7 @@ void setup() {
   Serial.begin(SERIAL_BAUD, SERIAL_8N1, SERIAL_TX_ONLY); // enable on ESP8266 to free some pin
 #    endif
 #  elif ESP32
+  xQueueMutex = xSemaphoreCreateMutex();
 #    if DEFAULT_LOW_POWER_MODE != -1 //  don't check preferences value if low power mode is not activated
   preferences.begin(Gateway_Short_Name, false);
   if (preferences.isKey("lowpowermode")) {
@@ -1477,6 +1508,7 @@ bool wifi_reconnect_bypass() {
   5 - User requested reboot
   6 - OTA Update
   7 - Parameters changed
+  8 - not enough memory to pursue
 */
 #if defined(ESP8266) || defined(ESP32)
 void ESPRestart(byte reason) {
@@ -2042,7 +2074,6 @@ void loop() {
         stateMeasures();
 #  ifdef ZgatewayBT
         stateBTMeasures(false);
-        btScanWDG();
 #  endif
 #  ifdef ZactuatorONOFF
         stateONOFFMeasures();
@@ -2348,6 +2379,9 @@ String stateMeasures() {
   uint32_t freeMem;
   uint32_t minFreeMem;
   freeMem = ESP.getFreeHeap();
+  if (freeMem < MinimumMemory) {
+    ESPRestart(8);
+  }
   SYSdata["freemem"] = freeMem;
   SYSdata["mqttport"] = mqtt_port;
   SYSdata["mqttsecure"] = mqtt_secure;
@@ -2428,7 +2462,7 @@ String stateMeasures() {
   SYSdata["modules"] = modules;
 
   SYSdata["origin"] = subjectSYStoMQTT;
-  enqueueJsonObject(SYSdata);
+  handleJsonEnqueue(SYSdata);
   pubOled(subjectSYStoMQTT, SYSdata);
 
   char jsonChar[100];
@@ -2665,7 +2699,8 @@ bool checkForUpdates() {
     latestVersion = jsondata["latest_version"].as<String>();
     jsondata["origin"] = subjectRLStoMQTT;
     jsondata["retain"] = true;
-    enqueueJsonObject(jsondata);
+    handleJsonEnqueue(jsondata);
+
     Log.trace(F("Update file found on server" CR));
     return true;
   } else {
@@ -2735,7 +2770,7 @@ void MQTTHttpsFWUpdate(char* topicOri, JsonObject& HttpsFwUpdateData) {
       StaticJsonDocument<JSON_MSG_BUFFER> jsondata;
       jsondata["release_summary"] = "Update in progress ...";
       jsondata["origin"] = subjectRLStoMQTT;
-      enqueueJsonObject(jsondata);
+      handleJsonEnqueue(jsondata);
 
       const char* ota_cert = HttpsFwUpdateData["server_cert"];
       if (!ota_cert && !strstr(url, "http:")) {
@@ -2803,7 +2838,7 @@ void MQTTHttpsFWUpdate(char* topicOri, JsonObject& HttpsFwUpdateData) {
           jsondata["release_summary"] = "Update success !";
           jsondata["installed_version"] = latestVersion;
           jsondata["origin"] = subjectRLStoMQTT;
-          enqueueJsonObject(jsondata);
+          handleJsonEnqueue(jsondata);
           ota_server_cert = ota_cert;
 #  ifndef ESPWifiManualSetup
           saveConfig();