Arduino NimBLE support (#732)

* NimBLE-arduino support

* Decorrelate discovery from BLEScan

To avoid Stack canary watchpoint triggered and store sensor model into BLEdevice vector

main/ZgatewayBT.ino

@@ -54,8 +54,10 @@ struct BLEdevice {
   bool isDisc;
   bool isWhtL;
   bool isBlkL;
+  ble_sensor_model sensorModel;
 };
 
+#  define device_flags_init     0 << 0
 #  define device_flags_isDisc   1 << 0
 #  define device_flags_isWhiteL 1 << 1
 #  define device_flags_isBlackL 1 << 2
@@ -68,12 +70,13 @@ struct decompose {
 
 vector<BLEdevice> devices;
 
-static BLEdevice NO_DEVICE_FOUND = {{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, false, false, false};
+static BLEdevice NO_DEVICE_FOUND = {{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, false, false, false, UNKNOWN_MODEL};
 static bool oneWhite = false;
 
 int minRssi = abs(MinimumRSSI); //minimum rssi value
 
 BLEdevice* getDeviceByMac(const char* mac);
+void createOrUpdateDevice(const char* mac, uint8_t flags, ble_sensor_model model);
 
 BLEdevice* getDeviceByMac(const char* mac) {
   Log.trace(F("getDeviceByMac %s" CR), mac);
@@ -96,13 +99,13 @@ bool updateWorB(JsonObject& BTdata, bool isWhite) {
 
   for (int i = 0; i < size; i++) {
     const char* mac = BTdata[jsonKey][i];
-    createOrUpdateDevice(mac, (isWhite ? device_flags_isWhiteL : device_flags_isBlackL));
+    createOrUpdateDevice(mac, (isWhite ? device_flags_isWhiteL : device_flags_isBlackL), UNKNOWN_MODEL);
   }
 
   return true;
 }
 
-void createOrUpdateDevice(const char* mac, uint8_t flags) {
+void createOrUpdateDevice(const char* mac, uint8_t flags, ble_sensor_model model) {
 #  ifdef ESP32
   if (!semaphoreCreateOrUpdateDevice.take(30000, "createOrUpdateDevice"))
     return;
@@ -117,6 +120,7 @@ void createOrUpdateDevice(const char* mac, uint8_t flags) {
     device->isDisc = flags & device_flags_isDisc;
     device->isWhtL = flags & device_flags_isWhiteL;
     device->isBlkL = flags & device_flags_isBlackL;
+    if (model != UNKNOWN_MODEL) device->sensorModel = model;
     devices.push_back(*device);
   } else {
     Log.trace(F("update %s" CR), mac);
@@ -125,6 +129,8 @@ void createOrUpdateDevice(const char* mac, uint8_t flags) {
       device->isDisc = true;
     }
 
+    if (model != UNKNOWN_MODEL) device->sensorModel = model;
+
     if (flags & device_flags_isWhiteL || flags & device_flags_isBlackL) {
       device->isWhtL = flags & device_flags_isWhiteL;
       device->isBlkL = flags & device_flags_isBlackL;
@@ -149,6 +155,7 @@ void dumpDevices() {
     Log.trace(F("isDisc %d" CR), p->isDisc);
     Log.trace(F("isWhtL %d" CR), p->isWhtL);
     Log.trace(F("isBlkL %d" CR), p->isBlkL);
+    Log.trace(F("sensorModel %d" CR), p->sensorModel);
   }
 }
 
@@ -350,10 +357,10 @@ void InkBirdDiscovery(char* mac) {}
     */
 // core task implementation thanks to https://techtutorialsx.com/2017/05/09/esp32-running-code-on-a-specific-core/
 
-#    include <BLEAdvertisedDevice.h>
+#    include <NimBLEAdvertisedDevice.h>
-#    include <BLEDevice.h>
+#    include <NimBLEDevice.h>
-#    include <BLEScan.h>
+#    include <NimBLEScan.h>
-#    include <BLEUtils.h>
+#    include <NimBLEUtils.h>
 
 #    include "soc/timer_group_reg.h"
 #    include "soc/timer_group_struct.h"
@@ -364,37 +371,37 @@ static int taskCore = 0;
 bool ProcessLock = false;
 
 class MyAdvertisedDeviceCallbacks : public BLEAdvertisedDeviceCallbacks {
-  void onResult(BLEAdvertisedDevice advertisedDevice) {
+  void onResult(BLEAdvertisedDevice* advertisedDevice) {
     Log.trace(F("Creating BLE buffer" CR));
     StaticJsonBuffer<JSON_MSG_BUFFER> jsonBuffer;
     JsonObject& BLEdata = jsonBuffer.createObject();
-    String mac_adress = advertisedDevice.getAddress().toString().c_str();
+    String mac_adress = advertisedDevice->getAddress().toString().c_str();
     mac_adress.toUpperCase();
     BLEdata.set("id", (char*)mac_adress.c_str());
     Log.notice(F("Device detected: %s" CR), (char*)mac_adress.c_str());
     BLEdevice* device = getDeviceByMac(BLEdata["id"].as<const char*>());
 
     if ((!oneWhite || isWhite(device)) && !isBlack(device)) { //if not black listed mac we go AND if we have no white mac or this mac is  white we go out
-      if (advertisedDevice.haveName())
+      if (advertisedDevice->haveName())
-        BLEdata.set("name", (char*)advertisedDevice.getName().c_str());
+        BLEdata.set("name", (char*)advertisedDevice->getName().c_str());
-      if (advertisedDevice.haveManufacturerData()) {
+      if (advertisedDevice->haveManufacturerData()) {
-        char* manufacturerdata = BLEUtils::buildHexData(NULL, (uint8_t*)advertisedDevice.getManufacturerData().data(), advertisedDevice.getManufacturerData().length());
+        char* manufacturerdata = BLEUtils::buildHexData(NULL, (uint8_t*)advertisedDevice->getManufacturerData().data(), advertisedDevice->getManufacturerData().length());
         Log.trace(F("Manufacturer Data: %s" CR), manufacturerdata);
         BLEdata.set("manufacturerdata", manufacturerdata);
       }
-      if (advertisedDevice.haveRSSI())
+      if (advertisedDevice->haveRSSI())
-        BLEdata.set("rssi", (int)advertisedDevice.getRSSI());
+        BLEdata.set("rssi", (int)advertisedDevice->getRSSI());
-      if (advertisedDevice.haveTXPower())
+      if (advertisedDevice->haveTXPower())
-        BLEdata.set("txpower", (int8_t)advertisedDevice.getTXPower());
+        BLEdata.set("txpower", (int8_t)advertisedDevice->getTXPower());
 #    ifdef subjectHomePresence
-      if (advertisedDevice.haveRSSI())
+      if (advertisedDevice->haveRSSI())
         haRoomPresence(BLEdata); // this device has an rssi in consequence we can use it for home assistant room presence component
 #    endif
-      if (advertisedDevice.haveServiceData()) {
+      if (advertisedDevice->haveServiceData()) {
-        int serviceDataCount = advertisedDevice.getServiceDataCount();
+        int serviceDataCount = advertisedDevice->getServiceDataCount();
         Log.trace(F("Get services data number: %d" CR), serviceDataCount);
         for (int j = 0; j < serviceDataCount; j++) {
-          std::string serviceData = advertisedDevice.getServiceData(j);
+          std::string serviceData = advertisedDevice->getServiceData(j);
           int serviceDataLength = serviceData.length();
           String returnedString = "";
           for (int i = 0; i < serviceDataLength; i++) {
@@ -409,7 +416,7 @@ class MyAdvertisedDeviceCallbacks : public BLEAdvertisedDeviceCallbacks {
           service_data[returnedString.length()] = '\0';
           Log.trace(F("Service data: %s" CR), service_data);
           BLEdata.set("servicedata", service_data);
-          std::string serviceDatauuid = advertisedDevice.getServiceDataUUID(j).toString();
+          std::string serviceDatauuid = advertisedDevice->getServiceDataUUID(j).toString();
           Log.trace(F("Service data UUID: %s" CR), (char*)serviceDatauuid.c_str());
           BLEdata.set("servicedatauuid", (char*)serviceDatauuid.c_str());
           PublishDeviceData(BLEdata);
@@ -430,12 +437,11 @@ void BLEscan() {
   Log.notice(F("Scan begin" CR));
   BLEDevice::init("");
   BLEScan* pBLEScan = BLEDevice::getScan(); //create new scan
-  MyAdvertisedDeviceCallbacks myCallbacks;
+  pBLEScan->setAdvertisedDeviceCallbacks(new MyAdvertisedDeviceCallbacks());
-  pBLEScan->setAdvertisedDeviceCallbacks(&myCallbacks);
   pBLEScan->setActiveScan(true); //active scan uses more power, but get results faster
   BLEScanResults foundDevices = pBLEScan->start(Scan_duration, false);
   Log.notice(F("Found %d devices, scan end deinit controller" CR), foundDevices.getCount());
-  esp_bt_controller_deinit();
+  BLEDevice::deinit(true);
 }
 
 void stopProcessing() {
@@ -465,6 +471,8 @@ void coreTask(void* pvParameters) {
         if (low_power_mode == 2)
           digitalWrite(LOW_POWER_LED, 1 - LOW_POWER_LED_OFF);
         BLEscan();
+        launch_discovery();
+        dumpDevices();
         //only change LOW_POWER_LED if low power mode is enabled
         if (low_power_mode)
           digitalWrite(LOW_POWER_LED, LOW_POWER_LED_OFF);
@@ -691,38 +699,35 @@ boolean valid_service_data(const char* data) {
   return false;
 }
 
-void launch_discovery(JsonObject& BLEdata) {
+void launch_discovery() {
-  const char* mac = BLEdata["id"].as<const char*>();
+  for (vector<BLEdevice>::iterator p = devices.begin(); p != devices.end(); ++p) {
-  BLEdevice* device = getDeviceByMac(mac);
+    if (p->sensorModel != UNKNOWN_MODEL && !isDiscovered(p)) {
-  String macWOdots = String(mac);
+      String macWOdots = String(p->macAdr);
       macWOdots.replace(":", "");
-  if (!isDiscovered(device) && BLEdata.containsKey("model")) {
+      Log.trace(F("Launching discovery of %s" CR), p->macAdr);
-    Log.trace(F("Launching discovery of %s" CR), mac);
+      if (p->sensorModel == HHCCJCY01HHCC) MiFloraDiscovery((char*)macWOdots.c_str());
-    if (strcmp(BLEdata["model"].as<const char*>(), "HHCCJCY01HHCC") == 0) MiFloraDiscovery((char*)macWOdots.c_str());
+      if (p->sensorModel == VEGTRUG) VegTrugDiscovery((char*)macWOdots.c_str());
-    if (strcmp(BLEdata["model"].as<const char*>(), "VegTrug") == 0) VegTrugDiscovery((char*)macWOdots.c_str());
+      if (p->sensorModel == LYWSDCGQ) MiJiaDiscovery((char*)macWOdots.c_str());
-    if (strcmp(BLEdata["model"].as<const char*>(), "LYWSDCGQ") == 0) MiJiaDiscovery((char*)macWOdots.c_str());
+      if (p->sensorModel == JQJCY01YM) FormalDiscovery((char*)macWOdots.c_str());
-    if (strcmp(BLEdata["model"].as<const char*>(), "JQJCY01YM") == 0) FormalDiscovery((char*)macWOdots.c_str());
+      if (p->sensorModel == LYWSD02) LYWSD02Discovery((char*)macWOdots.c_str());
-    if (strcmp(BLEdata["model"].as<const char*>(), "LYWSD02") == 0) LYWSD02Discovery((char*)macWOdots.c_str());
+      if (p->sensorModel == CGG1) CLEARGRASSTRHDiscovery((char*)macWOdots.c_str());
-    if (strcmp(BLEdata["model"].as<const char*>(), "CGG1") == 0) CLEARGRASSTRHDiscovery((char*)macWOdots.c_str());
+      if (p->sensorModel == CGP1W) CLEARGRASSTRHKPADiscovery((char*)macWOdots.c_str());
-    if (strcmp(BLEdata["model"].as<const char*>(), "CGP1W") == 0) CLEARGRASSTRHKPADiscovery((char*)macWOdots.c_str());
+      if (p->sensorModel == MUE4094RT) MiLampDiscovery((char*)macWOdots.c_str());
-    if (strcmp(BLEdata["model"].as<const char*>(), "MUE4094RT") == 0) MiLampDiscovery((char*)macWOdots.c_str());
+      if (p->sensorModel == CGD1) CLEARGRASSCGD1Discovery((char*)macWOdots.c_str());
-    if (strcmp(BLEdata["model"].as<const char*>(), "CGD1") == 0) CLEARGRASSCGD1Discovery((char*)macWOdots.c_str());
+      if (p->sensorModel == MIBAND) MiBandDiscovery((char*)macWOdots.c_str());
-    if (strcmp(BLEdata["model"].as<const char*>(), "MiBand") == 0) MiBandDiscovery((char*)macWOdots.c_str());
+      if ((p->sensorModel == XMTZC04HM) ||
-    if (strcmp(BLEdata["model"].as<const char*>(), "XMTZC04HM") == 0 ||
+          (p->sensorModel == XMTZC05HM)) MiScaleDiscovery((char*)macWOdots.c_str());
-        strcmp(BLEdata["model"].as<const char*>(), "XMTZC05HM") == 0) MiScaleDiscovery((char*)macWOdots.c_str());
+      if (p->sensorModel == INKBIRD) InkBirdDiscovery((char*)macWOdots.c_str());
-    if (strcmp(BLEdata["model"].as<const char*>(), "INKBIRD") == 0) InkBirdDiscovery((char*)macWOdots.c_str());
+      createOrUpdateDevice(p->macAdr, device_flags_isDisc, p->sensorModel);
-    createOrUpdateDevice(mac, device_flags_isDisc);
     } else {
-    Log.trace(F("Device already discovered or model not detected" CR));
+      Log.trace(F("Device already discovered or UNKNOWN_MODEL" CR));
+    }
   }
 }
 
 void PublishDeviceData(JsonObject& BLEdata) {
   if (abs((int)BLEdata["rssi"] | 0) < minRssi) { // process only the devices close enough
     JsonObject& BLEdataOut = process_bledata(BLEdata);
-#  ifdef ZmqttDiscovery
-    launch_discovery(BLEdataOut);
-#  endif
 #  if !pubBLEServiceUUID
     RemoveJsonPropertyIf(BLEdataOut, "servicedatauuid", BLEdataOut.containsKey("servicedatauuid"));
 #  endif
@@ -739,6 +744,8 @@ void PublishDeviceData(JsonObject& BLEdata) {
 }
 
 JsonObject& process_bledata(JsonObject& BLEdata) {
+  const char* mac = BLEdata["id"].as<const char*>();
+  BLEdevice* device = getDeviceByMac(mac);
   if (BLEdata.containsKey("servicedata")) {
     Log.trace(F("Checking BLE service data validity" CR));
     const char* service_data = (const char*)(BLEdata["servicedata"] | "");
@@ -748,70 +755,80 @@ JsonObject& process_bledata(JsonObject& BLEdata) {
       if (strstr(service_data, "209800") != NULL) {
         Log.trace(F("mi flora data reading" CR));
         BLEdata.set("model", "HHCCJCY01HHCC");
-
+        if (device->sensorModel == -1)
+          createOrUpdateDevice(mac, device_flags_init, HHCCJCY01HHCC);
         return process_sensors(2, BLEdata);
       }
       Log.trace(F("Is it a vegtrug ?" CR));
       if (strstr(service_data, "20bc03") != NULL && strlen(service_data) > ServicedataMinLength) {
         Log.trace(F("vegtrug data reading" CR));
-        BLEdata.set("model", "VegTrug");
+        BLEdata.set("model", "VEGTRUG");
-
+        if (device->sensorModel == -1)
+          createOrUpdateDevice(mac, device_flags_init, VEGTRUG);
         return process_sensors(2, BLEdata);
       }
       Log.trace(F("Is it a LYWSDCGQ?" CR));
       if (strstr(service_data, "20aa01") != NULL && strlen(service_data) > ServicedataMinLength) {
         Log.trace(F("LYWSDCGQ data reading" CR));
         BLEdata.set("model", "LYWSDCGQ");
-
+        if (device->sensorModel == -1)
+          createOrUpdateDevice(mac, device_flags_init, LYWSDCGQ);
         return process_sensors(0, BLEdata);
       }
       Log.trace(F("Is it a JQJCY01YM?" CR));
       if (strstr(service_data, "20df02") != NULL && strlen(service_data) > ServicedataMinLength) {
         Log.trace(F("JQJCY01YM data reading" CR));
         BLEdata.set("model", "JQJCY01YM");
-
+        if (device->sensorModel == -1)
+          createOrUpdateDevice(mac, device_flags_init, JQJCY01YM);
         return process_sensors(0, BLEdata);
       }
       Log.trace(F("Is it a LYWSD02?" CR));
       if (strstr(service_data, "205b04") != NULL && strlen(service_data) > ServicedataMinLength) {
         Log.trace(F("LYWSD02 data reading" CR));
         BLEdata.set("model", "LYWSD02");
-
+        if (device->sensorModel == -1)
+          createOrUpdateDevice(mac, device_flags_init, LYWSD02);
         return process_sensors(2, BLEdata);
       }
       Log.trace(F("Is it a CGG1?" CR));
       if (strstr(service_data, "304703") != NULL && strlen(service_data) > ServicedataMinLength) {
         Log.trace(F("CGG1 data reading method 1" CR));
         BLEdata.set("model", "CGG1");
-
+        if (device->sensorModel == -1)
+          createOrUpdateDevice(mac, device_flags_init, CGG1);
         return process_sensors(0, BLEdata);
       }
       Log.trace(F("Is it a MUE4094RT?" CR));
       if (strstr(service_data, "4030dd") != NULL) {
         Log.trace(F("MUE4094RT data reading" CR));
         BLEdata.set("model", "MUE4094RT");
-
+        if (device->sensorModel == -1)
+          createOrUpdateDevice(mac, device_flags_init, MUE4094RT);
         return process_milamp(BLEdata);
       }
       Log.trace(F("Is it a CGP1W?" CR));
       if (strstr(service_data, "08094c") != NULL && strlen(service_data) > ServicedataMinLength) {
         Log.trace(F("CGP1W data reading" CR));
         BLEdata.set("model", "CGP1W");
-
+        if (device->sensorModel == -1)
+          createOrUpdateDevice(mac, device_flags_init, CGP1W);
         return process_cleargrass(BLEdata, true);
       }
       Log.trace(F("Is it a CGG1" CR));
       if (strstr(service_data, "080774") != NULL) {
         Log.trace(F("CGG1 method 2" CR));
         BLEdata.set("model", "CGG1");
-        // no discovery as it is already available with method 1
+        if (device->sensorModel == -1)
+          createOrUpdateDevice(mac, device_flags_init, CGG1);
         return process_cleargrass(BLEdata, false);
       }
       Log.trace(F("Is it a CGD1?" CR));
       if ((strstr(service_data, "080caf") != NULL || strstr(service_data, "080c09") != NULL) && (strlen(service_data) > ServicedataMinLength)) {
         Log.trace(F("CGD1 data reading" CR));
         BLEdata.set("model", "CGD1");
-
+        if (device->sensorModel == -1)
+          createOrUpdateDevice(mac, device_flags_init, CGD1);
         return process_cleargrass(BLEdata, false);
       }
       if (BLEdata.containsKey("servicedatauuid")) {
@@ -820,22 +837,25 @@ JsonObject& process_bledata(JsonObject& BLEdata) {
         Log.trace(F("Is it a MiBand?" CR));
         if (strstr(service_datauuid, "fee0") != NULL) {
           Log.trace(F("Mi Band data reading" CR));
-          BLEdata.set("model", "MiBand");
+          BLEdata.set("model", "MIBAND");
-
+          if (device->sensorModel == -1)
+            createOrUpdateDevice(mac, device_flags_init, MIBAND);
           return process_miband(BLEdata);
         }
         Log.trace(F("Is it a XMTZC04HM?" CR));
         if (strstr(service_datauuid, "181d") != NULL) {
           Log.trace(F("XMTZC04HM data reading" CR));
           BLEdata.set("model", "XMTZC04HM");
-
+          if (device->sensorModel == -1)
+            createOrUpdateDevice(mac, device_flags_init, XMTZC04HM);
           return process_scale_v1(BLEdata);
         }
         Log.trace(F("Is it a XMTZC05HM?" CR));
         if (strstr(service_datauuid, "181b") != NULL) {
           Log.trace(F("XMTZC05HM data reading" CR));
           BLEdata.set("model", "XMTZC05HM");
-
+          if (device->sensorModel == -1)
+            createOrUpdateDevice(mac, device_flags_init, XMTZC05HM);
           return process_scale_v2(BLEdata);
         }
       }
@@ -860,7 +880,8 @@ JsonObject& process_bledata(JsonObject& BLEdata) {
       if (strcmp(name, "sps") == 0) {
         Log.trace(F("INKBIRD data reading" CR));
         BLEdata.set("model", "INKBIRD");
-
+        if (device->sensorModel == -1)
+          createOrUpdateDevice(mac, device_flags_init, INKBIRD);
         return process_inkbird(BLEdata);
       }
     }

main/config_BT.h

@@ -71,6 +71,24 @@ unsigned int BLEinterval = TimeBtw_Read; //time between 2 scans
 // if not commented Home presence integration with HOME ASSISTANT is activated
 #define subjectHomePresence "home_presence/" // will send Home Assistant room presence message to this topic (first part is same for all rooms, second is room name)
 
+enum ble_sensor_model {
+  UNKNOWN_MODEL = -1,
+  BEGINING = 0,
+  HHCCJCY01HHCC,
+  VEGTRUG,
+  LYWSDCGQ,
+  JQJCY01YM,
+  LYWSD02, //5
+  CGG1,
+  CGP1W,
+  MUE4094RT,
+  CGD1,
+  MIBAND, //10
+  XMTZC04HM,
+  XMTZC05HM,
+  INKBIRD,
+};
+
 /*-------------------PIN DEFINITIONS----------------------*/
 #if !defined(BT_RX) || !defined(BT_TX)
 #  ifdef ESP8266

main/main.ino

@@ -190,9 +190,6 @@ EthernetClient eClient;
 // client link to pubsub mqtt
 PubSubClient client(eClient);
 
-//MQTT last attemps reconnection date
-unsigned long lastMQTTReconnectAttempt = 0;
-
 void revert_hex_data(const char* in, char* out, int l) {
   //reverting array 2 by 2 to get the data in good order
   int i = l - 2, j = 0;
@@ -565,8 +562,6 @@ void setup() {
 
   delay(1500);
 
-  lastMQTTReconnectAttempt = 0;
-
 #ifdef ZsensorBME280
   setupZsensorBME280();
 #endif

platformio.ini

@@ -73,7 +73,7 @@ arduinolog = https://github.com/1technophile/Arduino-Log.git#d13cd80
 pubsubclient = PubSubClient@2.8
 rc-switch = https://github.com/1technophile/rc-switch.git#385a7e0
 newremoteswitch = https://github.com/1technophile/NewRemoteSwitch.git#8eb980e
-ble = https://github.com/eos1d3/ESP32_BLE.git#9d6c1ed
+ble = https://github.com/h2zero/NimBLE-Arduino.git#111ff3a
 irremoteesp = IRremoteESP8266@2.7.7
 irremote = https://github.com/z3t0/Arduino-IRremote.git#1c08a37
 lora = https://github.com/sandeepmistry/arduino-LoRa.git#f4a1d27