[breaking] Refactor the BLE devices identification (#720)

* [breaking] Refactor the BLE devices identification

generalize the use of the mac address with : for the lists, white, black and internal identification
Move structures from config_BT.h to ZgatewayBT.ino
update doc

* Move minRssi to INO and optimizations

change Minrssi name to match other variable convention
Optimize the add of : to mac address
Reduce the use of abs()

Note: I cannot move BLEinterval to the ZgatewayBT.ino due to its use on main.ino

docs/use/ble.md

@@ -60,17 +60,17 @@ OpenMQTTGateway publish the servicedata field of your BLE devices, with HM10 thi
 ## Setting a white or black list
 A black list is a list of mac adresses that will never be published by OMG
 to set black list
-`mosquitto_pub -t home/OpenMQTTGateway/commands/MQTTtoBT/config -m '{"black-list":["012314551615","4C6577889C79","4C65A6663C79"]}'`
+`mosquitto_pub -t home/OpenMQTTGateway/commands/MQTTtoBT/config -m '{"black-list":["01:23:14:55:16:15","4C:65:77:88:9C:79","4C:65:A6:66:3C:79"]}'`
 
 A white list is a list of mac adresses permitted to be published by OMG
 to set white list
-`mosquitto_pub -t home/OpenMQTTGateway/commands/MQTTtoBT/config -m '{"white-list":["012314551615","4C65A5553C79","4C65A6663C79"]}'`
+`mosquitto_pub -t home/OpenMQTTGateway/commands/MQTTtoBT/config -m '{"white-list":["01:23:14:55:16:15","4C:65:77:88:9C:79","4C:65:A6:66:3C:79"]}'`
 
 Note: if you want to filter (white or black list) on BLE sensors that are auto discovered, you need to wait for the discovery before applying the white or black list
 
 ::: tip
 So as to keep your white/black list persistent you can publish it with the retain option of MQTT (-r with mosquitto_pub or retain check box of MQTT Explorer)
-`mosquitto_pub -r -t home/OpenMQTTGateway/commands/MQTTtoBT/config -m '{"white-list":["012314551615","4C65A5553C79","4C65A6663C79"]}'`
+`mosquitto_pub -r -t home/OpenMQTTGateway/commands/MQTTtoBT/config -m '{"white-list":["01:23:14:55:16:15","4C:65:77:88:9C:79","4C:65:A6:66:3C:79"]}'`
 :::
 
 ## Setting the time between scans and force a scan

main/ZgatewayBT.ino

@@ -48,10 +48,33 @@ FreeRTOS::Semaphore semaphoreCreateOrUpdateDevice = FreeRTOS::Semaphore("createO
 
 #  include <vector>
 using namespace std;
+
+struct BLEdevice {
+  char macAdr[18];
+  bool isDisc;
+  bool isWhtL;
+  bool isBlkL;
+};
+
+#  define device_flags_isDisc   1 << 0
+#  define device_flags_isWhiteL 1 << 1
+#  define device_flags_isBlackL 1 << 2
+
+struct decompose {
+  char subject[4];
+  int start;
+  int len;
+  bool reverse;
+  char extract[60];
+};
+
 vector<BLEdevice> devices;
 
 static BLEdevice NO_DEVICE_FOUND = {{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, false, false, false};
 static bool oneWhite = false;
+
+int minRssi = abs(MinimumRSSI); //minimum rssi value
+
 BLEdevice* getDeviceByMac(const char* mac);
 
 BLEdevice* getDeviceByMac(const char* mac) {
@@ -75,7 +98,6 @@ bool updateWorB(JsonObject& BTdata, bool isWhite) {
 
   for (int i = 0; i < size; i++) {
     const char* mac = BTdata[jsonKey][i];
-
     createOrUpdateDevice(mac, (isWhite ? device_flags_isWhiteL : device_flags_isBlackL));
   }
 
@@ -152,7 +174,6 @@ void MiFloraDiscovery(char* mac) {
   };
 
   createDiscoveryFromList(mac, MiFlorasensor, MiFloraparametersCount);
-  createOrUpdateDevice(mac, device_flags_isDisc);
 }
 
 void VegTrugDiscovery(char* mac) {
@@ -169,7 +190,6 @@ void VegTrugDiscovery(char* mac) {
   };
 
   createDiscoveryFromList(mac, VegTrugsensor, VegTrugparametersCount);
-  createOrUpdateDevice(mac, device_flags_isDisc);
 }
 
 void MiJiaDiscovery(char* mac) {
@@ -185,7 +205,6 @@ void MiJiaDiscovery(char* mac) {
   };
 
   createDiscoveryFromList(mac, MiJiasensor, MiJiaparametersCount);
-  createOrUpdateDevice(mac, device_flags_isDisc);
 }
 
 void LYWSD02Discovery(char* mac) {
@@ -201,7 +220,6 @@ void LYWSD02Discovery(char* mac) {
   };
 
   createDiscoveryFromList(mac, LYWSD02sensor, LYWSD02parametersCount);
-  createOrUpdateDevice(mac, device_flags_isDisc);
 }
 
 void CLEARGRASSTRHDiscovery(char* mac) {
@@ -217,7 +235,6 @@ void CLEARGRASSTRHDiscovery(char* mac) {
   };
 
   createDiscoveryFromList(mac, CLEARGRASSTRHsensor, CLEARGRASSTRHparametersCount);
-  createOrUpdateDevice(mac, device_flags_isDisc);
 }
 
 void CLEARGRASSCGD1Discovery(char* mac) {
@@ -233,7 +250,6 @@ void CLEARGRASSCGD1Discovery(char* mac) {
   };
 
   createDiscoveryFromList(mac, CLEARGRASSCGD1sensor, CLEARGRASSCGD1parametersCount);
-  createOrUpdateDevice(mac, device_flags_isDisc);
 }
 
 void CLEARGRASSTRHKPADiscovery(char* mac) {
@@ -249,7 +265,6 @@ void CLEARGRASSTRHKPADiscovery(char* mac) {
   };
 
   createDiscoveryFromList(mac, CLEARGRASSTRHKPAsensor, CLEARGRASSTRHKPAparametersCount);
-  createOrUpdateDevice(mac, device_flags_isDisc);
 }
 
 void MiScaleDiscovery(char* mac) {
@@ -261,7 +276,6 @@ void MiScaleDiscovery(char* mac) {
   };
 
   createDiscoveryFromList(mac, MiScalesensor, MiScaleparametersCount);
-  createOrUpdateDevice(mac, device_flags_isDisc);
 }
 
 void MiLampDiscovery(char* mac) {
@@ -273,7 +287,6 @@ void MiLampDiscovery(char* mac) {
   };
 
   createDiscoveryFromList(mac, MiLampsensor, MiLampparametersCount);
-  createOrUpdateDevice(mac, device_flags_isDisc);
 }
 
 void MiBandDiscovery(char* mac) {
@@ -285,7 +298,6 @@ void MiBandDiscovery(char* mac) {
   };
 
   createDiscoveryFromList(mac, MiBandsensor, MiBandparametersCount);
-  createOrUpdateDevice(mac, device_flags_isDisc);
 }
 
 void InkBirdDiscovery(char* mac) {
@@ -301,7 +313,6 @@ void InkBirdDiscovery(char* mac) {
   };
 
   createDiscoveryFromList(mac, InkBirdsensor, InkBirdparametersCount);
-  createOrUpdateDevice(mac, device_flags_isDisc);
 }
 
 #  else
@@ -346,12 +357,8 @@ class MyAdvertisedDeviceCallbacks : public BLEAdvertisedDeviceCallbacks {
     String mac_adress = advertisedDevice.getAddress().toString().c_str();
     mac_adress.toUpperCase();
     BLEdata.set("id", (char*)mac_adress.c_str());
-    mac_adress.replace(":", "");
-    String mactopic = subjectBTtoMQTT + String("/") + mac_adress;
-    char mac[mac_adress.length() + 1];
-    mac_adress.toCharArray(mac, mac_adress.length() + 1);
-    Log.notice(F("Device detected: %s" CR), mac);
-    BLEdevice* device = getDeviceByMac(mac);
+    Log.notice(F("Device detected: %s" CR), (char*)mac_adress.c_str());
+    BLEdevice* device = getDeviceByMac(BLEdata["id"].as<const char*>());
 
     if ((!oneWhite || isWhite(device)) && !isBlack(device)) { //if not black listed mac we go AND if we have no white mac or this mac is  white we go out
       if (advertisedDevice.haveName())
@@ -503,8 +510,8 @@ void changelow_power_mode(int newLowPowerMode) {
 }
 
 void setupBT() {
-  Log.notice(F("BLEinterval: %d" CR), BLEinterval);
-  Log.notice(F("Minrssi: %d" CR), Minrssi);
+  Log.notice(F("BLE interval: %d" CR), BLEinterval);
+  Log.notice(F("minrssi: %d" CR), minRssi);
   Log.notice(F("Low Power Mode: %d" CR), low_power_mode);
 
   // we setup a task with priority one to avoid conflict with other gateways
@@ -537,9 +544,8 @@ struct decompose d[6] = {{"mac", 16, 12, true}, {"typ", 28, 2, false}, {"rsi", 3
 
 void setupBT() {
   BLEinterval = TimeBtw_Read;
-  Minrssi = MinimumRSSI;
-  Log.notice(F("BLEinterval: %d" CR), BLEinterval);
-  Log.notice(F("Minrssi: %d" CR), Minrssi);
+  Log.notice(F("BLE interval: %d" CR), BLEinterval);
+  Log.notice(F("minrssi: %d" CR), minRssi);
   softserial.begin(HMSerialSpeed);
   softserial.print(F("AT+ROLE1" CR));
   delay(100);
@@ -594,23 +600,21 @@ bool BTtoMQTT() {
             StaticJsonBuffer<JSON_MSG_BUFFER> jsonBuffer;
             JsonObject& BLEdata = jsonBuffer.createObject();
             strupp(d[0].extract);
-
-            BLEdevice* device = getDeviceByMac(d[0].extract);
+            String Id;
+            for (int i = 0; i < 12; i = i + 2) {
+              Id += String(d[0].extract[i]);
+              Id += String(d[0].extract[i + 1]);
+              if (i != 6)
+                Id += ":";
+            }
+            Log.trace(F("Id %s" CR), (char*)Id.c_str());
+            BLEdata.set("id", (char*)Id.c_str());
+            BLEdevice* device = getDeviceByMac((char*)Id.c_str());
 
             if (isBlack(device))
               return false; //if black listed mac we go out
             if (oneWhite && !isWhite(device))
               return false; //if we have at least one white mac and this mac is not white we go out
-#    ifdef subjectHomePresence
-            String HomePresenceId;
-            for (int i = 0; i < 12; i++) {
-              HomePresenceId += String(d[0].extract[i]);
-              if (((i - 1) % 2 == 0) && (i != 11))
-                HomePresenceId += ":";
-            }
-            Log.trace(F("HomePresenceId %s" CR), (char*)HomePresenceId.c_str());
-            BLEdata.set("id", (char*)HomePresenceId.c_str());
-#    endif
             int rssi = (int)strtol(d[2].extract, NULL, 16) - 256;
             BLEdata.set("rssi", (int)rssi);
 #    ifdef subjectHomePresence
@@ -664,36 +668,36 @@ boolean valid_service_data(const char* data) {
   return false;
 }
 
-void launch_discovery(JsonObject& BLEdata, char* mac) {
+void launch_discovery(JsonObject& BLEdata) {
+  const char* mac = BLEdata["id"].as<const char*>();
   BLEdevice* device = getDeviceByMac(mac);
+  String macWOdots = String(mac);
+  macWOdots.replace(":", "");
   if (!isDiscovered(device) && BLEdata.containsKey("model")) {
     Log.trace(F("Launching discovery of %s" CR), mac);
-    if (strcmp(BLEdata["model"].as<const char*>(), "HHCCJCY01HHCC") == 0) MiFloraDiscovery(mac);
-    if (strcmp(BLEdata["model"].as<const char*>(), "VegTrug") == 0) VegTrugDiscovery(mac);
-    if (strcmp(BLEdata["model"].as<const char*>(), "LYWSDCGQ") == 0) MiJiaDiscovery(mac);
-    if (strcmp(BLEdata["model"].as<const char*>(), "LYWSD02") == 0) LYWSD02Discovery(mac);
-    if (strcmp(BLEdata["model"].as<const char*>(), "CGG1") == 0) CLEARGRASSTRHDiscovery(mac);
-    if (strcmp(BLEdata["model"].as<const char*>(), "CGP1W") == 0) CLEARGRASSTRHKPADiscovery(mac);
-    if (strcmp(BLEdata["model"].as<const char*>(), "MUE4094RT") == 0) MiLampDiscovery(mac);
-    if (strcmp(BLEdata["model"].as<const char*>(), "CGD1") == 0) CLEARGRASSCGD1Discovery(mac);
-    if (strcmp(BLEdata["model"].as<const char*>(), "MiBand") == 0) MiBandDiscovery(mac);
+    if (strcmp(BLEdata["model"].as<const char*>(), "HHCCJCY01HHCC") == 0) MiFloraDiscovery((char*)macWOdots.c_str());
+    if (strcmp(BLEdata["model"].as<const char*>(), "VegTrug") == 0) VegTrugDiscovery((char*)macWOdots.c_str());
+    if (strcmp(BLEdata["model"].as<const char*>(), "LYWSDCGQ") == 0) MiJiaDiscovery((char*)macWOdots.c_str());
+    if (strcmp(BLEdata["model"].as<const char*>(), "LYWSD02") == 0) LYWSD02Discovery((char*)macWOdots.c_str());
+    if (strcmp(BLEdata["model"].as<const char*>(), "CGG1") == 0) CLEARGRASSTRHDiscovery((char*)macWOdots.c_str());
+    if (strcmp(BLEdata["model"].as<const char*>(), "CGP1W") == 0) CLEARGRASSTRHKPADiscovery((char*)macWOdots.c_str());
+    if (strcmp(BLEdata["model"].as<const char*>(), "MUE4094RT") == 0) MiLampDiscovery((char*)macWOdots.c_str());
+    if (strcmp(BLEdata["model"].as<const char*>(), "CGD1") == 0) CLEARGRASSCGD1Discovery((char*)macWOdots.c_str());
+    if (strcmp(BLEdata["model"].as<const char*>(), "MiBand") == 0) MiBandDiscovery((char*)macWOdots.c_str());
     if (strcmp(BLEdata["model"].as<const char*>(), "XMTZC04HM") == 0 ||
-        strcmp(BLEdata["model"].as<const char*>(), "XMTZC05HM") == 0) MiScaleDiscovery(mac);
-    if (strcmp(BLEdata["model"].as<const char*>(), "INKBIRD") == 0) InkBirdDiscovery(mac);
+        strcmp(BLEdata["model"].as<const char*>(), "XMTZC05HM") == 0) MiScaleDiscovery((char*)macWOdots.c_str());
+    if (strcmp(BLEdata["model"].as<const char*>(), "INKBIRD") == 0) InkBirdDiscovery((char*)macWOdots.c_str());
+    createOrUpdateDevice(mac, device_flags_isDisc);
   } else {
     Log.trace(F("Device already discovered or model not detected" CR));
   }
 }
 
 void PublishDeviceData(JsonObject& BLEdata) {
-  if (abs((int)BLEdata["rssi"] | 0) < abs(Minrssi)) { // process only the devices close enough
+  if (abs((int)BLEdata["rssi"] | 0) < minRssi) { // process only the devices close enough
     JsonObject& BLEdataOut = process_bledata(BLEdata);
-    String mac_adress = BLEdataOut["id"].as<const char*>();
-    mac_adress.replace(":", "");
-    char mac[mac_adress.length() + 1];
-    mac_adress.toCharArray(mac, mac_adress.length() + 1);
 #  ifdef ZmqttDiscovery
-    launch_discovery(BLEdataOut, mac);
+    launch_discovery(BLEdataOut);
 #  endif
 #  if !pubBLEServiceUUID
     RemoveJsonPropertyIf(BLEdataOut, "servicedatauuid", BLEdataOut.containsKey("servicedatauuid"));
@@ -701,7 +705,8 @@ void PublishDeviceData(JsonObject& BLEdata) {
 #  if !pubKnownBLEServiceData
     RemoveJsonPropertyIf(BLEdataOut, "servicedata", BLEdataOut.containsKey("model") && BLEdataOut.containsKey("servicedata"));
 #  endif
-    String mactopic(mac);
+    String mactopic = BLEdataOut["id"].as<const char*>();
+    mactopic.replace(":", "");
     mactopic = subjectBTtoMQTT + String("/") + mactopic;
     pub((char*)mactopic.c_str(), BLEdataOut);
   } else {
@@ -1035,10 +1040,10 @@ void MQTTtoBT(char* topicOri, JsonObject& BTdata) { // json object decoding
     // MinRSSI set
     if (BTdata.containsKey("minrssi")) {
       // storing Min RSSI for further use if needed
-      Log.trace(F("Previous Minrssi: %d" CR), Minrssi);
+      Log.trace(F("Previous minrssi: %d" CR), minRssi);
       // set Min RSSI if present if not setting default value
-      Minrssi = (unsigned int)BTdata["minrssi"];
-      Log.notice(F("New Minrssi: %d" CR), Minrssi);
+      minRssi = abs((unsigned int)BTdata["minrssi"]);
+      Log.notice(F("New minrssi: %d" CR), minRssi);
     }
 #  ifdef ESP32
     if (BTdata.containsKey("low_power_mode")) {

main/config_BT.h

@@ -45,6 +45,8 @@ extern void MQTTtoBT(char* topicOri, JsonObject& RFdata);
 #  define TimeBtw_Read 55555 //define default time between 2 scans
 #endif
 
+unsigned int BLEinterval = TimeBtw_Read; //time between 2 scans
+
 #ifndef pubKnownBLEServiceData
 #  define pubKnownBLEServiceData false // define true if you want to publish service data belonging to recognised sensors
 #endif
@@ -65,28 +67,6 @@ extern void MQTTtoBT(char* topicOri, JsonObject& RFdata);
 // if not commented Home presence integration with HOME ASSISTANT is activated
 #define subjectHomePresence "home_presence/" // will send Home Assistant room presence message to this topic (first part is same for all rooms, second is room name)
 
-unsigned int BLEinterval = TimeBtw_Read; //time between 2 scans
-int Minrssi = MinimumRSSI; //minimum rssi value
-
-struct BLEdevice {
-  char macAdr[13];
-  bool isDisc;
-  bool isWhtL;
-  bool isBlkL;
-};
-
-#define device_flags_isDisc   1 << 0
-#define device_flags_isWhiteL 1 << 1
-#define device_flags_isBlackL 1 << 2
-
-struct decompose {
-  char subject[4];
-  int start;
-  int len;
-  bool reverse;
-  char extract[60];
-};
-
 /*-------------------PIN DEFINITIONS----------------------*/
 #if !defined(BT_RX) || !defined(BT_TX)
 #  ifdef ESP8266