Refactor BLE gateways (#617)

* Refactor BLE gateways:
-merge HM10 and ESP32 code
-publish sensor values with the other BLE device parameters to avoid MQTT multiple publication
-make uniform the case of mac adress id between HM10 and ESP32
-add the possibility to remove service data publishing either when sensors is recognised or not
-correct manufacturing data display
-merge process cleargrass methods
-check service data validity before processing

docs/prerequisites/devices.md

@@ -17,13 +17,16 @@ OpenMQTTGateway is able to scan all the BLE devices that advertise their data so
 |-|:-:|:-:|
 | BLE watches with fixed mac||rssi for presence detection|
 | BLE beacons keychains||rssi for presence detection|
-| XIAOMI Mi Flora ||temperature/moisture/luminance/fertility|
+| Vegtrug ||temperature/moisture/luminance/fertility|
+| XIAOMI Mi Flora |HHCCJCY01HHCC|temperature/moisture/luminance/fertility|
 | XIAOMI Mi Jia |LYWSDCGO|temperature/humidity/battery|
+| XIAOMI Mi Lamp |MUE4094RT|presence|
 | ClearGrass |CGG1|temperature/humidity/battery|
 | ClearGrass alarm clock|CGD1|temperature/humidity|
 | ClearGrass with atmospheric pressure |CGP1W|temperature/humidity/air pressure|
 | Clock |LYWDS02|temperature/humidity|
-| XIAOMI Mi Scale ||weight|
+| XIAOMI Mi Scale v1|XMTZC04HM|weight|
+| XIAOMI Mi Scale v2|XMTZC05HM|weight|
 | XIAOMI Mi band ||steps|
 
 

docs/use/ble.md

@@ -39,6 +39,9 @@ Note that the gateway return one or two measurement value each time. The differe
 * Temperature
 * Moisture
 * Fertilization
+* Humidity
+* Pressure
+* Steps
 * Weight
 * Impedance
 * Battery (mi jia only)
@@ -49,6 +52,11 @@ The infos will appear like this on your MQTT broker:
 
 More info are available on [my blog](https://1technophile.blogspot.fr/2017/11/mi-flora-integration-to-openmqttgateway.html)  (especially about how it was implemented with HM10)
 
+::: tip
+The HM10 module doesn't read enough information (servicedata UUID is missing) to support Mi Scale and Mi Band. They are supported nevertheless with ESP32.
+OpenMQTTGateway publish the servicedata field of your BLE devices, with HM10 this field can be longer compared to ESP32 if the device is not recognised.
+:::
+
 ## Setting a white or black list
 A black list is a list of mac adresses that will never be published by OMG
 to set black list

main/ZgatewayBT.ino

@@ -51,6 +51,8 @@ static bool oneWhite = false;
 
 BLEdevice * getDeviceByMac(const char *mac)
 {
+  Log.trace(F("getDeviceByMac %s" CR), mac);
+
   for (vector<BLEdevice>::iterator p = devices.begin(); p != devices.end(); ++p)
   {
     if ((strcmp(p->macAdr, mac) == 0))
@@ -63,6 +65,7 @@ BLEdevice * getDeviceByMac(const char *mac)
 
 bool updateWorB(JsonObject &BTdata, bool isWhite)
 {
+  Log.trace(F("update WorB" CR));
   const char* jsonKey = isWhite ? "white-list" : "black-list";
 
   int size = BTdata[jsonKey].size();
@@ -327,14 +330,13 @@ class MyAdvertisedDeviceCallbacks : public BLEAdvertisedDeviceCallbacks
     StaticJsonBuffer<JSON_MSG_BUFFER> jsonBuffer;
     JsonObject &BLEdata = jsonBuffer.createObject();
     String mac_adress = advertisedDevice.getAddress().toString().c_str();
+    mac_adress.toUpperCase();
     BLEdata.set("id", (char *)mac_adress.c_str());
     mac_adress.replace(":", "");
-    mac_adress.toUpperCase();
     String mactopic = subjectBTtoMQTT + String("/") + mac_adress;
     char mac[mac_adress.length() + 1];
     mac_adress.toCharArray(mac, mac_adress.length() + 1);
-    Log.notice(F("Device detected: %s" CR),mac);
+    Log.notice(F("Device detected: %s" CR), mac);
-
     BLEdevice *device = getDeviceByMac(mac);
 
     if ((!oneWhite || isWhite(device)) && !isBlack(device))
@@ -343,7 +345,11 @@ class MyAdvertisedDeviceCallbacks : public BLEAdvertisedDeviceCallbacks
         BLEdata.set("name", (char *)advertisedDevice.getName().c_str());
       #if pubBLEManufacturerData
       if (advertisedDevice.haveManufacturerData())
-        BLEdata.set("manufacturerdata", (char *)advertisedDevice.getManufacturerData().c_str());
+      {
+        char* manufacturerdata = BLEUtils::buildHexData(NULL, (uint8_t*)advertisedDevice.getManufacturerData().data(), advertisedDevice.getManufacturerData().length());
+        Log.trace(F("Manufacturer Data: %s" CR), manufacturerdata);
+        BLEdata.set("manufacturerdata", manufacturerdata);
+      }
       #endif
       if (advertisedDevice.haveRSSI())
         BLEdata.set("rssi", (int)advertisedDevice.getRSSI());
@@ -356,7 +362,7 @@ class MyAdvertisedDeviceCallbacks : public BLEAdvertisedDeviceCallbacks
       if (advertisedDevice.haveServiceData())
       {
         int serviceDataCount = advertisedDevice.getServiceDataCount();
-        Log.trace(F("Get services data number: %d" CR),serviceDataCount);
+        Log.trace(F("Get services data number: %d" CR), serviceDataCount);
         for (int j = 0; j < serviceDataCount; j++)
         {
           std::string serviceData = advertisedDevice.getServiceData(j);
@@ -374,163 +380,17 @@ class MyAdvertisedDeviceCallbacks : public BLEAdvertisedDeviceCallbacks
           char service_data[returnedString.length() + 1];
           returnedString.toCharArray(service_data, returnedString.length() + 1);
           service_data[returnedString.length()] = '\0';
-          Log.notice(F("Service data: %s" CR),service_data);
+          Log.trace(F("Service data: %s" CR), service_data);
-          #ifdef pubBLEServiceData
           BLEdata.set("servicedata", service_data);
-          #endif
+          std::string serviceDatauuid = advertisedDevice.getServiceDataUUID(j).toString();
-          BLEdata.set("servicedatauuid", (char *)advertisedDevice.getServiceDataUUID(j).toString().c_str());
+          Log.trace(F("Service data UUID: %s" CR), (char *)serviceDatauuid.c_str());
-          if (abs((int)BLEdata["rssi"] | 0) < abs(Minrssi))
+          BLEdata.set("servicedatauuid", (char *)serviceDatauuid.c_str());
-          { // publish only the devices close enough
+          PublishDeviceData(BLEdata);
-            pub((char *)mactopic.c_str(), BLEdata);
-            if (strstr(BLEdata["servicedatauuid"].as<char *>(), "fe95") != NULL)
-            { //Mi FLora, Mi jia, Cleargrass Method 1, LYWSD02, VegTrug
-              Log.trace(F("Processing BLE device data" CR));
-              int pos = -1;
-              pos = strpos(service_data, "209800");
-              if (pos != -1)
-              {
-                Log.trace(F("mi flora data reading" CR));
-                //example "servicedata":"71209800bc63b6658d7cc40d0910023200"
-                
-                if (!isDiscovered(device))
-                  MiFloraDiscovery(mac);
-
-                process_sensors(pos - 24, service_data, mac);
-              }
-              pos = -1;
-              pos = strpos(service_data, "20bc03");
-              if (pos != -1)
-              {
-                Log.trace(F("vegtrug data reading" CR));
-                //example "servicedata":"7120bc0399c309688d7cc40d0910020000"
-
-                if (!isDiscovered(device))
-                  VegTrugDiscovery(mac);
-
-                process_sensors(pos - 24, service_data, mac);
-              }
-              pos = -1;
-              pos = strpos(service_data, "20aa01");
-              if (pos != -1)
-              {
-                Log.trace(F("mi jia data reading" CR));
-
-                if (!isDiscovered(device))
-                  MiJiaDiscovery(mac);
-
-                process_sensors(pos - 26, service_data, mac);
-              }
-              pos = -1;
-              pos = strpos(service_data, "205b04");
-              if (pos != -1)
-              {
-                Log.trace(F("LYWSD02 data reading" CR));
-                //example "servicedata":"70205b04b96ab883c8593f09041002e000"
-
-                if (!isDiscovered(device))
-                  LYWSD02Discovery(mac);
-
-                process_sensors(pos - 24, service_data, mac);
-              }
-              pos = -1;
-              pos = strpos(service_data, "304703");
-              if (pos != -1)
-              {
-                Log.trace(F("ClearGrass T RH data reading method 1" CR));
-                //example "servicedata":"5030470340743e10342d58041002d6000a100164"
-
-                if (!isDiscovered(device))
-                  CLEARGRASSTRHDiscovery(mac);
-
-                process_sensors(pos - 26, service_data, mac);
-              }
-              pos = -1;
-              pos = strpos(service_data, "4030dd");
-              if (pos != -1)
-              {
-                Log.trace(F("Mi Lamp data reading" CR));
-                //example "servicedata":4030DD031D0300010100
-
-                if (!isDiscovered(device))
-                  MiLampDiscovery(mac);
-
-                process_milamp(service_data, mac);
-              }
-            }
-            if (strstr(BLEdata["servicedatauuid"].as<char *>(), "181d") != NULL)
-            { // Mi Scale V1
-              Log.trace(F("Mi Scale V1 data reading" CR));
-              //example "servicedata":"a2ac2be307060207122b" /"a28039e3070602070e28"
-
-              if (!isDiscovered(device))
-                MiScaleDiscovery(mac);
-
-              process_scale_v1(service_data, mac);
-            }
-            if (strstr(BLEdata["servicedatauuid"].as<char *>(), "181b") != NULL)
-            { // Mi Scale V2
-              Log.trace(F("Mi Scale V2 data reading" CR));
-              //example "servicedata":02c4e1070b1e13050c00002607 / 02a6e20705150a251df401443e /02a6e20705180c0d04d701943e
-
-              if (!isDiscovered(device))
-                MiScaleDiscovery(mac);
-
-              process_scale_v2(service_data, mac);
-            }
-            if (strstr(BLEdata["servicedatauuid"].as<char *>(), "fee0") != NULL)
-            { // Mi Band //0000fee0-0000-1000-8000-00805f9b34fb // ESP32 only
-              Log.trace(F("Mi Band data reading" CR));
-              //example "servicedata":a21e0000
-
-              if (!isDiscovered(device))
-                MiBandDiscovery(mac);
-
-              process_miband(service_data, mac);
-            }
-            if (strstr(BLEdata["servicedata"].as<char *>(), "08094c") != NULL)
-            { // Clear grass with air pressure//08094c0140342d580104d8000c020702612702015a
-              Log.trace(F("Clear grass data with air pressure reading" CR));
-              //example "servicedata":08094c0140342d580104 c400 2402 0702 5d27 02015a
-
-              if (!isDiscovered(device))
-                CLEARGRASSTRHKPADiscovery(mac);
-
-              process_cleargrass_air(service_data, mac);
-            }
-            if (strstr(BLEdata["servicedata"].as<char *>(), "080774") != NULL)
-            { // Clear grass standard method 2/0807743e10342d580104c3002c0202012a
-              Log.trace(F("Clear grass data reading method 2" CR));
-              //example "servicedata":0807743e10342d580104 c300 2c02 02012a
-              // no discovery as it is already available with method 1
-              process_cleargrass(service_data, mac);
-            }
-            if (strstr(BLEdata["servicedata"].as<char *>(), "080caf") != NULL)
-            { // Clear grass CGD1 080caffd50342d580104c900a102
-              Log.trace(F("Clear grass CGD1 data reading" CR));
-              //example "servicedata":080caffd50342d580104 c900 a102
-
-              if (!isDiscovered(device))
-                CLEARGRASSCGD1Discovery(mac);
-
-              process_cleargrass(service_data, mac);
-            }
-          }
-          else
-          {
-            Log.trace(F("Low rssi, device filtered" CR));
-          }
         }
       }
       else
       {
-        if (abs((int)BLEdata["rssi"] | 0) < abs(Minrssi))
+        PublishDeviceData(BLEdata); // publish device even if there is no service data
-        {                                         // publish only the devices close enough
-          pub((char *)mactopic.c_str(), BLEdata); // publish device even if there is no service data
-        }
-        else
-        {
-          Log.trace(F("Low rssi, device filtered" CR));
-        }
       }
     }
     else
@@ -542,7 +402,6 @@ class MyAdvertisedDeviceCallbacks : public BLEAdvertisedDeviceCallbacks
 
 void BLEscan()
 {
-
   TIMERG0.wdt_wprotect = TIMG_WDT_WKEY_VALUE;
   TIMERG0.wdt_feed = 1;
   TIMERG0.wdt_wprotect = 0;
@@ -673,14 +532,14 @@ String returnedString = "";
 unsigned long timebt = 0;
 
 // this struct define which parts of the hexadecimal chain we extract and what to do with these parts
-struct decompose d[6] = {{"mac", 16, 12, true}, {"typ", 28, 2, false}, {"rsi", 30, 2, false}, {"rdl", 32, 2, false}, {"sty", 44, 4, true}, {"rda", 34, 60, false}};
+struct decompose d[6] = {{"mac", 16, 12, true}, {"typ", 28, 2, false}, {"rsi", 30, 2, false}, {"rdl", 32, 2, false}, {"sty", 44, 4, true}, {"rda", 56, 60, false}};
 
 void setupBT()
 {
   BLEinterval = TimeBtw_Read;
   Minrssi = MinimumRSSI;
-  Log.notice(F("BLEinterval: %d" CR),BLEinterval);
+  Log.notice(F("BLEinterval: %d" CR), BLEinterval);
-  Log.notice(F("Minrssi: %d" CR),Minrssi);
+  Log.notice(F("Minrssi: %d" CR), Minrssi);
   softserial.begin(HMSerialSpeed);
   softserial.print(F("AT+ROLE1" CR));
   delay(100);
@@ -709,13 +568,14 @@ bool BTtoMQTT()
   }
 
   if (millis() > (timebt + BLEinterval))
-  { //retriving data
+  { //retrieving data
     timebt = millis();
     #if defined(ESP8266)
     yield();
     #endif
     if (returnedString != "")
     {
+      Log.verbose(F("returnedString: %s" CR), (char *)returnedString.c_str());
       size_t pos = 0;
       while ((pos = returnedString.lastIndexOf(BLEdelimiter)) != -1)
       {
@@ -726,9 +586,9 @@ bool BTtoMQTT()
         returnedString.remove(pos, returnedString.length());
         char token_char[token.length() + 1];
         token.toCharArray(token_char, token.length() + 1);
-        Log.trace(F("Token: %s" CR),token_char);
+        Log.trace(F("Token: %s" CR), token_char);
         if (token.length() > 60)
-        { // we extract data only if we have detailled infos
+        { // we extract data only if we have infos (BLEdelimiter length + 4)
           for (int i = 0; i < 6; i++)
           {
             extract_char(token_char, d[i].extract, d[i].start, d[i].len, d[i].reverse, false);
@@ -736,7 +596,7 @@ bool BTtoMQTT()
               d[5].len = (int)strtol(d[i].extract, NULL, 16) * 2; // extracting the length of the rest data
           }
 
-          if ((strlen(d[0].extract)) == 12) // if a mac adress is detected we publish it
+          if ((strlen(d[0].extract)) == 12) // if a BLE device is detected we analyse it
           {
             Log.trace(F("Creating BLE buffer" CR));
             StaticJsonBuffer<JSON_MSG_BUFFER> jsonBuffer;
@@ -760,130 +620,14 @@ bool BTtoMQTT()
             Log.trace(F("HomePresenceId %s" CR),(char *)HomePresenceId.c_str());
             BLEdata.set("id", (char *)HomePresenceId.c_str());
             #endif
-            String topic = subjectBTtoMQTT + String("/") + String(d[0].extract);
             int rssi = (int)strtol(d[2].extract, NULL, 16) - 256;
             BLEdata.set("rssi", (int)rssi);
             #ifdef subjectHomePresence
             haRoomPresence(BLEdata); // this device has an rssi in consequence we can use it for home assistant room presence component
             #endif
-            String service_data(d[5].extract);
+            Log.trace(F("Service data: %s" CR), d[5].extract);
-            Log.notice(F("Service data: %s" CR),d[5].extract);
+            BLEdata.set("servicedata", d[5].extract);
-            service_data = service_data.substring(14);
+            PublishDeviceData(BLEdata);
-            #ifdef pubBLEServiceData
-            BLEdata.set("servicedata", (char *)service_data.c_str());
-            #endif
-            if (abs((int)BLEdata["rssi"] | 0) < abs(Minrssi))
-            { // publish only the devices close enough
-              pub((char *)topic.c_str(), BLEdata);
-              int pos = -1;
-              pos = strpos(d[5].extract, "209800");
-              if (pos != -1)
-              {
-                Log.trace(F("mi flora data reading" CR));
-
-                if (!isDiscovered(device))
-                  MiFloraDiscovery(d[0].extract);
-
-                bool result = process_sensors(pos - 38, (char *)service_data.c_str(), d[0].extract);
-              }
-              pos = -1;
-              pos = strpos(d[5].extract, "20aa01");
-              //example "servicedata":"5020aa0194dfaa33342d580d1004e3002c02"
-              if (pos != -1)
-              {
-                Log.trace(F("mi jia data reading" CR));
-
-                if (!isDiscovered(device))
-                  MiJiaDiscovery(d[0].extract);
-
-                bool result = process_sensors(pos - 40, (char *)service_data.c_str(), d[0].extract);
-              }
-              pos = -1;
-              pos = strpos(d[5].extract, "205b04");
-              //example "servicedata":"141695fe70205b0461298882c8593f09061002d002"
-              if (pos != -1)
-              {
-                Log.trace(F("LYWSD02 data reading" CR));
-
-                if (!isDiscovered(device))
-                  LYWSD02Discovery(d[0].extract);
-
-                bool result = process_sensors(pos - 38, (char *)service_data.c_str(), d[0].extract);
-              }
-              pos = -1;
-              pos = strpos(d[5].extract, "304703");
-              if (pos != -1)
-              {
-                Log.trace(F("CLEARGRASSTRH data reading method 1" CR));
-
-                if (!isDiscovered(device))
-                  CLEARGRASSTRHDiscovery(d[0].extract);
-
-                bool result = process_sensors(pos - 40, (char *)service_data.c_str(), d[0].extract);
-              }
-              pos = -1;
-              pos = strpos(d[5].extract, "e30706");
-              if (pos != -1)
-              {
-                Log.trace(F("Mi Scale data reading" CR));
-                //example "servicedata":"a2ac2be307060207122b" /"a28039e3070602070e28"
-
-                if (!isDiscovered(device))
-                  MiScaleDiscovery(d[0].extract);
-
-                bool result = process_scale_v1((char *)service_data.c_str(), d[0].extract);
-              }
-              pos = -1;
-              pos = strpos(d[5].extract, "4030dd");
-              if (pos != -1)
-              {
-                Log.trace(F("Mi Lamp data reading" CR));
-                //example "servicedata":4030dd31d0300010100
-
-                if (!isDiscovered(device))
-                  MiLampDiscovery(d[0].extract);
-
-                process_milamp((char *)service_data.c_str(), d[0].extract);
-              }
-              pos = -1;
-              pos = strpos(d[5].extract, "08094c"); // Clear grass with air pressure//08094c0140342d580104d8000c020702612702015a
-              if (pos != -1)
-              {
-                Log.trace(F("Clear grass data with air pressure reading" CR));
-                //example "servicedata":08094c0140342d580104 c400 2402 0702 5d27 02015a
-
-                if (!isDiscovered(device))
-                  CLEARGRASSTRHKPADiscovery(d[0].extract);
-
-                process_cleargrass_air((char *)service_data.c_str(), d[0].extract);
-              }
-              pos = -1;
-              pos = strpos(d[5].extract, "080774"); // Clear grass standard method 2/0807743e10342d580104c3002c0202012a
-              if (pos != -1)
-              {
-                Log.trace(F("Clear grass data reading method 2" CR));
-                //example "servicedata":0807743e10342d580104 c300 2c02 02012a
-                // no discovery as it is already available with method 1
-                process_cleargrass((char *)service_data.c_str(), d[0].extract);
-              }
-              pos = strpos(d[5].extract, "080caf"); // Clear grass CGD1 08094c0140342d580104d8000c020702612702015a
-              if (pos != -1)
-              {
-                Log.trace(F("Clear grass data CGD1" CR));
-                //example "servicedata":080caffd50342d580104 c900 a102
-
-                if (!isDiscovered(device))
-                  CLEARGRASSCGD1Discovery(d[0].extract);
-
-                process_cleargrass((char *)service_data.c_str(), d[0].extract);
-              }
-
-              return true;
-            }
-            else
-            {
-              Log.trace(F("Low rssi, device filtered" CR));
-            }
           }
         }
       }
@@ -900,7 +644,15 @@ bool BTtoMQTT()
 }
 #endif
 
-double value_from_service_data(char *service_data, int offset, int data_length)
+void RemoveJsonPropertyIf(JsonObject &obj, char* key, bool condition)
+{
+  if(condition){
+     Log.trace(F("Removing %s" CR), key);
+     obj.remove(key);
+  }
+}
+
+double value_from_service_data(const char *service_data, int offset, int data_length)
 {
   char rev_data[data_length + 1];
   char data[data_length + 1];
@@ -916,35 +668,219 @@ double value_from_service_data(char *service_data, int offset, int data_length)
   return value;
 }
 
-bool process_sensors(int offset, char *rest_data, char *mac_adress)
+boolean valid_service_data(const char * data)
 {
+  int size = strlen(data);
+  for (int i = 0; i < size; ++i) {
+    if(data[i] != 48) // 48 correspond to 0 in ASCII table
+      return true;
+  }
+  return false;
+}
 
-  Log.trace(F("Creating BLE buffer" CR));
+void launch_discovery(JsonObject &BLEdata, char * mac){
-  StaticJsonBuffer<JSON_MSG_BUFFER> jsonBuffer;
+  BLEdevice *device = getDeviceByMac(mac);
-  JsonObject &BLEdata = jsonBuffer.createObject();
+  if (!isDiscovered(device) && BLEdata.containsKey("model"))
+  {
+    Log.trace(F("Launching discovery of %s" CR), mac);
+    if(strcmp(BLEdata["model"].as<const char*>(), "HHCCJCY01HHCC") == 0)   MiFloraDiscovery(mac);
+    if(strcmp(BLEdata["model"].as<const char*>(), "VegTrug") == 0)         VegTrugDiscovery(mac);
+    if(strcmp(BLEdata["model"].as<const char*>(), "LYWSDCGQ") == 0)        MiJiaDiscovery(mac);
+    if(strcmp(BLEdata["model"].as<const char*>(), "LYWSD02") == 0)         LYWSD02Discovery(mac);
+    if(strcmp(BLEdata["model"].as<const char*>(), "CGG1") == 0)            CLEARGRASSTRHDiscovery(mac);
+    if(strcmp(BLEdata["model"].as<const char*>(), "CGP1W") == 0)           CLEARGRASSTRHKPADiscovery(mac);
+    if(strcmp(BLEdata["model"].as<const char*>(), "MUE4094RT") == 0)       MiLampDiscovery(mac);
+    if(strcmp(BLEdata["model"].as<const char*>(), "CGD1") == 0)            CLEARGRASSCGD1Discovery(mac);
+    if(strcmp(BLEdata["model"].as<const char*>(), "MiBand") == 0)          MiBandDiscovery(mac);
+    if(strcmp(BLEdata["model"].as<const char*>(), "XMTZC04HM") == 0 || 
+      strcmp(BLEdata["model"].as<const char*>(), "XMTZC05HM") == 0)        MiScaleDiscovery(mac);
+  }
+  else
+  {
+    Log.trace(F("Device already discovered or model not detected" CR));
+  }
+}
+
+void PublishDeviceData(JsonObject &BLEdata)
+{
+  if (abs((int)BLEdata["rssi"] | 0) < abs(Minrssi))
+  { // process only the devices close enough
+    JsonObject &BLEdataOut = process_bledata(BLEdata);
+    String mac_adress = BLEdataOut["id"].as<const char*>();
+    mac_adress.replace(":", "");
+    char mac[mac_adress.length() + 1];
+    mac_adress.toCharArray(mac, mac_adress.length() + 1);
+    #ifdef ZmqttDiscovery
+      launch_discovery(BLEdataOut, mac);
+    #endif
+    #if !pubBLEServiceUUID
+      RemoveJsonPropertyIf(BLEdataOut, "servicedatauuid", BLEdataOut.containsKey("servicedatauuid"));
+    #endif
+    #if !pubKnownBLEServiceData
+      RemoveJsonPropertyIf(BLEdataOut, "servicedata", BLEdataOut.containsKey("model") && BLEdataOut.containsKey("servicedata"));
+    #endif
+    String mactopic(mac);
+    mactopic = subjectBTtoMQTT + String("/") + mactopic;
+    pub((char *)mactopic.c_str(), BLEdataOut);
+  }
+  else
+  {
+    Log.trace(F("Low rssi, device filtered" CR));
+  }
+}
+
+JsonObject& process_bledata(JsonObject &BLEdata){
+  if(BLEdata.containsKey("servicedata"))
+  {
+    Log.trace(F("Checking BLE service data validity" CR));
+    const char * service_data =  (const char *)(BLEdata["servicedata"]|"");
+    if(valid_service_data(service_data))
+    {
+      Log.trace(F("Searching BLE device data %s size %d" CR), service_data, strlen(service_data));
+      Log.trace(F("Is it a mi flora ?" CR));
+      if (strstr(service_data, "209800") != NULL)
+      {
+        Log.trace(F("mi flora data reading" CR));
+        BLEdata.set("model", "HHCCJCY01HHCC");
+
+        return process_sensors(2, BLEdata);
+      }
+      Log.trace(F("Is it a vegtrug ?" CR));
+      if (strstr(service_data, "20bc03") != NULL && strlen(service_data) > ServicedataMinLength)
+      {
+        Log.trace(F("vegtrug data reading" CR));
+        BLEdata.set("model", "VegTrug");
+
+        return process_sensors(2, BLEdata);
+      }
+      Log.trace(F("Is it a LYWSDCGQ?" CR));
+      if (strstr(service_data, "20aa01") != NULL && strlen(service_data) > ServicedataMinLength)
+      {
+        Log.trace(F("LYWSDCGQ data reading" CR));
+        BLEdata.set("model", "LYWSDCGQ");
+
+        return process_sensors(0, BLEdata);
+      }
+      Log.trace(F("Is it a LYWSD02?" CR));
+      if (strstr(service_data, "205b04") != NULL && strlen(service_data)  > ServicedataMinLength)
+      {
+        Log.trace(F("LYWSD02 data reading" CR));
+        BLEdata.set("model", "LYWSD02");
+
+        return process_sensors(2, BLEdata);
+      }
+      Log.trace(F("Is it a CGG1?" CR));
+      if (strstr(service_data, "304703") != NULL && strlen(service_data)  > ServicedataMinLength)
+      {
+        Log.trace(F("CGG1 data reading method 1" CR));
+        BLEdata.set("model", "CGG1");
+
+        return process_sensors(0, BLEdata);
+      }
+      Log.trace(F("Is it a MUE4094RT?" CR));
+      if (strstr(service_data, "4030dd") != NULL)
+      {
+        Log.trace(F("MUE4094RT data reading" CR));
+        BLEdata.set("model", "MUE4094RT");
+
+        return process_milamp(BLEdata);
+      }
+      Log.trace(F("Is it a CGP1W?" CR));
+      if (strstr(service_data, "08094c") != NULL &&  strlen(service_data) > ServicedataMinLength)
+      {
+        Log.trace(F("CGP1W data reading" CR));
+        BLEdata.set("model", "CGP1W");
+
+        return process_cleargrass(BLEdata, true);
+      }
+      Log.trace(F("Is it a CGG1" CR));
+      if (strstr(service_data, "080774") != NULL)
+      {
+        Log.trace(F("CGG1 method 2" CR));
+        BLEdata.set("model", "CGG1");
+        // no discovery as it is already available with method 1
+        return process_cleargrass(BLEdata, false);
+      }
+      Log.trace(F("Is it a CGD1?" CR));
+      if ((strstr(service_data, "080caf") != NULL || strstr(service_data, "080c09") != NULL)
+      && (strlen(service_data) > ServicedataMinLength))
+      {
+        Log.trace(F("CGD1 data reading" CR));
+        BLEdata.set("model", "CGD1");
+
+        return process_cleargrass(BLEdata, false);
+      }
+      if(BLEdata.containsKey("servicedatauuid"))
+      {
+        const char * service_datauuid = (const char *)(BLEdata["servicedatauuid"]|"");
+        Log.trace(F("servicedatauuid %s" CR), service_datauuid);
+        Log.trace(F("Is it a MiBand?" CR));
+        if (strstr(service_datauuid, "fee0") != NULL)
+        {
+          Log.trace(F("Mi Band data reading" CR));
+          BLEdata.set("model", "MiBand");
+
+          return process_miband(BLEdata);
+        }
+        Log.trace(F("Is it a XMTZC04HM?" CR));
+        if (strstr(service_datauuid, "181d") != NULL)
+        {
+          Log.trace(F("XMTZC04HM data reading" CR));
+          BLEdata.set("model", "XMTZC04HM");
+
+          return process_scale_v1(BLEdata);
+        }
+        Log.trace(F("Is it a XMTZC05HM?" CR));
+        if (strstr(service_datauuid, "181b") != NULL)
+        {
+          Log.trace(F("XMTZC05HM data reading" CR));
+          BLEdata.set("model", "XMTZC05HM");;
+
+          return process_scale_v2(BLEdata);
+        }
+      }
+    }
+    else
+    {
+      Log.trace(F("Non valid service data, removing it" CR));
+      BLEdata.remove("servicedata");
+    }
+
+    #if !pubUnknownBLEServiceData
+      Log.trace(F("Unknown service data, removing it" CR));
+      BLEdata.remove("servicedata");
+    #endif
+  }
+
+  return BLEdata;
+}
+
+JsonObject& process_sensors(int offset, JsonObject &BLEdata)
+{
+  const char * servicedata = BLEdata["servicedata"].as<const char *>();
   int data_length = 0;
   
-  switch (rest_data[51 + offset])
+  switch (servicedata[27 + offset])
   {
   case '1':
   case '2':
   case '3':
   case '4':
-    data_length = ((rest_data[51 + offset] - '0') * 2);
+    data_length = ((servicedata[27 + offset] - '0') * 2);
     Log.trace(F("Valid data_length: %d" CR), data_length);
     break;
   default:
-    Log.trace(F("Invalid data_length" CR));
+    Log.trace(F("Invalid data_length, not enriching the device data" CR));
-    return false;
+    return BLEdata;
   }
 
   double value = 9999;
-  value = value_from_service_data(rest_data, 52 + offset, data_length);
+  value = value_from_service_data(servicedata, 28 + offset, data_length);
 
   // Mi flora provides tem(perature), (earth) moi(sture), fer(tility) and lux (illuminance)
   // Mi Jia provides tem(perature), batt(erry) and hum(idity)
-  // following the value of digit 47 we determine the type of data we get from the sensor
+  // following the value of digit 23 + offset we determine the type of data we get from the sensor
-  switch (rest_data[47 + offset])
+  switch (servicedata[23 + offset])
   {
   case '9':
     BLEdata.set("fer", (double)value);
@@ -961,152 +897,94 @@ bool process_sensors(int offset, char *rest_data, char *mac_adress)
   case '8':
     BLEdata.set("moi", (double)value);
     break;
-
   case 'a':
     BLEdata.set("batt", (double)value);
     break;
-
   case 'd':
     // humidity
-    value = value_from_service_data(rest_data, 52 + offset, 4);
+    value = value_from_service_data(servicedata, 28 + offset, 4);
     BLEdata.set("tem", (double)value / 10);
     // temperature
-    value = value_from_service_data(rest_data, 56 + offset, 4);
+    value = value_from_service_data(servicedata, 32 + offset, 4);
     BLEdata.set("hum", (double)value / 10);
     break;
   default:
     Log.trace(F("can't read values" CR));
-    return false;
   }
-  String mactopic(mac_adress);
+
-  mactopic = subjectBTtoMQTT + String("/") + mactopic;
+  return BLEdata;
-  pub((char *)mactopic.c_str(), BLEdata);
-  return true;
 }
 
-bool process_scale_v1(char *rest_data, char *mac_adress)
+JsonObject& process_scale_v1(JsonObject &BLEdata)
 {
-  //example "servicedata":"a2ac2be307060207122b" /"a28039e3070602070e28"
+  const char * servicedata = BLEdata["servicedata"].as<const char *>();
-  Log.trace(F("Creating BLE buffer" CR));
-  StaticJsonBuffer<JSON_MSG_BUFFER> jsonBuffer;
-  JsonObject &BLEdata = jsonBuffer.createObject();
 
-  double weight = value_from_service_data(rest_data, 2, 4) / 200;
+  double weight = value_from_service_data(servicedata, 2, 4) / 200;
 
   //Set Json value
   BLEdata.set("weight", (double)weight);
 
-  // Publish weight
+  return BLEdata;
-  String mactopic(mac_adress);
-  mactopic = subjectBTtoMQTT + String("/") + mactopic;
-  pub((char *)mactopic.c_str(), BLEdata);
-  return true;
 }
 
-bool process_scale_v2(char *rest_data, char *mac_adress)
+JsonObject& process_scale_v2(JsonObject &BLEdata)
 {
-  //example "servicedata":"a2ac2be307060207122b" /"a28039e3070602070e28"
+  const char * servicedata = BLEdata["servicedata"].as<const char *>();
-  Log.trace(F("Creating BLE buffer" CR));
-  StaticJsonBuffer<JSON_MSG_BUFFER> jsonBuffer;
-  JsonObject &BLEdata = jsonBuffer.createObject();
 
-  double weight = value_from_service_data(rest_data, 22, 4) / 200;
+  double weight = value_from_service_data(servicedata, 22, 4) / 200;
-  double impedance = value_from_service_data(rest_data, 18, 4);
+  double impedance = value_from_service_data(servicedata, 18, 4);
 
   //Set Json values
   BLEdata.set("weight", (double)weight);
   BLEdata.set("impedance", (double)impedance);
 
-  // Publish weight
+  return BLEdata;
-  String mactopic(mac_adress);
-  mactopic = subjectBTtoMQTT + String("/") + mactopic;
-  pub((char *)mactopic.c_str(), BLEdata);
-  return true;
 }
 
-bool process_miband(char *rest_data, char *mac_adress)
+JsonObject& process_miband(JsonObject &BLEdata)
 {
-  //example "servicedata":"a21e0000"
+  const char * servicedata = BLEdata["servicedata"].as<const char *>();
-  Log.trace(F("Creating BLE buffer" CR));
-  StaticJsonBuffer<JSON_MSG_BUFFER> jsonBuffer;
-  JsonObject &BLEdata = jsonBuffer.createObject();
 
-  double steps = value_from_service_data(rest_data, 0, 4);
+  double steps = value_from_service_data(servicedata, 0, 4);
 
   //Set Json value
   BLEdata.set("steps", (double)steps);
 
-  // Publish weight
+  return BLEdata;
-  String mactopic(mac_adress);
-  mactopic = subjectBTtoMQTT + String("/") + mactopic;
-  pub((char *)mactopic.c_str(), BLEdata);
-  return true;
 }
 
-bool process_milamp(char *rest_data, char *mac_adress)
+JsonObject& process_milamp(JsonObject &BLEdata)
 {
-  //example "servicedata":"4030dd31d0300010100"
+  const char * servicedata = BLEdata["servicedata"].as<const char *>();
-  Log.trace(F("Creating BLE buffer" CR));
-  StaticJsonBuffer<JSON_MSG_BUFFER> jsonBuffer;
-  JsonObject &BLEdata = jsonBuffer.createObject();
 
-  long darkness = value_from_service_data(rest_data, 8, 2);
+  long darkness = value_from_service_data(servicedata, 8, 2);
 
   //Set Json value
   BLEdata.set("presence", (bool)"true");
   BLEdata.set("darkness", (long)darkness);
 
-  // Publish
+  return BLEdata;
-  String mactopic(mac_adress);
-  mactopic = subjectBTtoMQTT + String("/") + mactopic;
-  pub((char *)mactopic.c_str(), BLEdata);
-  return true;
 }
 
-bool process_cleargrass_air(char *rest_data, char *mac_adress)
+JsonObject& process_cleargrass(JsonObject &BLEdata, boolean air)
 {
-  //example "servicedata":"08094c0140342d580104c400240207025d2702015a"
+  const char * servicedata = BLEdata["servicedata"].as<const char *>();
-  //decoding "08094c0140342d580104 temp:c400 hum:2402 0702 pressure:5d27 02015a"
-  Log.trace(F("Creating BLE buffer" CR));
-  StaticJsonBuffer<JSON_MSG_BUFFER> jsonBuffer;
-  JsonObject &BLEdata = jsonBuffer.createObject();
 
   double value = 9999;
   // humidity
-  value = value_from_service_data(rest_data, 20, 4);
+  value = value_from_service_data(servicedata, 20, 4);
   BLEdata.set("tem", (double)value / 10);
   // temperature
-  value = value_from_service_data(rest_data, 24, 4);
+  value = value_from_service_data(servicedata, 24, 4);
   BLEdata.set("hum", (double)value / 10);
-  // air pressure
+  if (air)
-  value = value_from_service_data(rest_data, 32, 4);
+  {
-  BLEdata.set("pres", (double)value / 100);
+    // air pressure
+    value = value_from_service_data(servicedata, 32, 4);
+    BLEdata.set("pres", (double)value / 100);
+  }
 
-  String mactopic(mac_adress);
+  return BLEdata;
-  mactopic = subjectBTtoMQTT + String("/") + mactopic;
-  pub((char *)mactopic.c_str(), BLEdata);
-  return true;
-}
-
-bool process_cleargrass(char *rest_data, char *mac_adress)
-{
-  //example "servicedata":0807743e10342d580104 c300 2c02 02012a
-  Log.trace(F("Creating BLE buffer" CR));
-  StaticJsonBuffer<JSON_MSG_BUFFER> jsonBuffer;
-  JsonObject &BLEdata = jsonBuffer.createObject();
-
-  double value = 9999;
-  // humidity
-  value = value_from_service_data(rest_data, 20, 4);
-  BLEdata.set("tem", (double)value / 10);
-  // temperature
-  value = value_from_service_data(rest_data, 24, 4);
-  BLEdata.set("hum", (double)value / 10);
-
-  String mactopic(mac_adress);
-  mactopic = subjectBTtoMQTT + String("/") + mactopic;
-  pub((char *)mactopic.c_str(), BLEdata);
-  return true;
 }
 
 #ifdef subjectHomePresence
@@ -1183,5 +1061,4 @@ void MQTTtoBT(char *topicOri, JsonObject &BTdata)
     #endif
   }
 }
-
+#endif
-#endif

main/config_BT.h

@@ -38,20 +38,29 @@ extern void MQTTtoBT(char * topicOri, JsonObject& RFdata);
 #define HMSerialSpeed 9600 // Communication speed with the HM module, softwareserial doesn't support 115200
 //#define HM-11 // uncomment this line if you use HM-11 and comment the line above
 //#define HM_BLUE_LED_STOP true //uncomment to stop the blue led light of HM1X
-#define BLEdelimiter "4f4b2b444953413a"
+#define BLEdelimiter "4f4b2b444953413a" // OK+DISA:
+#define ServicedataMinLength 30
 
 #ifndef TimeBtw_Read
   #define TimeBtw_Read 55555 //define default time between 2 scans
 #endif
 
-#ifndef pubBLEServiceData
+#ifndef pubKnownBLEServiceData
-  #define pubBLEServiceData true // comment if you don't want to publish service data (in case you are having too heavy service data) https://github.com/1technophile/OpenMQTTGateway/issues/318#issuecomment-446064707
+  #define pubKnownBLEServiceData false // define true if you want to publish service data belonging to recognised sensors
+#endif
+
+#ifndef pubUnknownBLEServiceData
+  #define pubUnknownBLEServiceData true // define false if you don't want to publish service data to unrecognised sensors (in case you are having too heavy service data) https://github.com/1technophile/OpenMQTTGateway/issues/318#issuecomment-446064707
 #endif
 
 #ifndef pubBLEManufacturerData
   #define pubBLEManufacturerData false // define true if you want to publish the manufacturer's data (sometimes contains characters that aren't valid with receiving client)
 #endif
 
+#ifndef pubBLEServiceUUID
+  #define pubBLEServiceUUID false // define true if you want to publish the service UUID data
+#endif
+
 /*-------------------HOME ASSISTANT ROOM PRESENCE ----------------------*/
 // if not commented Home presence integration with HOME ASSISTANT is activated
 #define subjectHomePresence "home_presence/" // will send Home Assistant room presence message to this topic (first part is same for all rooms, second is room name)

main/main.ino

@@ -202,14 +202,6 @@ void extract_char(char *token_char, char *subset, int start, int l, bool reverse
   subset[l] = '\0';
 }
 
-int strpos(char *haystack, char *needle) //from @miere https://stackoverflow.com/users/548685/miere
-{
-  char *p = strstr(haystack, needle);
-  if (p)
-    return p - haystack;
-  return -1;
-}
-
 char *ip2CharArray(IPAddress ip)
 { //from Nick Lee https://stackoverflow.com/questions/28119653/arduino-display-ethernet-localip
   static char a[16];