From d6d42bfce52d5f8275cc445b7377e662ba596c06 Mon Sep 17 00:00:00 2001 From: Florian <1technophile@users.noreply.github.com> Date: Tue, 18 Aug 2020 12:28:45 -0500 Subject: [PATCH] BLE sensors data through a connect, LYWSD03MMC and MHO-C401 add (#736) * Add LYWSD03MMC and MHO-C401 device and an associated BLE connect mechanism as these devices advertise only encrypted data -free manufacturer data -update library * Implement scans number before connect -Add the possibility to do a connect every X scan -Rename variables for consistency -Some refactoring for improved code reading --- docs/prerequisites/devices.md | 2 + docs/use/ble.md | 21 +++- main/ZgatewayBT.ino | 212 ++++++++++++++++++++++++++++++---- main/config_BT.h | 38 +++--- main/main.ino | 1 + platformio.ini | 2 +- 6 files changed, 237 insertions(+), 39 deletions(-) diff --git a/docs/prerequisites/devices.md b/docs/prerequisites/devices.md index cba6120..5986491 100644 --- a/docs/prerequisites/devices.md +++ b/docs/prerequisites/devices.md @@ -20,6 +20,8 @@ OpenMQTTGateway is able to scan all the BLE devices that advertise their data so | Vegtrug ||temperature/moisture/luminance/fertility| | XIAOMI Mi Flora |HHCCJCY01HHCC|temperature/moisture/luminance/fertility| | XIAOMI Mi Jia |LYWSDCGO|temperature/humidity/battery| +| XIAOMI Mi Jia 2 *|LYWSD03MMC|temperature/humidity/battery/volt| +| XIAOMI MHO-C401 *|MHO-C401|temperature/humidity/battery/volt| | XIAOMI Mi Lamp |MUE4094RT|presence| | HONEYWELL |JQJCY01YM|formaldehyde/temperature/humidity/battery| | INKBIRD *|IBS-TH1|temperature/humidity/battery| diff --git a/docs/use/ble.md b/docs/use/ble.md index 6eb5a2b..cdf1ebe 100644 --- a/docs/use/ble.md +++ b/docs/use/ble.md @@ -73,9 +73,10 @@ So as to keep your white/black list persistent you can publish it with the retai `mosquitto_pub -r -t home/OpenMQTTGateway/commands/MQTTtoBT/config -m '{"white-list":["01:23:14:55:16:15","4C:65:77:88:9C:79","4C:65:A6:66:3C:79"]}'` ::: -## Setting the time between scans and force a scan +## Setting the time between BLE scans and force a scan -If you want to change the time between readings you can change the interval by MQTT, if you want the BLE scan every 66seconds: +If you want to change the time between readings you can change the interval by MQTT. +For example, if you want the BLE to scan every 66 seconds: `mosquitto_pub -t home/OpenMQTTGateway/commands/MQTTtoBT/config -m '{"interval":66000}'` @@ -83,9 +84,21 @@ you can also force a scan to be done by the following command: `mosquitto_pub -t home/OpenMQTTGateway/commands/MQTTtoBT/config -m '{"interval":0}'` -Once done the previous value of interval will be recovered. +Once the forced scan has completed, the previous scan interval value will be restored. Forcing a scan command trigger also a BLE connect process after the scan (see below). -The default value is set into config_BT.h +The default value `TimeBtwRead` is set into config_BT.h or into your .ini file for platformio users. + +::: info +For certain devices like LYWSD03MMC OpenMQTTGateway use a connection (due to the fact that the advertized data are encrypted), this connection mechanism is launched after every `ScanBeforeConnect` per default, you can modify it by following the procedure below. +::: + +## Setting the number of scans between before connect attempt + +If you want to change the number of BLE scans that are done before a BLE connect you can change it by MQTT, if you want the BLE connect to be every 30 scans: + +`mosquitto_pub -t home/OpenMQTTGateway/commands/MQTTtoBT/config -m '{"scanbcnct":30}'` + +The BLE connect will be done every 30 * (`TimeBtwRead` + `Scan_duration`), 30 * (55000 + 10000) = 1950000ms : ## Setting the minimum RSSI accepted to publish device data diff --git a/main/ZgatewayBT.ino b/main/ZgatewayBT.ino index 8f1b6e5..e18041d 100644 --- a/main/ZgatewayBT.ino +++ b/main/ZgatewayBT.ino @@ -36,10 +36,23 @@ Thanks to wolass https://github.com/wolass for suggesting me HM 10 and dinosd ht # include "FreeRTOS.h" FreeRTOS::Semaphore semaphoreCreateOrUpdateDevice = FreeRTOS::Semaphore("createOrUpdateDevice"); // Headers used for deep sleep functions +# include +# include +# include +# include # include # include # include # include + +# include "soc/timer_group_reg.h" +# include "soc/timer_group_struct.h" + +void notifyCB( + BLERemoteCharacteristic* pBLERemoteCharacteristic, + uint8_t* pData, + size_t length, + bool isNotify); # endif # if !defined(ESP32) && !defined(ESP8266) @@ -75,6 +88,10 @@ static bool oneWhite = false; int minRssi = abs(MinimumRSSI); //minimum rssi value +unsigned int scanCount = 0; + +bool ProcessLock = false; // Process lock when we want to use a critical function like OTA for example + BLEdevice* getDeviceByMac(const char* mac); void createOrUpdateDevice(const char* mac, uint8_t flags, ble_sensor_model model); @@ -335,6 +352,36 @@ void InkBirdDiscovery(char* mac) { createDiscoveryFromList(mac, InkBirdsensor, InkBirdparametersCount); } +void LYWSD03MMCDiscovery(char* mac) { +# define LYWSD03MMCparametersCount 5 + Log.trace(F("LYWSD03MMCDiscovery" CR)); + char* LYWSD03MMCsensor[LYWSD03MMCparametersCount][8] = { + {"sensor", "LYWSD03MMC-batt", mac, "battery", jsonBatt, "", "", "%"}, + {"sensor", "LYWSD03MMC-volt", mac, "voltage", jsonVolt, "", "", "V"}, + {"sensor", "LYWSD03MMC-tempc", mac, "temperature", jsonTempc, "", "", "C"}, + {"sensor", "LYWSD03MMC-tempf", mac, "temperature", jsonTempf, "", "", "F"}, + {"sensor", "LYWSD03MMC-hum", mac, "humidity", jsonHum, "", "", "%"} + //component type,name,availability topic,device class,value template,payload on, payload off, unit of measurement + }; + + createDiscoveryFromList(mac, LYWSD03MMCsensor, LYWSD03MMCparametersCount); +} + +void MHO_C401Discovery(char* mac) { +# define MHO_C401parametersCount 5 + Log.trace(F("MHO_C401Discovery" CR)); + char* MHO_C401sensor[MHO_C401parametersCount][8] = { + {"sensor", "MHO_C401-batt", mac, "battery", jsonBatt, "", "", "%"}, + {"sensor", "MHO_C401-volt", mac, "voltage", jsonVolt, "", "", "V"}, + {"sensor", "MHO_C401-tempc", mac, "temperature", jsonTempc, "", "", "C"}, + {"sensor", "MHO_C401-tempf", mac, "temperature", jsonTempf, "", "", "F"}, + {"sensor", "MHO_C401-hum", mac, "humidity", jsonHum, "", "", "%"} + //component type,name,availability topic,device class,value template,payload on, payload off, unit of measurement + }; + + createDiscoveryFromList(mac, MHO_C401sensor, MHO_C401parametersCount); +} + # else void MiFloraDiscovery(char* mac) {} void VegTrugDiscovery(char* mac) {} @@ -348,6 +395,8 @@ void MiScaleDiscovery(char* mac) {} void MiLampDiscovery(char* mac) {} void MiBandDiscovery(char* mac) {} void InkBirdDiscovery(char* mac) {} +void LYWSD03MMCDiscovery(char* mac) {} +void MHO_C401Discovery(char* mac) {} # endif # ifdef ESP32 @@ -357,18 +406,8 @@ void InkBirdDiscovery(char* mac) {} */ // core task implementation thanks to https://techtutorialsx.com/2017/05/09/esp32-running-code-on-a-specific-core/ -# include -# include -# include -# include - -# include "soc/timer_group_reg.h" -# include "soc/timer_group_struct.h" - //core on which the BLE detection task will run static int taskCore = 0; -// Process lock when we want to use a critical function like OTA for example -bool ProcessLock = false; class MyAdvertisedDeviceCallbacks : public BLEAdvertisedDeviceCallbacks { void onResult(BLEAdvertisedDevice* advertisedDevice) { @@ -388,6 +427,7 @@ class MyAdvertisedDeviceCallbacks : public BLEAdvertisedDeviceCallbacks { char* manufacturerdata = BLEUtils::buildHexData(NULL, (uint8_t*)advertisedDevice->getManufacturerData().data(), advertisedDevice->getManufacturerData().length()); Log.trace(F("Manufacturer Data: %s" CR), manufacturerdata); BLEdata.set("manufacturerdata", manufacturerdata); + free(manufacturerdata); } if (advertisedDevice->haveRSSI()) BLEdata.set("rssi", (int)advertisedDevice->getRSSI()); @@ -430,6 +470,9 @@ class MyAdvertisedDeviceCallbacks : public BLEAdvertisedDeviceCallbacks { } }; +/** + * BLEscan used to retrieve BLE advertized data from devices without connection + */ void BLEscan() { TIMERG0.wdt_wprotect = TIMG_WDT_WKEY_VALUE; TIMERG0.wdt_feed = 1; @@ -439,11 +482,105 @@ void BLEscan() { BLEScan* pBLEScan = BLEDevice::getScan(); //create new scan pBLEScan->setAdvertisedDeviceCallbacks(new MyAdvertisedDeviceCallbacks()); pBLEScan->setActiveScan(true); //active scan uses more power, but get results faster - BLEScanResults foundDevices = pBLEScan->start(Scan_duration, false); - Log.notice(F("Found %d devices, scan end deinit controller" CR), foundDevices.getCount()); + BLEScanResults foundDevices = pBLEScan->start(Scan_duration / 1000, false); + scanCount++; + Log.notice(F("Found %d devices, scan number %d end deinit controller" CR), foundDevices.getCount(), scanCount); BLEDevice::deinit(true); } +/** + * Callback method to retrieve data from devices characteristics + */ +void notifyCB( + BLERemoteCharacteristic* pBLERemoteCharacteristic, + uint8_t* pData, + size_t length, + bool isNotify) { + if (!ProcessLock) { + Log.trace(F("Callback from %s characteristic" CR), pBLERemoteCharacteristic->getUUID().toString().c_str()); + + if (length == 5) { + Log.trace(F("Device identified creating BLE buffer" CR)); + StaticJsonBuffer jsonBuffer; + JsonObject& BLEdata = jsonBuffer.createObject(); + String mac_adress = pBLERemoteCharacteristic->getRemoteService()->getClient()->getPeerAddress().toString().c_str(); + mac_adress.toUpperCase(); + for (vector::iterator p = devices.begin(); p != devices.end(); ++p) { + if ((strcmp(p->macAdr, (char*)mac_adress.c_str()) == 0)) { + if (p->sensorModel == LYWSD03MMC) + BLEdata.set("model", "LYWSD03MMC"); + else if (p->sensorModel == MHO_C401) + BLEdata.set("model", "MHO_C401"); + } + } + BLEdata.set("id", (char*)mac_adress.c_str()); + Log.trace(F("Device identified in CB: %s" CR), (char*)mac_adress.c_str()); + BLEdata.set("tempc", (float)((pData[0] | (pData[1] << 8)) * 0.01)); + BLEdata.set("tempf", (float)(convertTemp_CtoF((pData[0] | (pData[1] << 8)) * 0.01))); + BLEdata.set("hum", (float)(pData[2])); + BLEdata.set("volt", (float)(((pData[4] * 256) + pData[3]) / 1000.0)); + BLEdata.set("batt", (float)(((((pData[4] * 256) + pData[3]) / 1000.0) - 2.1) * 100)); + + mac_adress.replace(":", ""); + String mactopic = subjectBTtoMQTT + String("/") + mac_adress; + pub((char*)mactopic.c_str(), BLEdata); + } else { + Log.warning(F("Device not identified" CR)); + } + } else { + Log.trace(F("Callback process canceled by processLock" CR)); + } + pBLERemoteCharacteristic->unsubscribe(); +} + +/** + * Connect to BLE devices and initiate the callbacks with a service/characteristic request + */ +void BLEconnect() { + Log.notice(F("BLE Connect begin" CR)); + BLEDevice::init(""); + for (vector::iterator p = devices.begin(); p != devices.end(); ++p) { + if (p->sensorModel == LYWSD03MMC || p->sensorModel == MHO_C401) { + Log.trace(F("Model to connect found" CR)); + NimBLEClient* pClient; + pClient = BLEDevice::createClient(); + BLEUUID serviceUUID("ebe0ccb0-7a0a-4b0c-8a1a-6ff2997da3a6"); + BLEUUID charUUID("ebe0ccc1-7a0a-4b0c-8a1a-6ff2997da3a6"); + BLEAddress sensorAddress(p->macAdr); + if (!pClient->connect(sensorAddress)) { + Log.warning(F("Failed to find client: %s" CR), p->macAdr); + NimBLEDevice::deleteClient(pClient); + } else { + BLERemoteService* pRemoteService = pClient->getService(serviceUUID); + if (!pRemoteService) { + Log.warning(F("Failed to find service UUID: %s" CR), serviceUUID.toString().c_str()); + pClient->disconnect(); + } else { + Log.trace(F("Found service: %s" CR), serviceUUID.toString().c_str()); + // Obtain a reference to the characteristic in the service of the remote BLE server. + if (pClient->isConnected()) { + Log.trace(F("Client isConnected, freeHeap: %d" CR), ESP.getFreeHeap()); + BLERemoteCharacteristic* pRemoteCharacteristic = pRemoteService->getCharacteristic(charUUID); + if (!pRemoteCharacteristic) { + Log.warning(F("Failed to find characteristic UUID: %s" CR), charUUID.toString().c_str()); + pClient->disconnect(); + } else { + if (pRemoteCharacteristic->canNotify()) { + Log.trace(F("Registering notification" CR)); + pRemoteCharacteristic->subscribe(true, notifyCB); + } else { + Log.warning(F("Failed registering notification" CR)); + pClient->disconnect(); + } + } + } + } + } + } + } + Log.notice(F("BLE Connect end" CR)); +} + void stopProcessing() { Log.notice(F("Stop BLE processing" CR)); ProcessLock = true; @@ -471,9 +608,12 @@ void coreTask(void* pvParameters) { if (low_power_mode == 2) digitalWrite(LOW_POWER_LED, 1 - LOW_POWER_LED_OFF); BLEscan(); - launch_discovery(); + // Launching a connect every BLEscanBeforeConnect + if (!(scanCount % BLEscanBeforeConnect) || scanCount == 1) + BLEconnect(); + launchDiscovery(); dumpDevices(); - //only change LOW_POWER_LED if low power mode is enabled + // Only change LOW_POWER_LED if low power mode is enabled if (low_power_mode) digitalWrite(LOW_POWER_LED, LOW_POWER_LED_OFF); } @@ -482,6 +622,8 @@ void coreTask(void* pvParameters) { } else { delay(BLEinterval); } + } else { + Log.trace(F("BLE core task canceled by processLock" CR)); } } } @@ -532,7 +674,8 @@ void changelow_power_mode(int newLowPowerMode) { } void setupBT() { - Log.notice(F("BLE interval: %d" CR), BLEinterval); + Log.notice(F("BLE scans interval: %d" CR), BLEinterval); + Log.notice(F("BLE scans number before connect: %d" CR), BLEscanBeforeConnect); Log.notice(F("minrssi: %d" CR), minRssi); Log.notice(F("Low Power Mode: %d" CR), low_power_mode); @@ -566,8 +709,8 @@ unsigned long timebt = 0; struct decompose d[6] = {{0, 12, true}, {12, 2, false}, {14, 2, false}, {16, 2, false}, {28, 4, true}, {32, 60, false}}; void setupBT() { - BLEinterval = TimeBtw_Read; Log.notice(F("BLE interval: %d" CR), BLEinterval); + Log.notice(F("BLE scans number before connect: %d" CR), BLEscanBeforeConnect); Log.notice(F("minrssi: %d" CR), minRssi); softserial.begin(HMSerialSpeed); softserial.print(F("AT+ROLE1" CR)); @@ -611,6 +754,8 @@ bool BTtoMQTT() { if (returnedString.equals(F(BLEEndOfDiscovery))) //OK+DISCE { returnedString.remove(0); //clear data string + scanCount++; + Log.notice(F("Scan number %d end " CR), scanCount); return false; } size_t pos = 0, eolPos = 0; @@ -699,7 +844,7 @@ boolean valid_service_data(const char* data) { return false; } -void launch_discovery() { +void launchDiscovery() { for (vector::iterator p = devices.begin(); p != devices.end(); ++p) { if (p->sensorModel != UNKNOWN_MODEL && !isDiscovered(p)) { String macWOdots = String(p->macAdr); @@ -718,6 +863,8 @@ void launch_discovery() { if ((p->sensorModel == XMTZC04HM) || (p->sensorModel == XMTZC05HM)) MiScaleDiscovery((char*)macWOdots.c_str()); if (p->sensorModel == INKBIRD) InkBirdDiscovery((char*)macWOdots.c_str()); + if (p->sensorModel == LYWSD03MMC) LYWSD03MMCDiscovery((char*)macWOdots.c_str()); + if (p->sensorModel == MHO_C401) MHO_C401Discovery((char*)macWOdots.c_str()); createOrUpdateDevice(p->macAdr, device_flags_isDisc, p->sensorModel); } else { Log.trace(F("Device already discovered or UNKNOWN_MODEL" CR)); @@ -831,6 +978,18 @@ JsonObject& process_bledata(JsonObject& BLEdata) { createOrUpdateDevice(mac, device_flags_init, CGD1); return process_cleargrass(BLEdata, false); } + Log.trace(F("Is it a MHO_C401?" CR)); + if (strstr(service_data, "588703") != NULL) { + Log.trace(F("MHO_C401 add to list for future connect" CR)); + if (device->sensorModel == -1) + createOrUpdateDevice(mac, device_flags_init, MHO_C401); + } + Log.trace(F("Is it a LYWSD03MMC?" CR)); + if (strstr(service_data, "585b05") != NULL) { + Log.trace(F("LYWSD03MMC add to list for future connect" CR)); + if (device->sensorModel == -1) + createOrUpdateDevice(mac, device_flags_init, LYWSD03MMC); + } if (BLEdata.containsKey("servicedatauuid")) { const char* service_datauuid = (const char*)(BLEdata["servicedatauuid"] | ""); Log.trace(F("servicedatauuid %s" CR), service_datauuid); @@ -1083,15 +1242,28 @@ void MQTTtoBT(char* topicOri, JsonObject& BTdata) { // json object decoding // storing BLE interval for further use if needed unsigned int prevBLEinterval = BLEinterval; Log.trace(F("Previous interval: %d ms" CR), BLEinterval); - // set BLE interval if present if not setting default value BLEinterval = (unsigned int)BTdata["interval"]; Log.notice(F("New interval: %d ms" CR), BLEinterval); if (BLEinterval == 0) { - if (BTtoMQTT()) // as BLEinterval is = to 0 we can launch the loop and the scan will execute immediately + if (!ProcessLock) { + BTtoMQTT(); Log.trace(F("Scan done" CR)); - BLEinterval = prevBLEinterval; // as 0 was just used as a command we recover previous scan duration +# ifdef ESP32 + BLEconnect(); +# endif + BLEinterval = prevBLEinterval; // as 0 was just used as a command we recover previous scan duration + } else { + Log.trace(F("Cannot launch scan due to other process running" CR)); + } } } + // Number of scan before a connect set + if (BTdata.containsKey("scanbcnct")) { + Log.trace(F("BLE scans number before a connect setup" CR)); + Log.trace(F("Previous number: %d" CR), BLEscanBeforeConnect); + BLEscanBeforeConnect = (unsigned int)BTdata["scanbcnct"]; + Log.notice(F("New scan number before connect: %d" CR), BLEscanBeforeConnect); + } // MinRSSI set if (BTdata.containsKey("minrssi")) { // storing Min RSSI for further use if needed diff --git a/main/config_BT.h b/main/config_BT.h index 355bd55..114bddb 100644 --- a/main/config_BT.h +++ b/main/config_BT.h @@ -29,27 +29,35 @@ extern void setupBT(); extern bool BTtoMQTT(); extern void MQTTtoBT(char* topicOri, JsonObject& RFdata); + /*----------------------BT topics & parameters-------------------------*/ #define subjectBTtoMQTT "/BTtoMQTT" #define subjectMQTTtoBTset "/commands/MQTTtoBT/config" #define MinimumRSSI -100 //default minimum rssi value, all the devices below -90 will not be reported -#define Scan_duration 10 //define the time for a scan --WARNING-- changing this value can lead to instability on ESP32 -#define HM -10 -#define HMSerialSpeed 9600 // Communication speed with the HM module, softwareserial doesn't support 115200 -//#define HM-11 // uncomment this line if you use HM-11 and comment the line above -//#define HM_BLUE_LED_STOP true //uncomment to stop the blue led light of HM1X -#define BLEdelimiter "4f4b2b444953413a" // OK+DISA: -#define BLEEndOfDiscovery "4f4b2b4449534345" // OK+DISCE -#define BLEdelimiterLength 16 -#define CRLR "0d0a" -#define CRLR_Length 4 -#define ServicedataMinLength 29 -#ifndef TimeBtw_Read -# define TimeBtw_Read 55555 //define default time between 2 scans +#ifndef Scan_duration +# define Scan_duration 10000 //define the time for a scan --WARNING-- changing this value can lead to instability on ESP32 +#endif +#ifndef ScanBeforeConnect +# define ScanBeforeConnect 10 //define number of scans before connecting to BLE devices (ESP32 only, minimum 1) +#endif +#ifndef TimeBtwRead +# define TimeBtwRead 55555 //define default time between 2 scans #endif -unsigned int BLEinterval = TimeBtw_Read; //time between 2 scans +#define HMSerialSpeed 9600 // Communication speed with the HM module, softwareserial doesn't support 115200 +//#define HM_BLUE_LED_STOP true //uncomment to stop the blue led light of HM1X + +#define BLEdelimiter "4f4b2b444953413a" // OK+DISA: +#define BLEEndOfDiscovery "4f4b2b4449534345" // OK+DISCE +#define BLEdelimiterLength 16 +#define CRLR "0d0a" +#define CRLR_Length 4 + +#define ServicedataMinLength 29 + +unsigned int BLEinterval = TimeBtwRead; //time between 2 scans +unsigned int BLEscanBeforeConnect = ScanBeforeConnect; //Number of BLE scans between connection cycles #ifndef pubKnownBLEServiceData # define pubKnownBLEServiceData false // define true if you want to publish service data belonging to recognised sensors @@ -87,6 +95,8 @@ enum ble_sensor_model { XMTZC04HM, XMTZC05HM, INKBIRD, + LYWSD03MMC, + MHO_C401, }; /*-------------------PIN DEFINITIONS----------------------*/ diff --git a/main/main.ino b/main/main.ino index 740a1fb..b4ee782 100644 --- a/main/main.ino +++ b/main/main.ino @@ -1293,6 +1293,7 @@ void stateMeasures() { SYSdata["lowpowermode"] = (int)low_power_mode; # endif SYSdata["interval"] = BLEinterval; + SYSdata["scanbcnct"] = BLEscanBeforeConnect; # endif # ifdef ZgatewayRFM69 modules = modules + ZgatewayRFM69; diff --git a/platformio.ini b/platformio.ini index 20bf895..69524d9 100644 --- a/platformio.ini +++ b/platformio.ini @@ -73,7 +73,7 @@ arduinolog = https://github.com/1technophile/Arduino-Log.git#d13cd80 pubsubclient = PubSubClient@2.8 rc-switch = https://github.com/1technophile/rc-switch.git#385a7e0 newremoteswitch = https://github.com/1technophile/NewRemoteSwitch.git#8eb980e -ble = https://github.com/h2zero/NimBLE-Arduino.git#111ff3a +ble = https://github.com/h2zero/NimBLE-Arduino.git#e53bbf9 irremoteesp = IRremoteESP8266@2.7.7 irremote = https://github.com/z3t0/Arduino-IRremote.git#1c08a37 lora = https://github.com/sandeepmistry/arduino-LoRa.git#f4a1d27