[BT] - Adaptive scan (#1440)

* [BT] Adaptive scan

Add adaptive scan to avoid users setting parameters and deduct automatically the scan requirements from the list of devices

---------
Co-authored-by: DigiH <github@digih.net>

.github/workflows/build.yml

@@ -22,7 +22,6 @@ jobs:
           - "esp32dev-ble-openhab"
           - "esp32dev-ble-mqtt-undecoded"
           - "esp32dev-ble-aws"
-          - "esp32dev-ble-cont"
           - "esp32feather-ble"
           - "esp32-lolin32lite-ble"
           - "esp32-olimex-gtw-ble-eth"

docs/use/ble.md

@@ -134,11 +134,15 @@ With Home Assistant, this command is directly available through MQTT auto discov
 
 The gateway will publish only the detected sensors like Mi Flora, Mi jia, LYWSD03MMC... and not the other BLE devices. This is useful if you don't use the gateway for presence detection but only to retrieve sensors data.
 
-## Setting if the gateway use active or passive scanning
+## Setting if the gateway use adaptive scanning
+
+Adaptive scanning lets the gateway decide for you the best passive `interval` and active `intervalacts` scan interval, depending on the characteristics of your devices.
+The gateway retrieves your devices' information from [Theengs Decoder](https://decoder.theengs.io) and adapts its parameters accordingly.
+For example a door or a PIR sensor will require continuous scanning, so if detected the gateway is going to reduce its time between scans to the minimum. Or your devices may also require active scanning to retrieve data, in this case the gateway will also trigger active scans at regular intervals.
 
 If you want to change this characteristic (default:true):
 
-`mosquitto_pub -t home/OpenMQTTGateway/commands/MQTTtoBT/config -m '{"activescan":false}'`
+`mosquitto_pub -t home/OpenMQTTGateway/commands/MQTTtoBT/config -m '{"adaptivescan":false}'`
 
 ::: tip
 With Home Assistant, this command is directly available through MQTT auto discovery as a switch into the HASS OpenMQTTGateway device entities list.
@@ -154,10 +158,6 @@ If you want to change the time between active scans you can change it by MQTT. F
 
 `mosquitto_pub -t home/OpenMQTTGateway/commands/MQTTtoBT/config -m '{"intervalacts":300000}'`
 
-::: tip
-Setting `{"intervalacts":0}` deactivates intermittent active scanning to only have consistent passive scanning if `'{"activescan":false}'`.
-:::
-
 ## Setting if the gateway connects to BLE devices eligibles on ESP32
 
 If you want to change this characteristic:

environments.ini

@@ -239,7 +239,7 @@ build_flags =
   '-DLED_SEND_RECEIVE=2'
   '-DLED_SEND_RECEIVE_ON=0'
   '-DGateway_Name="OpenMQTTGateway_ESP32_BLE"'
-custom_description = Regular BLE gateway with one scan every 55s per default
+custom_description = Regular BLE gateway with adaptive scanning activated, automatically adapts the scan parameters depending on your devices
 
 [env:esp32dev-ble-openhab]
 platform = ${com.esp32_platform}
@@ -257,7 +257,7 @@ build_flags =
   '-DLED_SEND_RECEIVE_ON=0'
   '-DOpenHABDiscovery=true'
   '-DGateway_Name="OpenMQTTGateway_ESP32_BLE_OH"'
-custom_description = BLE gateway with one scan every 55s per default and OpenHAB discovery
+custom_description = BLE gateway with OpenHAB discovery and adaptive scanning activated, automatically adapts the scan parameters depending on your devices
 
 [env:esp32dev-ble-mqtt-undecoded]
 platform = ${com.esp32_platform}
@@ -302,26 +302,6 @@ build_flags =
   '-DAWS_IOT=true'
 custom_description = BLE gateway with AWS integration capability
 
-[env:esp32dev-ble-cont]
-platform = ${com.esp32_platform}
-board = esp32dev
-board_build.partitions = min_spiffs.csv
-lib_deps =
-  ${com-esp.lib_deps}
-  ${libraries.wifimanager32}
-  ${libraries.ble}
-  ${libraries.decoder}
-build_flags =
-  ${com-esp.build_flags}
-  '-DZgatewayBT="BT"'
-  '-DLED_SEND_RECEIVE=2'
-  '-DLED_SEND_RECEIVE_ON=0'
-  '-DGateway_Name="OpenMQTTGateway_ESP32_BLE_C"'
-  '-DTimeBtwRead=0'
-  '-DScan_duration=3000'
-  '-DAttemptBLEConnect=false'
-custom_description = Regular BLE gateway with continuous scan and no BLE connection attempt per default
-
 [env:esp32feather-ble]
 platform = ${com.esp32_platform}
 board = featheresp32
@@ -360,7 +340,7 @@ build_flags =
 ;  '-DTimeBtwRead=155000'
 ;  '-DScan_duration=2000'
 ;  '-DAttemptBLEConnect=false'
-;  '-DActiveBLEScan=false'
+;  '-DAdaptiveBLEScan=false'
 ;  '-DESPWifiManualSetup=true'
 ;  '-DMQTT_USER="lolin-esp32"'
 ;  '-DMQTT_PASS="your_password"'

main/ZgatewayBT.ino

@@ -82,7 +82,7 @@ static bool oneWhite = false;
 void BTConfig_init() {
   BTConfig.bleConnect = AttemptBLEConnect;
   BTConfig.BLEinterval = TimeBtwRead;
-  BTConfig.activeScan = ActiveBLEScan;
+  BTConfig.adaptiveScan = AdaptiveBLEScan;
   BTConfig.intervalActiveScan = TimeBtwActive;
   BTConfig.intervalConnect = TimeBtwConnect;
   BTConfig.pubOnlySensors = PublishOnlySensors;
@@ -120,7 +120,7 @@ void stateBTMeasures(bool start) {
   JsonObject jo = jsonBuffer.to<JsonObject>();
   jo["bleconnect"] = BTConfig.bleConnect;
   jo["interval"] = BTConfig.BLEinterval;
-  jo["activescan"] = BTConfig.activeScan;
+  jo["adaptivescan"] = BTConfig.adaptiveScan;
   jo["intervalacts"] = BTConfig.intervalActiveScan;
   jo["intervalcnct"] = BTConfig.intervalConnect;
   jo["onlysensors"] = BTConfig.pubOnlySensors;
@@ -155,7 +155,7 @@ void BTConfig_fromJson(JsonObject& BTdata, bool startup = false) {
   if (BTdata.containsKey("interval") && BTdata["interval"] != 0)
     BTConfig_update(BTdata, "interval", BTConfig.BLEinterval);
   // Define if the scan is active or passive
-  BTConfig_update(BTdata, "activescan", BTConfig.activeScan);
+  BTConfig_update(BTdata, "adaptivescan", BTConfig.adaptiveScan);
   // Time before before active scan
   BTConfig_update(BTdata, "intervalacts", BTConfig.intervalActiveScan);
   // Time before a connect set
@@ -199,7 +199,7 @@ void BTConfig_fromJson(JsonObject& BTdata, bool startup = false) {
     JsonObject jo = jsonBuffer.to<JsonObject>();
     jo["bleconnect"] = BTConfig.bleConnect;
     jo["interval"] = BTConfig.BLEinterval;
-    jo["activescan"] = BTConfig.activeScan;
+    jo["adaptivescan"] = BTConfig.adaptiveScan;
     jo["intervalacts"] = BTConfig.intervalActiveScan;
     jo["intervalcnct"] = BTConfig.intervalConnect;
     jo["onlysensors"] = BTConfig.pubOnlySensors;
@@ -625,11 +625,11 @@ void BLEscan() {
   BLEScan* pBLEScan = BLEDevice::getScan();
   MyAdvertisedDeviceCallbacks myCallbacks;
   pBLEScan->setAdvertisedDeviceCallbacks(&myCallbacks);
-  if (millis() > (timeBetweenActive + BTConfig.intervalActiveScan) && !BTConfig.activeScan && BTConfig.intervalActiveScan != 0) {
+  if (millis() > (timeBetweenActive + BTConfig.intervalActiveScan)) {
     pBLEScan->setActiveScan(true);
     timeBetweenActive = millis();
   } else {
-    pBLEScan->setActiveScan(BTConfig.activeScan);
+    pBLEScan->setActiveScan(false);
   }
   pBLEScan->setInterval(BLEScanInterval);
   pBLEScan->setWindow(BLEScanWindow);
@@ -810,7 +810,7 @@ void setupBT() {
   Log.notice(F("BLE scans interval: %d" CR), BTConfig.BLEinterval);
   Log.notice(F("BLE connects interval: %d" CR), BTConfig.intervalConnect);
   Log.notice(F("Publishing only BLE sensors: %T" CR), BTConfig.pubOnlySensors);
-  Log.notice(F("Active BLE scan: %T" CR), BTConfig.activeScan);
+  Log.notice(F("Adaptive BLE scan: %T" CR), BTConfig.adaptiveScan);
   Log.notice(F("Active BLE scan interval: %d" CR), BTConfig.intervalActiveScan);
   Log.notice(F("minrssi: %d" CR), -abs(BTConfig.minRssi));
   Log.notice(F("Low Power Mode: %d" CR), lowpowermode);
@@ -1002,6 +1002,22 @@ void process_bledata(JsonObject& BLEdata) {
       createOrUpdateDevice(mac, device_flags_connect, model_id, mac_type); // Device that broadcast and can be connected
     } else {
       createOrUpdateDevice(mac, device_flags_init, model_id, mac_type);
+      if (BTConfig.adaptiveScan == true && (BTConfig.BLEinterval != MinTimeBtwScan || BTConfig.intervalActiveScan != MinTimeBtwScan)) {
+        if (BLEdata.containsKey("acts") && BLEdata.containsKey("cont")) {
+          if (BLEdata["acts"] && BLEdata["cont"]) {
+            BTConfig.BLEinterval = MinTimeBtwScan;
+            BTConfig.intervalActiveScan = MinTimeBtwScan;
+            Log.notice(F("Active and continuous scanning required, paramaters adapted" CR));
+            stateBTMeasures(false);
+          }
+        } else if (BLEdata.containsKey("cont")) {
+          if (BLEdata["cont"]) {
+            BTConfig.BLEinterval = MinTimeBtwScan;
+            Log.notice(F("Passive continuous scanning required, paramaters adapted" CR));
+            stateBTMeasures(false);
+          }
+        }
+      }
     }
   } else {
     if (BLEdata.containsKey("name")) { // Connectable devices

main/ZmqttDiscovery.ino

@@ -940,6 +940,15 @@ void pubMqttDiscovery() {
                   "", "", "", "", false, // device name, device manufacturer, device model, device ID, retain,
                   stateClassNone //State Class
   );
+  createDiscovery("number", //set Type
+                  subjectBTtoMQTT, "BT: Interval between active scans", (char*)getUniqueId("intervalacts", "").c_str(), //set state_topic,name,uniqueId
+                  "", "", "{{ value_json.intervalacts/1000 }}", //set availability_topic,device_class,value_template,
+                  "{\"intervalacts\":{{value*1000}},\"save\":true}", "", "s", //set,payload_on,payload_off,unit_of_meas,
+                  0, //set  off_delay
+                  "", "", true, subjectMQTTtoBTset, //set,payload_available,payload_not available   ,is a gateway entity, command topic
+                  "", "", "", "", false, // device name, device manufacturer, device model, device ID, retain,
+                  stateClassNone //State Class
+  );
   createDiscovery("number", //set Type
                   subjectBTtoMQTT, "BT: Connnect interval", (char*)getUniqueId("intervalcnct", "").c_str(), //set state_topic,name,uniqueId
                   "", "", "{{ value_json.intervalcnct/60000 }}", //set availability_topic,device_class,value_template,
@@ -978,9 +987,9 @@ void pubMqttDiscovery() {
                   "false", "true" //state_off, state_on
   );
   createDiscovery("switch", //set Type
-                  subjectBTtoMQTT, "BT: Active scan", (char*)getUniqueId("active_scan", "").c_str(), //set state_topic,name,uniqueId
-                  "", "", "{{ value_json.activescan }}", //set availability_topic,device_class,value_template,
-                  "{\"activescan\":true,\"save\":true}", "{\"activescan\":false,\"save\":true}", "", //set,payload_on,payload_off,unit_of_meas,
+                  subjectBTtoMQTT, "BT: Adaptive scan", (char*)getUniqueId("adaptive_scan", "").c_str(), //set state_topic,name,uniqueId
+                  "", "", "{{ value_json.adaptivescan }}", //set availability_topic,device_class,value_template,
+                  "{\"adaptivescan\":true,\"save\":true}", "{\"adaptivescan\":false,\"save\":true}", "", //set,payload_on,payload_off,unit_of_meas,
                   0, //set  off_delay
                   Gateway_AnnouncementMsg, will_Message, true, subjectMQTTtoBTset, //set,payload_available,payload_not available   ,is a gateway entity, command topic
                   "", "", "", "", false, // device name, device manufacturer, device model, device MAC, retain

main/config_BT.h

@@ -74,12 +74,15 @@ extern int btQueueLengthCount;
 #ifndef BLEScanWindow
 #  define BLEScanWindow 30 // How long to scan during the interval; in milliseconds.
 #endif
-#ifndef ActiveBLEScan
-#  define ActiveBLEScan true // Set active scanning, this will get more data from the advertiser.
+#ifndef AdaptiveBLEScan
+#  define AdaptiveBLEScan true // Sets adaptive scanning, this will automatically decide on the best passive and active scanning intervals
 #endif
 #ifndef TimeBtwActive
 #  define TimeBtwActive 55555 //define default time between two BLE active scans when general passive scanning is selected; in milliseconds
 #endif
+#ifndef MinTimeBtwScan
+#  define MinTimeBtwScan 100 //define the time between two scans; in milliseconds
+#endif
 #ifndef TimeBtwConnect
 #  define TimeBtwConnect 3600000 //define default time between BLE connection attempt (not used for immediate actions); in milliseconds
 #endif
@@ -133,7 +136,7 @@ unsigned long scanCount = 0;
 /*----------------CONFIGURABLE PARAMETERS-----------------*/
 struct BTConfig_s {
   bool bleConnect; // Attempt a BLE connection to sensors with ESP32
-  bool activeScan;
+  bool adaptiveScan;
   unsigned long intervalActiveScan; // Time between 2 active scans when generally passive scanning
   unsigned long BLEinterval; // Time between 2 scans
   unsigned long intervalConnect; // Time between 2 connects

platformio.ini

@@ -143,7 +143,7 @@ somfy_remote=Somfy_Remote_Lib@0.3.0
 rtl_433_ESP = https://github.com/NorthernMan54/rtl_433_ESP.git#v0.1.6
 emodbus =  miq19/eModbus@1.0.0
 gfSunInverter = https://github.com/BlackSmith/GFSunInverter.git#v1.0.1
-decoder = https://github.com/theengs/decoder.git#1.1.0
+decoder = https://github.com/theengs/decoder.git
 ssd1306 = thingpulse/ESP8266 and ESP32 OLED driver for SSD1306 displays@^4.3.0
 lm75 = jeremycole/I2C Temperature Sensors derived from the LM75@^1.0.3
 rn8209 = https://github.com/theengs/RN8209C-SDK.git#arduino