Add immediate parameter to BLE connection commands. (#1137)

* Add immediate parameter to BLE connection commands.

When sending a command to read/write a characteristic on a BLE server this adds a new parameter to cause the action to be
performed immediately. This will stop the scan in progress to perform the actions queued.

Example use:
```
mosquitto_pub -t home/OpenMQTTGateway/commands/MQTTtoBT/config -m '{
  "ble_write_address":"AA:BB:CC:DD:EE:FF",
  "ble_write_service":"cba20d00-224d-11e6-9fb8-0002a5d5c51b",
  "ble_write_char":"cba20002-224d-11e6-9fb8-0002a5d5c51b",
  "ble_write_value":"TEST",
  "value_type":"STRING",
  "ttl":4,
  "immediate":true }'
```

Stop BLE processing when immeditate command sent.

* Use a shorter connection timeout.

* Update docs.

* Remove MQTT spam for connection retrys

* Prevent crashing when the normal connection process occurs at the same time as an immediate action.

* Use a semaphore to protect BLE device vector.

docs/use/ble.md

@@ -164,7 +164,8 @@ The default value is set into config_BT.h
 
 The gateway can read and write BLE characteristics from devices and provide the results in an MQTT message.  
 ::: tip
-These actions will be taken on the next BLE connection, which occurs after scanning and after the scan count is reached, [see above to set this.](#setting-the-number-of-scans-between-connection-attempts)
+These actions will be taken on the next BLE connection, which occurs after scanning and after the scan count is reached, [see above to set this.](#setting-the-number-of-scans-between-connection-attempts)  
+This can be overridden by providing an (optional) parameter `"immediate": true` within the command. This will cause the BLE scan to stop if currently in progress, allowing the command to be immediately processed. All other connection commands in queue will also be processed for the same device, commands for other devices will be deferred until the next normally scheduled connection.
 :::
 
 ### Example write command
@@ -175,7 +176,8 @@ mosquitto_pub -t home/OpenMQTTGateway/commands/MQTTtoBT/config -m '{
   "ble_write_char":"cba20002-224d-11e6-9fb8-0002a5d5c51b",
   "ble_write_value":"TEST",
   "value_type":"STRING",
-  "ttl":4 }'
+  "ttl":4,
+  "immediate":true }'
 ```
 Response:
 ```

main/ZgatewayBLEConnect.h

@@ -12,7 +12,10 @@ class zBLEConnect {
 public:
   NimBLEClient* m_pClient;
   TaskHandle_t m_taskHandle = nullptr;
-  zBLEConnect(NimBLEAddress& addr) { m_pClient = NimBLEDevice::createClient(addr); }
+  zBLEConnect(NimBLEAddress& addr) {
+    m_pClient = NimBLEDevice::createClient(addr);
+    m_pClient->setConnectTimeout(5);
+  }
   virtual ~zBLEConnect() { NimBLEDevice::deleteClient(m_pClient); }
   virtual bool writeData(BLEAction* action);
   virtual bool readData(BLEAction* action);

main/ZgatewayBLEConnect.ino

@@ -50,14 +50,14 @@ bool zBLEConnect::writeData(BLEAction* action) {
           std::string temp = action->value.substr(i, 2);
           buf.push_back((uint8_t)strtoul(temp.c_str(), nullptr, 16));
         }
-        return pChar->writeValue((const uint8_t*)&buf[0], buf.size(), !pChar->canWrite());
+        return pChar->writeValue((const uint8_t*)&buf[0], buf.size(), !pChar->canWriteNoResponse());
       }
       case BLE_VAL_INT:
-        return pChar->writeValue(strtol(action->value.c_str(), nullptr, 0), !pChar->canWrite());
+        return pChar->writeValue(strtol(action->value.c_str(), nullptr, 0), !pChar->canWriteNoResponse());
       case BLE_VAL_FLOAT:
-        return pChar->writeValue(strtod(action->value.c_str(), nullptr), !pChar->canWrite());
+        return pChar->writeValue(strtod(action->value.c_str(), nullptr), !pChar->canWriteNoResponse());
       default:
-        return pChar->writeValue(action->value, !pChar->canWrite());
+        return pChar->writeValue(action->value, !pChar->canWriteNoResponse());
     }
   }
   return false;
@@ -82,12 +82,12 @@ bool zBLEConnect::processActions(std::vector<BLEAction>& actions) {
       if (NimBLEAddress(it.addr) == m_pClient->getPeerAddress()) {
         JsonObject BLEresult = getBTJsonObject();
         BLEresult["id"] = it.addr;
-        BLEresult["service"] = (char*)it.service.toString().c_str();
-        BLEresult["characteristic"] = (char*)it.characteristic.toString().c_str();
+        BLEresult["service"] = it.service.toString();
+        BLEresult["characteristic"] = it.characteristic.toString();
 
         if (it.write) {
           Log.trace(F("processing BLE write" CR));
-          BLEresult["write"] = it.value.c_str();
+          BLEresult["write"] = it.value;
           result = writeData(&it);
         } else {
           Log.trace(F("processing BLE read" CR));
@@ -117,9 +117,11 @@ bool zBLEConnect::processActions(std::vector<BLEAction>& actions) {
           }
         }
 
-        it.complete = true;
+        it.complete = result;
         BLEresult["success"] = result;
-        pubBT(BLEresult);
+        if (result || it.ttl <= 1) {
+          pubBT(BLEresult);
+        }
       }
     }
   }

main/ZgatewayBT.ino

@@ -35,6 +35,7 @@ Thanks to wolass https://github.com/wolass for suggesting me HM 10 and dinosd ht
 #  ifdef ESP32
 #    include "FreeRTOS.h"
 SemaphoreHandle_t semaphoreCreateOrUpdateDevice;
+SemaphoreHandle_t semaphoreBLEOperation;
 QueueHandle_t BLEQueue;
 // Headers used for deep sleep functions
 #    include <NimBLEAdvertisedDevice.h>
@@ -255,7 +256,7 @@ void createOrUpdateDevice(const char* mac, uint8_t flags, std::string model) {
       device->connect = true;
     }
 
-    if (model.compare("") == 0) device->sensorModel_id = model;
+    if (!model.empty()) device->sensorModel_id = model;
 
     if (flags & device_flags_isWhiteL || flags & device_flags_isBlackL) {
       device->isWhtL = flags & device_flags_isWhiteL;
@@ -458,43 +459,56 @@ void BLEscan() {
  * Connect to BLE devices and initiate the callbacks with a service/characteristic request
  */
 void BLEconnect() {
-  Log.notice(F("BLE Connect begin" CR));
-  for (vector<BLEdevice*>::iterator it = devices.begin(); it != devices.end(); ++it) {
-    BLEdevice* p = *it;
-    if (p->connect) {
-      Log.trace(F("Model to connect found: %s" CR), p->macAdr);
-      NimBLEAddress addr(std::string(p->macAdr));
-      if (p->sensorModel_id.compare("LYWSD03MMC") == 0 || p->sensorModel_id.compare("MHO-C401") == 0) {
-        LYWSD03MMC_connect BLEclient(addr);
-        BLEclient.processActions(BLEactions);
-        BLEclient.publishData();
-      }
-      if (p->sensorModel_id.compare("DT24-BLE") == 0) {
-        DT24_connect BLEclient(addr);
-        BLEclient.processActions(BLEactions);
-        BLEclient.publishData();
-      }
-      if (p->sensorModel_id.compare("GENERIC") == 0) {
-        GENERIC_connect BLEclient(addr);
-        BLEclient.processActions(BLEactions);
-      }
-      if (p->sensorModel_id.compare("HHCCJCY01HHCC") == 0) {
-        HHCCJCY01HHCC_connect BLEclient(addr);
-        BLEclient.processActions(BLEactions);
-        BLEclient.publishData();
-      }
-      if (BLEactions.size() > 0) {
-        std::vector<BLEAction> swap;
-        for (auto& it : BLEactions) {
-          if (!it.complete && --it.ttl) {
-            swap.push_back(it);
+  if (!ProcessLock) {
+    Log.notice(F("BLE Connect begin" CR));
+    do {
+      for (vector<BLEdevice*>::iterator it = devices.begin(); it != devices.end(); ++it) {
+        BLEdevice* p = *it;
+        if (p->connect) {
+          Log.trace(F("Model to connect found: %s" CR), p->macAdr);
+          NimBLEAddress addr(std::string(p->macAdr));
+          if (p->sensorModel_id.compare("LYWSD03MMC") == 0 || p->sensorModel_id.compare("MHO-C401") == 0) {
+            LYWSD03MMC_connect BLEclient(addr);
+            BLEclient.processActions(BLEactions);
+            BLEclient.publishData();
+          } else if (p->sensorModel_id.compare("DT24-BLE") == 0) {
+            DT24_connect BLEclient(addr);
+            BLEclient.processActions(BLEactions);
+            BLEclient.publishData();
+          } else if (p->sensorModel_id.compare("HHCCJCY01HHCC") == 0) {
+            HHCCJCY01HHCC_connect BLEclient(addr);
+            BLEclient.processActions(BLEactions);
+            BLEclient.publishData();
+          } else {
+            GENERIC_connect BLEclient(addr);
+            if (BLEclient.processActions(BLEactions)) {
+              // If we don't regularly connect to this, disable connections so advertisements
+              // won't be filtered if BLE_FILTER_CONNECTABLE is set.
+              p->connect = false;
+            }
+          }
+          if (BLEactions.size() > 0) {
+            std::vector<BLEAction> swap;
+            for (auto& it : BLEactions) {
+              if (!it.complete && --it.ttl) {
+                swap.push_back(it);
+              } else if (memcmp(it.addr, p->macAdr, sizeof(it.addr)) == 0) {
+                if (p->sensorModel_id != "DT24-BLE" &&
+                    p->sensorModel_id != "HHCCJCY01HHCC" &&
+                    p->sensorModel_id != "LYWSD03MMC" &&
+                    p->sensorModel_id != "MHO-C401") {
+                  // if irregulary connected to and connection failed clear the connect flag.
+                  p->connect = false;
+                }
+              }
+            }
+            std::swap(BLEactions, swap);
           }
         }
-        std::swap(BLEactions, swap);
       }
-    }
+    } while (BLEactions.size() > 0);
+    Log.notice(F("BLE Connect end" CR));
   }
-  Log.notice(F("BLE Connect end" CR));
 }
 
 void stopProcessing() {
@@ -520,11 +534,16 @@ void coreTask(void* pvParameters) {
       if (client.state() != 0) {
         Log.warning(F("MQTT client disconnected no BLE scan" CR));
       } else if (!ProcessLock) {
-        BLEscan();
-        // Launching a connect every BLEscanBeforeConnect
-        if ((!(scanCount % BLEscanBeforeConnect) || scanCount == 1) && bleConnect)
-          BLEconnect();
-        dumpDevices();
+        if (xSemaphoreTake(semaphoreBLEOperation, pdMS_TO_TICKS(30000)) == pdTRUE) {
+          BLEscan();
+          // Launching a connect every BLEscanBeforeConnect
+          if ((!(scanCount % BLEscanBeforeConnect) || scanCount == 1) && bleConnect)
+            BLEconnect();
+          dumpDevices();
+          xSemaphoreGive(semaphoreBLEOperation);
+        } else {
+          Log.error(F("Failed to start scan - BLE busy" CR));
+        }
       }
       if (lowpowermode) {
         lowPowerESP32();
@@ -604,6 +623,9 @@ void setupBT() {
   semaphoreCreateOrUpdateDevice = xSemaphoreCreateBinary();
   xSemaphoreGive(semaphoreCreateOrUpdateDevice);
 
+  semaphoreBLEOperation = xSemaphoreCreateBinary();
+  xSemaphoreGive(semaphoreBLEOperation);
+
   BLEQueue = xQueueCreate(32, sizeof(NimBLEAdvertisedDevice*));
 
   BLEDevice::setScanDuplicateCacheSize(BLEScanDuplicateCacheSize);
@@ -968,8 +990,47 @@ void MQTTtoBTAction(JsonObject& BTdata) {
   } else {
     return;
   }
-  createOrUpdateDevice(action.addr, device_flags_connect, "GENERIC");
-  BLEactions.push_back(action);
+
+  createOrUpdateDevice(action.addr, device_flags_connect, "");
+
+  if (BTdata.containsKey("immediate") && BTdata["immediate"].as<bool>()) {
+    // Immediate action; we need to prevent the normal connection action and stop scanning
+    ProcessLock = true;
+    NimBLEScan* pScan = NimBLEDevice::getScan();
+    if (pScan->isScanning()) {
+      pScan->stop();
+    }
+
+    if (xSemaphoreTake(semaphoreBLEOperation, pdMS_TO_TICKS(5000)) == pdTRUE) {
+      // swap the vectors so only this device is processed
+      std::vector<BLEdevice*> dev_swap;
+      dev_swap.push_back(getDeviceByMac(action.addr));
+      std::swap(devices, dev_swap);
+
+      std::vector<BLEAction> act_swap;
+      act_swap.push_back(action);
+      std::swap(BLEactions, act_swap);
+
+      // Unlock here to allow the action to be performed
+      ProcessLock = false;
+      BLEconnect();
+      // back to normal
+      std::swap(devices, dev_swap);
+      std::swap(BLEactions, act_swap);
+      // If we stopped the scheduled connect for this action, do the scheduled now
+      if ((!(scanCount % BLEscanBeforeConnect) || scanCount == 1) && bleConnect)
+        BLEconnect();
+      xSemaphoreGive(semaphoreBLEOperation);
+    } else {
+      Log.error(F("BLE busy - command not sent" CR));
+      JsonObject result = getBTJsonObject();
+      result["id"] = action.addr;
+      result["success"] = false;
+      pubBT(result);
+    }
+  } else {
+    BLEactions.push_back(action);
+  }
 #  endif
 }