Remove Secure Connection macro and add the config to wifimanager. (#961)

This adds the ability to specify if the MQTT broker connection is secure via a checkbox on the config page of WiFiManager.
Also added to the WiFiManager config page is a text box to enter the brokers TLS certificate.

In the case of using manual configuration a new macro is created MQTT_SECURE_DEFAULT.
When defined as true a secure connection will be requested and the certificate defined in user_config.h will be used.

docs/upload/advanced-configuration.md

@@ -13,7 +13,7 @@ The MQTT broker is configured for TLS and you have access to the CA certificate
 You are using ESP8266 or ESP32, for other boards TLS is not supported.
 
 ### Configure secure connection in the gateway
-To enable the secure connection and use TLS uncomment `//#define SECURE_CONNECTION` in `User_config.h`.
+To enable the secure connection and use TLS set the `#define MQTT_DEFAULT_SECURE` to true.
 Set `MQTT_SERVER` to the Common Name (CN) of the certificate of the broker.
 This can be the hostname or the ip of the broker.
 
@@ -28,9 +28,6 @@ const char* certificate CERT_ATTRIBUTE = R"EOF("
 ")EOF";
 ```
 
-If you have no ntp server in your local network (included in the router) or not using dhcp, you should uncomment `//#    define NTP_SERVER "pool.ntp.org"` to use a ntp server for time synchronization.
-This is related to the `W: failed, ssl error code=54` error message, which indicate that the time of the esp is not correct.
-
 You can know compile and upload to your board and the gateway should connect with TLS to your broker.
 
 ## Add the reveived "value" at the end of the topic

docs/upload/portal.md

@@ -14,6 +14,9 @@ From your smartphone search for your OpenMQTTGateway wifi network and connect to
 * Select your wifi
 * Set your wifi password
 * Set your MQTT Server IP
+* Set your MQTT Server Port
+* Set the MQTT secure connection box to select whether or not the connection should be secure
+* Copy/paste the MQTT server certificate in the MQTT server cert box (only required if using a secure connection), be sure to include the `-----BEGIN CERIFICATE-----` and `-----END CERTIFICATE-----` markers  
 * Set your MQTT Server username (facultative)
 * Set your MQTT Server password (facultative)
 * Set your MQTT base topic if you need to change it (you must keep the / at the end)

main/User_config.h

@@ -80,6 +80,10 @@ const byte ip[] = {192, 168, 1, 99};
 const byte mac[] = {0xDE, 0xED, 0xBA, 0xFE, 0x54, 0x95}; //W5100 ethernet shield mac adress
 #endif
 
+#ifndef NTP_SERVER
+#  define NTP_SERVER "pool.ntp.org"
+#endif
+
 #ifdef MQTT_HTTPS_FW_UPDATE
 #  if defined(ESP8266) || defined(ESP32)
 //If used, this should be set to the root CA certificate of the server hosting the firmware.
@@ -89,7 +93,6 @@ const char* https_fw_server_cert PROGMEM = R"EOF("
 ...
 -----END CERTIFICATE-----
 ")EOF";
-#    define NTP_SERVER "pool.ntp.org"
 #    ifndef MQTT_HTTPS_FW_UPDATE_USE_PASSWORD
 #      define MQTT_HTTPS_FW_UPDATE_USE_PASSWORD 1 // Set this to 0 if not using TLS connection to MQTT broker to prevent clear text passwords being sent.
 #    endif
@@ -142,16 +145,6 @@ const char* https_fw_server_cert PROGMEM = R"EOF("
 #  define mqtt_max_packet_size 128
 #endif
 
-// activate the use of TLS for secure connection to the MQTT broker
-// MQTT_SERVER must be set to the Common Name (CN) of the broker's certificate
-//#define SECURE_CONNECTION
-
-#ifdef SECURE_CONNECTION
-#  define MQTT_DEFAULT_PORT "8883"
-#else
-#  define MQTT_DEFAULT_PORT "1883"
-#endif
-
 #ifndef MQTT_USER
 #  define MQTT_USER "your_username"
 #endif
@@ -162,11 +155,13 @@ const char* https_fw_server_cert PROGMEM = R"EOF("
 #  define MQTT_SERVER "192.168.1.17"
 #endif
 #ifndef MQTT_PORT
-#  define MQTT_PORT MQTT_DEFAULT_PORT
+#  define MQTT_PORT "1883"
+#endif
+#ifndef MQTT_SECURE_DEFAULT
+#  define MQTT_SECURE_DEFAULT false
 #endif
 
-#ifdef SECURE_CONNECTION
+#if defined(ESP8266) || defined(ESP32)
-#  if defined(ESP8266) || defined(ESP32)
 // The root ca certificate used for validating the MQTT broker
 // The certificate must be in PEM ascii format
 const char* certificate PROGMEM = R"EOF("
@@ -175,16 +170,6 @@ const char* certificate PROGMEM = R"EOF("
 -----END CERTIFICATE-----
 ")EOF";
 
-// specify a NTP server here or else the NTP server from DHCP is used
-#    ifndef NTP_SERVER
-//#    define NTP_SERVER "pool.ntp.org"
-#    endif
-#  else
-#    error "only ESP8266 and ESP32 support SECURE_CONNECTION with TLS"
-#  endif
-#endif
-
-#if defined(ESP8266) || defined(ESP32)
 #  define ATTEMPTS_BEFORE_BG 10 // Number of wifi connection attempts before going to BG protocol
 #  define ATTEMPTS_BEFORE_B  20 // Number of wifi connection attempts before going to B protocol
 #endif

main/main.ino

@@ -173,6 +173,9 @@ int failure_number_mqtt = 0; // number of failure connecting to MQTT
 bool disc = true; // Auto discovery with Home Assistant convention
 #endif
 unsigned long timer_led_measures = 0;
+static void* eClient = nullptr;
+static bool mqtt_secure = MQTT_SECURE_DEFAULT;
+static String mqtt_cert = "";
 
 #ifdef ESP32
 #  include <ArduinoOTA.h>
@@ -186,13 +189,8 @@ unsigned long timer_led_measures = 0;
 void WiFiEvent(WiFiEvent_t event);
 static bool esp32EthConnected = false;
 #  endif
-#  ifdef SECURE_CONNECTION
+
-#    include <WiFiClientSecure.h>
+#  include <WiFiClientSecure.h>
-WiFiClientSecure eClient;
-#  else
-#    include <WiFi.h>
-WiFiClient eClient;
-#  endif
 #  include <WiFiMulti.h>
 WiFiMulti wifiMulti;
 #  include <Preferences.h>
@@ -201,6 +199,7 @@ Preferences preferences;
 #  ifdef MDNS_SD
 #    include <ESPmDNS.h>
 #  endif
+
 #elif defined(ESP8266)
 #  include <ArduinoOTA.h>
 #  include <DNSServer.h>
@@ -209,26 +208,21 @@ Preferences preferences;
 #  include <ESP8266WiFiMulti.h>
 #  include <FS.h>
 #  include <WiFiManager.h>
-#  ifdef SECURE_CONNECTION
+X509List caCert;
-WiFiClientSecure eClient;
-X509List caCert(certificate);
-#  else
-WiFiClient eClient;
-#  endif
 ESP8266WiFiMulti wifiMulti;
 #  ifdef MDNS_SD
 #    include <ESP8266mDNS.h>
 #  endif
+
 #else
 #  include <Ethernet.h>
-EthernetClient eClient;
 #endif
 
 #define convertTemp_CtoF(c) ((c * 1.8) + 32)
 #define convertTemp_FtoC(f) ((f - 32) * 5 / 9)
 
 // client link to pubsub mqtt
-PubSubClient client(eClient);
+PubSubClient client;
 
 void revert_hex_data(const char* in, char* out, int l) {
   //reverting array 2 by 2 to get the data in good order
@@ -517,10 +511,10 @@ void connectMQTT() {
     failure_number_mqtt++; // we count the failure
     Log.warning(F("failure_number_mqtt: %d" CR), failure_number_mqtt);
     Log.warning(F("failed, rc=%d" CR), client.state());
-#if defined(SECURE_CONNECTION) && defined(ESP32)
+#if defined(ESP32)
-    Log.warning(F("failed, ssl error code=%d" CR), eClient.lastError(nullptr, 0));
+    Log.warning(F("failed, ssl error code=%d" CR), ((WiFiClientSecure*)eClient)->lastError(nullptr, 0));
-#elif defined(SECURE_CONNECTION) && defined(ESP8266)
+#elif defined(ESP8266)
-    Log.warning(F("failed, ssl error code=%d" CR), eClient.getLastSSLError());
+    Log.warning(F("failed, ssl error code=%d" CR), ((WiFiClientSecure*)eClient)->getLastSSLError());
 #endif
     digitalWrite(LED_INFO, LED_INFO_ON);
     delay(1000);
@@ -590,9 +584,6 @@ void setup() {
 #  endif
 
   setOTA();
-#  ifdef SECURE_CONNECTION
-  setupTLS();
-#  endif
 #else // In case of arduino platform
 
   //Launch serial for debugging purposes
@@ -617,6 +608,18 @@ void setup() {
   port = strtol(mqtt_port, NULL, 10);
   Log.trace(F("Port: %l" CR), port);
   Log.trace(F("Mqtt server: %s" CR), mqtt_server);
+#  if defined(ESP8266) || defined(ESP32)
+  if (mqtt_secure) {
+    eClient = new WiFiClientSecure;
+    setupTLS();
+  } else {
+    eClient = new WiFiClient;
+  }
+#  else
+  eClient = new EthernetClient;
+#  endif
+
+  client.setClient(*(Client*)eClient);
   client.setServer(mqtt_server, port);
 #endif
 
@@ -848,19 +851,24 @@ void setOTA() {
   ArduinoOTA.begin();
 }
 
-#  ifdef SECURE_CONNECTION
 void setupTLS() {
-#    if defined(NTP_SERVER)
   configTime(0, 0, NTP_SERVER);
-#    endif
+  WiFiClientSecure* sClient = (WiFiClientSecure*)eClient;
-#    if defined(ESP32)
+  if (mqtt_cert.length() > 0) {
-  eClient.setCACert(certificate);
+#  if defined(ESP32)
-#    elif defined(ESP8266)
+    sClient->setCACert(mqtt_cert.c_str());
-  eClient.setTrustAnchors(&caCert);
+  } else {
-  eClient.setBufferSizes(512, 512);
+    sClient->setCACert(certificate);
-#    endif
+  }
-}
+#  elif defined(ESP8266)
+    caCert.append(mqtt_cert.c_str());
+  } else {
+    caCert.append(certificate);
+  }
+  sClient->setTrustAnchors(&caCert);
+  sClient->setBufferSizes(512, 512);
 #  endif
+}
 #endif
 
 #if defined(ESPWifiManualSetup)
@@ -904,7 +912,7 @@ void setup_wifi() {
 WiFiManager wifiManager;
 
 //flag for saving data
-bool shouldSaveConfig = true;
+bool shouldSaveConfig = false;
 //do we have been connected once to mqtt
 
 //callback notifying us of the need to save config
@@ -999,6 +1007,10 @@ void setup_wifimanager(bool reset_settings) {
           strcpy(mqtt_pass, json["mqtt_pass"]);
         if (json.containsKey("mqtt_topic"))
           strcpy(mqtt_topic, json["mqtt_topic"]);
+        if (json.containsKey("mqtt_secure"))
+          mqtt_secure = json.get<bool>("mqtt_secure");
+        if (json.containsKey("mqtt_cert"))
+          mqtt_cert = json.get<const char*>("mqtt_cert");
         if (json.containsKey("gateway_name"))
           strcpy(gateway_name, json["gateway_name"]);
       } else {
@@ -1015,6 +1027,8 @@ void setup_wifimanager(bool reset_settings) {
   WiFiManagerParameter custom_mqtt_user("user", "mqtt user", mqtt_user, parameters_size);
   WiFiManagerParameter custom_mqtt_pass("pass", "mqtt pass", mqtt_pass, parameters_size * 2);
   WiFiManagerParameter custom_mqtt_topic("topic", "mqtt base topic", mqtt_topic, mqtt_topic_max_size);
+  WiFiManagerParameter custom_mqtt_secure("secure", "mqtt secure", "1", 1, mqtt_secure ? "type=\"checkbox\" checked" : "type=\"checkbox\"");
+  WiFiManagerParameter custom_mqtt_cert("cert", "mqtt broker cert", mqtt_cert.c_str(), 2048);
   WiFiManagerParameter custom_gateway_name("name", "gateway name", gateway_name, parameters_size * 2);
 
   //WiFiManager
@@ -1041,6 +1055,8 @@ void setup_wifimanager(bool reset_settings) {
   wifiManager.addParameter(&custom_mqtt_port);
   wifiManager.addParameter(&custom_mqtt_user);
   wifiManager.addParameter(&custom_mqtt_pass);
+  wifiManager.addParameter(&custom_mqtt_secure);
+  wifiManager.addParameter(&custom_mqtt_cert);
   wifiManager.addParameter(&custom_gateway_name);
   wifiManager.addParameter(&custom_mqtt_topic);
 
@@ -1083,16 +1099,35 @@ void setup_wifimanager(bool reset_settings) {
     M5Display("Wifi connected", "", "");
 #  endif
 
-  //read updated parameters
-  strcpy(mqtt_server, custom_mqtt_server.getValue());
-  strcpy(mqtt_port, custom_mqtt_port.getValue());
-  strcpy(mqtt_user, custom_mqtt_user.getValue());
-  strcpy(mqtt_pass, custom_mqtt_pass.getValue());
-  strcpy(mqtt_topic, custom_mqtt_topic.getValue());
-  strcpy(gateway_name, custom_gateway_name.getValue());
-
-  //save the custom parameters to FS
   if (shouldSaveConfig) {
+    //read updated parameters
+    strcpy(mqtt_server, custom_mqtt_server.getValue());
+    strcpy(mqtt_port, custom_mqtt_port.getValue());
+    strcpy(mqtt_user, custom_mqtt_user.getValue());
+    strcpy(mqtt_pass, custom_mqtt_pass.getValue());
+    strcpy(mqtt_topic, custom_mqtt_topic.getValue());
+    strcpy(gateway_name, custom_gateway_name.getValue());
+    mqtt_secure = *custom_mqtt_secure.getValue();
+
+    int cert_len = strlen(custom_mqtt_cert.getValue());
+    if (cert_len) {
+      char* cert_in = (char*)custom_mqtt_cert.getValue();
+      while (*cert_in == ' ' || *cert_in == '\t') {
+        cert_in++;
+      }
+
+      char* cert_begin = cert_in;
+      while (*cert_in != NULL) {
+        if (*cert_in == ' ' && (strncmp((cert_in + 1), "CERTIFICATE", 11) != 0)) {
+          *cert_in = '\n';
+        }
+        cert_in++;
+      }
+
+      mqtt_cert = cert_begin;
+    }
+
+    //save the custom parameters to FS
     Log.trace(F("saving config" CR));
     DynamicJsonBuffer jsonBuffer;
     JsonObject& json = jsonBuffer.createObject();
@@ -1102,6 +1137,8 @@ void setup_wifimanager(bool reset_settings) {
     json["mqtt_pass"] = mqtt_pass;
     json["mqtt_topic"] = mqtt_topic;
     json["gateway_name"] = gateway_name;
+    json["mqtt_secure"] = mqtt_secure;
+    json["mqtt_cert"] = mqtt_cert;
 
     File configFile = SPIFFS.open("/config.json", "w");
     if (!configFile) {
@@ -1623,12 +1660,11 @@ void receivingMQTT(char* topicOri, char* datacallback) {
 }
 
 #ifdef MQTT_HTTPS_FW_UPDATE
-#  ifndef NTP_SERVER
-#    error no NTP_SERVER defined
-#  endif
 #  include <WiFiClientSecure.h>
+
+#  include "Ota_github.h"
+
 #  ifdef ESP32
-#    include "Ota_github.h"
 #    include "zzHTTPUpdate.h"
 #  elif ESP8266
 #    include <ESP8266httpUpdate.h>
@@ -1649,9 +1685,6 @@ void MQTTHttpsFWUpdate(char* topicOri, JsonObject& HttpsFwUpdateData) {
       }
 
 #  if MQTT_HTTPS_FW_UPDATE_USE_PASSWORD > 0
-#    ifndef SECURE_CONNECTION
-#      warning using a password with an unsecure MQTT connection will send it as clear text!!!
-#    endif
       const char* pwd = HttpsFwUpdateData["password"];
       if (pwd) {
         if (strcmp(pwd, ota_password) != 0) {
@@ -1683,38 +1716,46 @@ void MQTTHttpsFWUpdate(char* topicOri, JsonObject& HttpsFwUpdateData) {
 
       } else {
         WiFiClientSecure update_client;
-#  ifdef SECURE_CONNECTION
+        if (mqtt_secure) {
-        client.disconnect();
+          client.disconnect();
-        update_client = eClient;
+          update_client = *(WiFiClientSecure*)eClient;
-#  else
+        } else {
-        configTime(0, 0, NTP_SERVER);
+          configTime(0, 0, NTP_SERVER);
-        time_t now = time(nullptr);
+          time_t now = time(nullptr);
-        uint8_t count = 0;
+          uint8_t count = 0;
-        Log.trace(F("Waiting for NTP time sync" CR));
+          Log.trace(F("Waiting for NTP time sync" CR));
-        while ((now < 8 * 3600 * 2) && count++ < 60) {
+          while ((now < 8 * 3600 * 2) && count++ < 60) {
-          vTaskDelay(500);
+            delay(500);
-          now = time(nullptr);
+            now = time(nullptr);
+          }
+
+          if (count >= 60) {
+            Log.error(F("Unable to update - invalid time" CR));
+#  if defined(ZgatewayBT) && defined(ESP32)
+            startProcessing();
+#  endif
+            return;
+          }
         }
 
-        if (count >= 60) {
-          Log.error(F("Unable to update - invalid time" CR));
-#    if defined(ZgatewayBT) && defined(ESP32)
-          startProcessing();
-#    endif
-          return;
-        }
-#  endif
 #  ifdef ESP32
         if (strstr(url, "github") != 0) {
           update_client.setCACert(_github_cert);
         } else {
           update_client.setCACert(https_fw_server_cert);
         }
+
         update_client.setTimeout(12);
         httpUpdate.setFollowRedirects(HTTPC_STRICT_FOLLOW_REDIRECTS);
         result = httpUpdate.update(update_client, url);
 #  elif ESP8266
-        update_client.setInsecure(); // TODO: replace with cert checking
+        if (strstr(url, "github") != 0) {
+          caCert.append(_github_cert);
+        } else {
+          caCert.append(https_fw_server_cert);
+        }
+
+        update_client.setTrustAnchors(&caCert);
         update_client.setTimeout(12000);
         ESPhttpUpdate.setFollowRedirects(HTTPC_STRICT_FOLLOW_REDIRECTS);
         result = ESPhttpUpdate.update(update_client, url);

platformio.ini

@@ -71,8 +71,8 @@ extra_configs =
 ;default_envs = uno-fastled
 ;default_envs = atmega-all-test
 ;default_envs = manual-wifi-test
-;default_envs = esp32dev-tls-test
+;default_envs = esp32dev-mqtt-fw-test
-;default_envs = nodemcuv2-tls-test
+;default_envs = nodemcuv2-mqtt-fw-test
 ;default_envs = nodemcuv2-rs232
 ;default_envs = sonoff-rfbridge-direct
 
@@ -105,7 +105,7 @@ bmp180 = BMP180#efac46bd8d
 htu21 = SparkFun HTU21D Humidity and Temperature Sensor Breakout@1.1.3
 ahtx0 = Adafruit AHTX0
 ina226 = https://github.com/jarzebski/Arduino-INA226.git#968a684
-a6lib = https://github.com/1technophile/A6lib.git
+a6lib = https://github.com/h2zero/A6lib
 wifimanager = https://github.com/tzapu/WiFiManager.git#c3ff582
 ethernet = Ethernet
 esp8266_mdns = esp8266_mdns
@@ -826,29 +826,27 @@ build_flags =
   '-DGateway_Name="OpenMQTTGateway_TEST_MANUAL_WIFI"'
 board_build.flash_mode = dout
 
-[env:esp32dev-tls-test]
+[env:esp32dev-mqtt-fw-test]
 platform = ${com.esp32_platform}
 board = esp32dev
 lib_deps =
   ${com-esp.lib_deps}
 build_flags =
   ${com-esp.build_flags}
-  '-DSECURE_CONNECTION'
   '-DMQTT_HTTPS_FW_UPDATE'
-  '-DGateway_Name="OpenMQTTGateway_TEST_TLS"'
+  '-DGateway_Name="OpenMQTTGateway_TEST_MQTT_FW"'
 board_build.flash_mode = dout
 
 
-[env:nodemcuv2-tls-test]
+[env:nodemcuv2-mqtt-fw-test]
 platform = ${com.esp8266_platform}
 board = nodemcuv2
 lib_deps =
   ${com-esp.lib_deps}
 build_flags =
   ${com-esp.build_flags}
-  '-DSECURE_CONNECTION'
   '-DMQTT_HTTPS_FW_UPDATE'
-  '-DGateway_Name="OpenMQTTGateway_TEST_TLS"'
+  '-DGateway_Name="OpenMQTTGateway_TEST_MQTT_FW"'
 board_build.flash_mode = dout
 
 [env:rf-wifi-gateway]