[SYS] Better handling of certs and OTA checks (#1695)

Add a macro to enable removal of update checks
Separate certificates from user_config.h
Consider latest version as the default
Do not use a cert if the url is not
Simplify AWS macro scope

.gitignore

@@ -4,3 +4,4 @@
 **/.DS_Store
 node_modules
 *.ps1
+main/certs/private*

main/User_config.h

@@ -148,6 +148,7 @@ const byte mac[] = {0xDE, 0xED, 0xBA, 0xFE, 0x54, 0x95}; //W5100 ethernet shield
 #  define mqtt_topic_max_size 150
 #  ifndef mqtt_max_packet_size
 #    ifdef MQTT_HTTPS_FW_UPDATE
+#      define CHECK_OTA_UPDATE     true // enable to check for the presence of a new version for your environment on Github
 #      define mqtt_max_packet_size 2560
 #    else
 #      define mqtt_max_packet_size 1024
@@ -208,32 +209,11 @@ const char* certificate PROGMEM = R"EOF("
 
 #  ifdef MQTT_HTTPS_FW_UPDATE
 // If used, this should be set to the root CA certificate of the server hosting the firmware.
-// The certificate must be in PEM ascii format.
+#    ifdef PRIVATE_CERTS
-// The default certificate is for github.
+#      include "certs/private_ota_cert.h"
-const char* OTAserver_cert PROGMEM = R"EOF("
+#    else
------BEGIN CERTIFICATE-----
+#      include "certs/default_ota_cert.h"
-MIIDrzCCApegAwIBAgIQCDvgVpBCRrGhdWrJWZHHSjANBgkqhkiG9w0BAQUFADBh
+#    endif
-MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
-d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBD
-QTAeFw0wNjExMTAwMDAwMDBaFw0zMTExMTAwMDAwMDBaMGExCzAJBgNVBAYTAlVT
-MRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5j
-b20xIDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IENBMIIBIjANBgkqhkiG
-9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4jvhEXLeqKTTo1eqUKKPC3eQyaKl7hLOllsB
-CSDMAZOnTjC3U/dDxGkAV53ijSLdhwZAAIEJzs4bg7/fzTtxRuLWZscFs3YnFo97
-nh6Vfe63SKMI2tavegw5BmV/Sl0fvBf4q77uKNd0f3p4mVmFaG5cIzJLv07A6Fpt
-43C/dxC//AH2hdmoRBBYMql1GNXRor5H4idq9Joz+EkIYIvUX7Q6hL+hqkpMfT7P
-T19sdl6gSzeRntwi5m3OFBqOasv+zbMUZBfHWymeMr/y7vrTC0LUq7dBMtoM1O/4
-gdW7jVg/tRvoSSiicNoxBN33shbyTApOB6jtSj1etX+jkMOvJwIDAQABo2MwYTAO
-BgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUA95QNVbR
-TLtm8KPiGxvDl7I90VUwHwYDVR0jBBgwFoAUA95QNVbRTLtm8KPiGxvDl7I90VUw
-DQYJKoZIhvcNAQEFBQADggEBAMucN6pIExIK+t1EnE9SsPTfrgT1eXkIoyQY/Esr
-hMAtudXH/vTBH1jLuG2cenTnmCmrEbXjcKChzUyImZOMkXDiqw8cvpOp/2PV5Adg
-06O/nVsJ8dWO41P0jmP6P6fbtGbfYmbW0W5BjfIttep3Sp+dWOIrWcBAI+0tKIJF
-PnlUkiaY4IBIqDfv8NZ5YBberOgOzW6sRBc4L0na4UU+Krk2U886UAb3LujEV0ls
-YSEY1QSteDwsOoBrp+uvFRTp2InBuThs4pFsiv9kuXclVzDAGySj4dzp30d8tbQk
-CAUw7C29C79Fv1C5qfPrmAESrciIxpg0X40KPMbp1ZWVbd4=
------END CERTIFICATE-----
-")EOF";
 
 #    ifndef MQTT_HTTPS_FW_UPDATE_USE_PASSWORD
 #      define MQTT_HTTPS_FW_UPDATE_USE_PASSWORD 1 // Set this to 0 if not using TLS connection to MQTT broker to prevent clear text passwords being sent.
@@ -261,23 +241,15 @@ CAUw7C29C79Fv1C5qfPrmAESrciIxpg0X40KPMbp1ZWVbd4=
 #  endif
 
 #  if MQTT_SECURE_SELF_SIGNED
-const char* ss_server_cert PROGMEM = R"EOF("
+#    ifdef PRIVATE_CERTS
------BEGIN CERTIFICATE-----
+#      include "certs/private_client_cert.h"
-...
+#      include "certs/private_client_key.h"
------END CERTIFICATE-----
+#      include "certs/private_server_cert.h"
-")EOF";
+#    else
-
+#      include "certs/default_client_cert.h"
-const char* ss_client_cert PROGMEM = R"EOF("
+#      include "certs/default_client_key.h"
------BEGIN CERTIFICATE-----
+#      include "certs/default_server_cert.h"
-...
+#    endif
------END CERTIFICATE-----
-")EOF";
-
-const char* ss_client_key PROGMEM = R"EOF("
------BEGIN RSA PRIVATE KEY-----
-...
------END RSA PRIVATE KEY-----
-")EOF";
 
 struct ss_certs {
   const char* server_cert;

main/certs/default_client_cert.h

@@ -0,0 +1,5 @@
+const char* ss_client_cert PROGMEM = R"EOF("
+-----BEGIN CERTIFICATE-----
+...
+-----END CERTIFICATE-----
+")EOF";

main/certs/default_client_key.h

@@ -0,0 +1,5 @@
+const char* ss_client_key PROGMEM = R"EOF("
+-----BEGIN RSA PRIVATE KEY-----
+...
+-----END RSA PRIVATE KEY-----
+")EOF";

main/certs/default_ota_cert.h

@@ -0,0 +1,26 @@
+// The certificate must be in PEM ascii format.
+// The default certificate is for github.
+const char* OTAserver_cert PROGMEM = R"EOF("
+-----BEGIN CERTIFICATE-----
+MIIDrzCCApegAwIBAgIQCDvgVpBCRrGhdWrJWZHHSjANBgkqhkiG9w0BAQUFADBh
+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
+d3cuZGlnaWNlcnQuY29tMSAwHgYDVQQDExdEaWdpQ2VydCBHbG9iYWwgUm9vdCBD
+QTAeFw0wNjExMTAwMDAwMDBaFw0zMTExMTAwMDAwMDBaMGExCzAJBgNVBAYTAlVT
+MRUwEwYDVQQKEwxEaWdpQ2VydCBJbmMxGTAXBgNVBAsTEHd3dy5kaWdpY2VydC5j
+b20xIDAeBgNVBAMTF0RpZ2lDZXJ0IEdsb2JhbCBSb290IENBMIIBIjANBgkqhkiG
+9w0BAQEFAAOCAQ8AMIIBCgKCAQEA4jvhEXLeqKTTo1eqUKKPC3eQyaKl7hLOllsB
+CSDMAZOnTjC3U/dDxGkAV53ijSLdhwZAAIEJzs4bg7/fzTtxRuLWZscFs3YnFo97
+nh6Vfe63SKMI2tavegw5BmV/Sl0fvBf4q77uKNd0f3p4mVmFaG5cIzJLv07A6Fpt
+43C/dxC//AH2hdmoRBBYMql1GNXRor5H4idq9Joz+EkIYIvUX7Q6hL+hqkpMfT7P
+T19sdl6gSzeRntwi5m3OFBqOasv+zbMUZBfHWymeMr/y7vrTC0LUq7dBMtoM1O/4
+gdW7jVg/tRvoSSiicNoxBN33shbyTApOB6jtSj1etX+jkMOvJwIDAQABo2MwYTAO
+BgNVHQ8BAf8EBAMCAYYwDwYDVR0TAQH/BAUwAwEB/zAdBgNVHQ4EFgQUA95QNVbR
+TLtm8KPiGxvDl7I90VUwHwYDVR0jBBgwFoAUA95QNVbRTLtm8KPiGxvDl7I90VUw
+DQYJKoZIhvcNAQEFBQADggEBAMucN6pIExIK+t1EnE9SsPTfrgT1eXkIoyQY/Esr
+hMAtudXH/vTBH1jLuG2cenTnmCmrEbXjcKChzUyImZOMkXDiqw8cvpOp/2PV5Adg
+06O/nVsJ8dWO41P0jmP6P6fbtGbfYmbW0W5BjfIttep3Sp+dWOIrWcBAI+0tKIJF
+PnlUkiaY4IBIqDfv8NZ5YBberOgOzW6sRBc4L0na4UU+Krk2U886UAb3LujEV0ls
+YSEY1QSteDwsOoBrp+uvFRTp2InBuThs4pFsiv9kuXclVzDAGySj4dzp30d8tbQk
+CAUw7C29C79Fv1C5qfPrmAESrciIxpg0X40KPMbp1ZWVbd4=
+-----END CERTIFICATE-----
+")EOF";

main/certs/default_server_cert.h

@@ -0,0 +1,5 @@
+const char* ss_server_cert PROGMEM = R"EOF("
+-----BEGIN CERTIFICATE-----
+...
+-----END CERTIFICATE-----
+")EOF";

main/main.ino

@@ -503,11 +503,7 @@ void pubMQTT(const char* topic, const char* payload, bool retainFlag) {
   if (client.connected()) {
     SendReceiveIndicatorON();
     Log.trace(F("[ OMG->MQTT ] topic: %s msg: %s " CR), topic, payload);
-#if AWS_IOT
-    client.publish(topic, payload); // AWS IOT doesn't support retain flag for the moment
-#else
     client.publish(topic, payload, retainFlag);
-#endif
   } else {
     Log.warning(F("Client not connected, aborting the publication" CR));
   }
@@ -2276,6 +2272,7 @@ String latestVersion;
 
 #    include "zzHTTPUpdate.h"
 
+#    ifdef CHECK_OTA_UPDATE
 /**
  * Check on a server the latest version information to build a releaseLink
  * The release link will be used when the user trigger an OTA update command
@@ -2317,13 +2314,17 @@ bool checkForUpdates() {
   }
   Log.notice(F("Update check done, free heap: %d"), ESP.getFreeHeap());
 }
+
+#    else
+bool checkForUpdates() {}
+#    endif
 #  elif ESP8266
 #    include <ESP8266httpUpdate.h>
 #  endif
 
 void MQTTHttpsFWUpdate(char* topicOri, JsonObject& HttpsFwUpdateData) {
   if (strstr(topicOri, subjectMQTTtoSYSupdate) != NULL) {
-    const char* version = HttpsFwUpdateData["version"];
+    const char* version = HttpsFwUpdateData["version"] | "latest";
     if (version && ((strlen(version) != strlen(OMG_VERSION)) || strcmp(version, OMG_VERSION) != 0)) {
       const char* url = HttpsFwUpdateData["url"];
       String systemUrl;
@@ -2374,18 +2375,19 @@ void MQTTHttpsFWUpdate(char* topicOri, JsonObject& HttpsFwUpdateData) {
       pub(subjectRLStoMQTT, jsondata);
 
       const char* ota_cert = HttpsFwUpdateData["server_cert"];
-      if (!ota_cert) {
+      if (!ota_cert && !strstr(url, "http:")) {
         if (ota_server_cert.length() > 0) {
-          Log.notice(F("using stored cert" CR));
+          Log.notice(F("Using stored cert" CR));
           ota_cert = ota_server_cert.c_str();
         } else {
-          Log.notice(F("using config cert" CR));
+          Log.notice(F("Using config cert" CR));
           ota_cert = OTAserver_cert;
         }
       }
 
       t_httpUpdate_return result = HTTP_UPDATE_FAILED;
       if (strstr(url, "http:")) {
+        Log.notice(F("Http update" CR));
         WiFiClient update_client;
 #  ifdef ESP32
         httpUpdate.setFollowRedirects(HTTPC_STRICT_FOLLOW_REDIRECTS);