[OTA] Enable 2 clicks OTA update / one simple MQTT command (#1485)

* [OTA] Add automatic OTA updates

This feature will enable to update the gateway with one button or a simple MQTT command.

* [CI] Enable automatic OTA for nightly development builds

And integrate the latest_version file creation into the CI

Enable to specify a particular version
and switch between dev and production builds

scripts/gen_wu.py

@@ -3,6 +3,7 @@ import requests
 import json
 import string
 import argparse
+import shutil
 
 mf_temp32 = string.Template('''{
   "name": "OpenMQTTGateway",
@@ -86,10 +87,11 @@ repo = args.repo
 manif_folder = "/firmware_build/"
 manif_path = 'docs/.vuepress/public/firmware_build/'
 vue_path = 'docs/.vuepress/components/'
-cors_proxy = '' #'https://cors.bridged.cc/'
+cors_proxy = ''  # 'https://cors.bridged.cc/'
 esp32_blurl = 'https://github.com/espressif/arduino-esp32/raw/2.0.5/tools/sdk/esp32/bin/bootloader_dio_80m.bin'
-esp32_boot =  'https://github.com/espressif/arduino-esp32/raw/2.0.5/tools/partitions/boot_app0.bin'
-release = requests.get('https://api.github.com/repos/' + repo + '/releases/latest')
+esp32_boot = 'https://github.com/espressif/arduino-esp32/raw/2.0.5/tools/partitions/boot_app0.bin'
+release = requests.get('https://api.github.com/repos/' +
+                       repo + '/releases/latest')
 rel_data = json.loads(release.text)
 
 if 'assets' in rel_data:
@@ -101,6 +103,9 @@ else:
 if not os.path.exists(manif_path):
     os.makedirs(manif_path)
 
+# copy OTA latest version definition
+shutil.move("scripts/latest_version.json", manif_path + "latest_version.json")
+
 if not os.path.exists(vue_path):
     os.makedirs(vue_path)
 
@@ -109,12 +114,12 @@ wu_file.write(wu_temp_p1)
 
 bl_bin = requests.get(esp32_blurl)
 filename = esp32_blurl.split('/')[-1]
-with open(manif_path + filename,'wb') as output_file:
+with open(manif_path + filename, 'wb') as output_file:
     output_file.write(bl_bin.content)
 
 boot_bin = requests.get(esp32_boot)
 filename = esp32_boot.split('/')[-1]
-with open(manif_path + filename,'wb') as output_file:
+with open(manif_path + filename, 'wb') as output_file:
     output_file.write(boot_bin.content)
 
 for item in range(len(assets)):
@@ -124,21 +129,23 @@ for item in range(len(assets)):
         man_file = fw + '.manifest.json'
         fw_url = assets[item]['browser_download_url']
         fwp_url = fw_url.split('-firmware')[0] + '-partitions.bin'
-        mani_str = mf_temp32.substitute({'cp':cors_proxy, 'part':manif_folder + fwp_url.split('/')[-1], 'bin':manif_folder + fw_url.split('/')[-1], 'bl':manif_folder + esp32_blurl.split('/')[-1], 'boot':manif_folder + esp32_boot.split('/')[-1]})
+        mani_str = mf_temp32.substitute({'cp': cors_proxy, 'part': manif_folder + fwp_url.split('/')[-1], 'bin': manif_folder + fw_url.split(
+            '/')[-1], 'bl': manif_folder + esp32_blurl.split('/')[-1], 'boot': manif_folder + esp32_boot.split('/')[-1]})
 
         with open(manif_path + man_file, 'w') as nf:
             nf.write(mani_str)
 
-        wu_file.write(wu_temp_opt.substitute({'mff':manif_folder + man_file, 'mfn':fw}))
+        wu_file.write(wu_temp_opt.substitute(
+            {'mff': manif_folder + man_file, 'mfn': fw}))
 
         fw_bin = requests.get(fw_url)
         filename = fw_url.split('/')[-1]
-        with open(manif_path + filename,'wb') as output_file:
+        with open(manif_path + filename, 'wb') as output_file:
             output_file.write(fw_bin.content)
 
         part_bin = requests.get(fwp_url)
         filename = fwp_url.split('/')[-1]
-        with open(manif_path + filename,'wb') as output_file:
+        with open(manif_path + filename, 'wb') as output_file:
             output_file.write(part_bin.content)
 
         print('Created: ' + os.path.abspath(man_file))
@@ -151,20 +158,22 @@ for item in range(len(assets)):
         fw = name.split('-firmware')[0]
         man_file = fw + '.manifest.json'
         fw_url = assets[item]['browser_download_url']
-        mani_str = mf_temp8266.substitute({'cp':cors_proxy, 'bin':manif_folder + fw_url.split('/')[-1]})
+        mani_str = mf_temp8266.substitute(
+            {'cp': cors_proxy, 'bin': manif_folder + fw_url.split('/')[-1]})
 
         with open(manif_path + man_file, 'w') as nf:
             nf.write(mani_str)
 
-        wu_file.write(manif_folder + wu_temp_opt.substitute({'mff':manif_folder + man_file, 'mfn':fw}))
+        wu_file.write(
+            manif_folder + wu_temp_opt.substitute({'mff': manif_folder + man_file, 'mfn': fw}))
         fw_bin = requests.get(fw_url)
         filename = fw_url.split('/')[-1]
-        with open(manif_path + filename,'wb') as output_file:
+        with open(manif_path + filename, 'wb') as output_file:
             output_file.write(fw_bin.content)
 
         part_bin = requests.get(fwp_url)
         filename = fwp_url.split('/')[-1]
-        with open(manif_path + filename,'wb') as output_file:
+        with open(manif_path + filename, 'wb') as output_file:
             output_file.write(part_bin.content)
         print('Created: ' + os.path.abspath(man_file))
 

scripts/gen_wu_dev.py

@@ -83,13 +83,17 @@ manif_folder = "/dev/firmware_build/"
 manif_path = 'docs/.vuepress/public/firmware_build/'
 vue_path = 'docs/.vuepress/components/'
 bin_path = 'toDeploy/'
-cors_proxy = '' #'https://cors.bridged.cc/'
+cors_proxy = ''  # 'https://cors.bridged.cc/'
 esp32_blurl = 'https://github.com/espressif/arduino-esp32/raw/2.0.5/tools/sdk/esp32/bin/bootloader_dio_80m.bin'
-esp32_boot =  'https://github.com/espressif/arduino-esp32/raw/2.0.5/tools/partitions/boot_app0.bin'
+esp32_boot = 'https://github.com/espressif/arduino-esp32/raw/2.0.5/tools/partitions/boot_app0.bin'
 
 if not os.path.exists(manif_path):
     os.makedirs(manif_path)
 
+# copy OTA latest version definition
+shutil.move("scripts/latest_version_dev.json",
+            manif_path + "latest_version_dev.json")
+
 if not os.path.exists(vue_path):
     os.makedirs(vue_path)
 
@@ -98,12 +102,12 @@ wu_file.write(wu_temp_p1)
 
 bl_bin = requests.get(esp32_blurl)
 filename = esp32_blurl.split('/')[-1]
-with open(manif_path + filename,'wb') as output_file:
+with open(manif_path + filename, 'wb') as output_file:
     output_file.write(bl_bin.content)
 
 boot_bin = requests.get(esp32_boot)
 filename = esp32_boot.split('/')[-1]
-with open(manif_path + filename,'wb') as output_file:
+with open(manif_path + filename, 'wb') as output_file:
     output_file.write(boot_bin.content)
 
 for name in os.listdir(bin_path):
@@ -113,12 +117,14 @@ for name in os.listdir(bin_path):
         print('Bin name:' + name)
         part_name = name.split('-firmware')[0] + '-partitions.bin'
         print('Partition name:' + part_name)
-        mani_str = mf_temp32.substitute({'cp':cors_proxy, 'part':manif_folder + part_name.split('/')[-1], 'bin':manif_folder + name.split('/')[-1], 'bl':manif_folder + esp32_blurl.split('/')[-1], 'boot':manif_folder + esp32_boot.split('/')[-1]})
+        mani_str = mf_temp32.substitute({'cp': cors_proxy, 'part': manif_folder + part_name.split('/')[-1], 'bin': manif_folder + name.split(
+            '/')[-1], 'bl': manif_folder + esp32_blurl.split('/')[-1], 'boot': manif_folder + esp32_boot.split('/')[-1]})
 
         with open(manif_path + man_file, 'w') as nf:
             nf.write(mani_str)
 
-        wu_file.write(wu_temp_opt.substitute({'mff':manif_folder + man_file, 'mfn':fw}))
+        wu_file.write(wu_temp_opt.substitute(
+            {'mff': manif_folder + man_file, 'mfn': fw}))
 
         shutil.copyfile(bin_path + name, (manif_path + name))
         shutil.copyfile(bin_path + part_name, (manif_path + part_name))
@@ -134,12 +140,14 @@ for name in os.listdir(bin_path):
         print('Bin name:' + name)
         part_name = name.split('-firmware')[0] + '-partitions.bin'
         print('Partition name:' + part_name)
-        mani_str = mf_temp8266.substitute({'cp':cors_proxy, 'bin':manif_folder + name.split('/')[-1]})
+        mani_str = mf_temp8266.substitute(
+            {'cp': cors_proxy, 'bin': manif_folder + name.split('/')[-1]})
 
         with open(manif_path + man_file, 'w') as nf:
             nf.write(mani_str)
 
-        wu_file.write(manif_folder + wu_temp_opt.substitute({'mff':manif_folder + man_file, 'mfn':fw}))
+        wu_file.write(
+            manif_folder + wu_temp_opt.substitute({'mff': manif_folder + man_file, 'mfn': fw}))
 
         shutil.copyfile(bin_path + name, (manif_path + name))
 

scripts/latest_version.json

@@ -0,0 +1,5 @@
+{
+    "latest_version": "version_tag",
+    "title": "OpenMQTTGateway version_tag",
+    "release_url": "https://github.com/1technophile/OpenMQTTGateway/releases/tag/version_tag",
+}

scripts/latest_version_dev.json

@@ -0,0 +1,4 @@
+{
+    "latest_version": "version_tag",
+    "title": "OpenMQTTGateway Nightly (TEST ONLY) version_tag",
+}

scripts/prepare_deploy.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 set -e
-echo "renaming bin files with the board name"
+echo "renaming bin files with the environment name"
 rename -v 's:/:-:g' .pio/build/*/*.bin
 mkdir toDeploy
 rename 's/.pio-build-//' .*.bin