Add uploading from documentation option. (#1048)

This allows for directly uploading OMG to ESP32 and ESp8266 devices from the upload section of the documentation.

This will upload the latest release binary for the device selected in the dropdown menu on the webpage.

A script has been added to generate the upload page when a new release is issued.

.github/workflows/manual_docs.yml

@@ -0,0 +1,34 @@
+name: Create and publish documentation
+on:
+  workflow_dispatch:
+jobs:
+  documentation:
+    runs-on: ubuntu-latest
+    name: Create the documentation and deploy it to GitHub Pages
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Node.js
+        uses: actions/setup-node@v1
+        with:
+          node-version: "14.x"
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.7"
+      - name: Install build dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install requests
+          npm install
+      - name: Set version tag from git
+        run: sed -i "s/version_tag/${GITHUB_REF#refs/tags/}/g" docs/.vuepress/config.js
+      - name: Build documentation
+        run: |
+          python ./scripts/gen_wu.py ${GITHUB_REPOSITORY}
+          npm run docs:build
+      - name: Deploy to GitHub Pages
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./docs/.vuepress/dist
+          cname: docs.openmqttgateway.com

.github/workflows/release.yml

@@ -40,6 +40,7 @@ jobs:
           allow-overwrite: true
 
   documentation:
+    needs: build-upload
     runs-on: ubuntu-latest
     name: Create the documentation and deploy it to GitHub Pages
     steps:
@@ -48,12 +49,21 @@ jobs:
         uses: actions/setup-node@v1
         with:
           node-version: "14.x"
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.7"
       - name: Install build dependencies
-        run: npm install
+        run: |
+          python -m pip install --upgrade pip
+          pip install requests
+          npm install
       - name: Set version tag from git
         run: sed -i "s/version_tag/${GITHUB_REF#refs/tags/}/g" docs/.vuepress/config.js
       - name: Build documentation
-        run: npm run docs:build
+        run: |
+          python ./scripts/gen_wu.py ${GITHUB_REPOSITORY}
+          npm run docs:build
       - name: Deploy to GitHub Pages
         uses: peaceiris/actions-gh-pages@v3
         with:

docs/.vuepress/config.js

@@ -15,7 +15,8 @@ module.exports = {
       ['link', { rel: "apple-touch-icon", sizes: "180x180", href: "/img/apple-touch-icon.png"}],
       ['link', { rel: 'mask-icon', href: '/icons/safari-pinned-tab.svg', color: '#3eaf7c' }],
       ['meta', { name: 'msapplication-TileImage', content: '/icons/msapplication-icon-144x144.png' }],
-      ['meta', { name: 'msapplication-TileColor', content: '#000000' }]
+      ['meta', { name: 'msapplication-TileColor', content: '#000000' }],
+      ['script', {type: 'module', src: 'https://unpkg.com/esp-web-tools@3.4.2/dist/web/install-button.js?module'}]
     ],
     themeConfig: {
       smoothScroll: true,
@@ -62,6 +63,7 @@ module.exports = {
           title: '3 - Upload ➡️',   // required
           sidebarDepth: 1,    // optional, defaults to 1
           children: [
+            'upload/web-install',
             'upload/binaries',
             'upload/builds',
             'upload/portal',

docs/upload/binaries.md

@@ -1,4 +1,4 @@
-# (Option 1) Upload ready-to-go binaries
+# (Option 2) Upload ready-to-go binaries
 This section is useful if you want to directly flash your ESP from your desktop. Once flashed you can change  wifi and broker settings.
 Nevertheless you will not be able to change advanced parameters; if you want to do so, refer to [Upload your configurations](builds.md) section.
 

docs/upload/builds.md

@@ -1,4 +1,4 @@
-# (Option 2) Upload your configurations
+# (Option 3) Upload your configurations
 
 ## Introduction
 

docs/upload/web-install.md

@@ -0,0 +1,8 @@
+# (Option 1) Upload from the web
+You can upload the firmware to your ESP device directly from here.
+1. Plug in your ESP to a USB port.
+2. Select the firmware in the box below.
+3. Click the install button and choose the port that the ESP is connected to.
+4. Wait until the process is complete.
+
+<web-uploader/>

scripts/gen_wu.py

@@ -0,0 +1,172 @@
+import os
+import requests
+import json
+import string
+import argparse
+
+mf_temp32 = string.Template('''{
+  "name": "OpenMQTTGateway",
+  "builds": [
+    {
+      "chipFamily": "ESP32",
+      "improv": false,
+      "parts": [
+        { "path": "$cp$bl", "offset": 4096 },
+        { "path": "$cp$part", "offset": 32768 },
+        { "path": "$cp$boot", "offset": 57344 },
+        { "path": "$cp$bin", "offset": 65536 }
+      ]
+    }
+  ]
+}''')
+
+mf_temp8266 = string.Template('''{
+  "name": "OpenMQTTGateway",
+  "builds": [
+    {
+      "chipFamily": "ESP8266",
+      "parts": [{ "path": "$cp$bin", "offset": 0 }]
+    }
+  ]
+}''')
+
+wu_temp_opt = string.Template('''
+        <option
+          value="$mff"
+        >
+        $mfn
+        </option>
+        ''')
+
+wu_temp_p1 = '''<template>
+  <div align="center">
+    <select>
+      <optgroup label="ESP32">'''
+
+wu_temp_p2 = '''
+      </optgroup>
+      <optgroup label="ESP8266">'''
+
+wu_temp_end = '''
+      </optgroup>
+    </select><br><br>
+    <input type="checkbox" id="erase" checked>
+    <label for="erase"> Erase Flash (erases all saved data)</label><br><br>
+    <esp-web-install-button erase-first></esp-web-install-button>
+  </div>
+</template>
+
+<script>
+export default {
+  mounted () {
+    const espWebInstallButton = document.querySelector("esp-web-install-button");
+    const eraseCheck = document.getElementById("erase");
+    espWebInstallButton.addEventListener("state-changed", (ev) => { console.log(ev.detail) });
+    const selectFW = document.querySelector("select");
+    espWebInstallButton.manifest = selectFW.value;
+    selectFW.addEventListener("change", () => {
+      espWebInstallButton.manifest = selectFW.value;
+    });
+    eraseCheck.addEventListener("change", (ev) => {
+      if (eraseCheck.checked) {
+        espWebInstallButton.setAttribute('erase-first','');
+      } else {
+        espWebInstallButton.removeAttribute('erase-first');
+      }
+    });
+  }
+}
+</script>'''
+
+parser = argparse.ArgumentParser()
+parser.add_argument('repo')
+args = parser.parse_args()
+repo = args.repo
+
+manif_folder = "/firmware_build/"
+manif_path = 'docs/.vuepress/public/firmware_build/'
+vue_path = 'docs/.vuepress/components/'
+cors_proxy = '' #'https://cors.bridged.cc/'
+esp32_blurl = 'https://github.com/espressif/arduino-esp32/raw/master/tools/sdk/esp32/bin/bootloader_dio_80m.bin'
+esp32_boot =  'https://github.com/espressif/arduino-esp32/raw/master/tools/partitions/boot_app0.bin'
+release = requests.get('https://api.github.com/repos/' + repo + '/releases/latest')
+rel_data = json.loads(release.text)
+
+if 'assets' in rel_data:
+    assets = rel_data['assets']
+else:
+    print('Assets not found')
+    os._exit(1)
+
+if not os.path.exists(manif_path):
+    os.makedirs(manif_path)
+
+if not os.path.exists(vue_path):
+    os.makedirs(vue_path)
+
+wu_file = open(vue_path + 'web-uploader.vue', 'w')
+wu_file.write(wu_temp_p1)
+
+bl_bin = requests.get(esp32_blurl)
+filename = esp32_blurl.split('/')[-1]
+with open(manif_path + filename,'wb') as output_file:
+    output_file.write(bl_bin.content)
+
+boot_bin = requests.get(esp32_boot)
+filename = esp32_boot.split('/')[-1]
+with open(manif_path + filename,'wb') as output_file:
+    output_file.write(boot_bin.content)
+
+for item in range(len(assets)):
+    name = assets[item]['name']
+    if 'firmware.bin' in name and ('esp32' in name or 'ttgo' in name or 'heltec' in name):
+        fw = name.split('-firmware')[0]
+        man_file = fw + '.manifest.json'
+        fw_url = assets[item]['browser_download_url']
+        fwp_url = fw_url.split('-firmware')[0] + '-partitions.bin'
+        mani_str = mf_temp32.substitute({'cp':cors_proxy, 'part':manif_folder + fwp_url.split('/')[-1], 'bin':manif_folder + fw_url.split('/')[-1], 'bl':manif_folder + esp32_blurl.split('/')[-1], 'boot':manif_folder + esp32_boot.split('/')[-1]})
+
+        with open(manif_path + man_file, 'w') as nf:
+            nf.write(mani_str)
+
+        wu_file.write(wu_temp_opt.substitute({'mff':manif_folder + man_file, 'mfn':fw}))
+
+        fw_bin = requests.get(fw_url)
+        filename = fw_url.split('/')[-1]
+        with open(manif_path + filename,'wb') as output_file:
+            output_file.write(fw_bin.content)
+
+        part_bin = requests.get(fwp_url)
+        filename = fwp_url.split('/')[-1]
+        with open(manif_path + filename,'wb') as output_file:
+            output_file.write(part_bin.content)
+
+        print('Created: ' + os.path.abspath(man_file))
+
+wu_file.write(wu_temp_p2)
+
+for item in range(len(assets)):
+    name = assets[item]['name']
+    if 'firmware.bin' in name and ('nodemcu' in name or 'sonoff' in name or 'rf-wifi-gateway' in name or 'manual-wifi-test' in name or 'rfbridge' in name):
+        fw = name.split('-firmware')[0]
+        man_file = fw + '.manifest.json'
+        fw_url = assets[item]['browser_download_url']
+        mani_str = mf_temp8266.substitute({'cp':cors_proxy, 'bin':manif_folder + fw_url.split('/')[-1]})
+
+        with open(manif_path + man_file, 'w') as nf:
+            nf.write(mani_str)
+
+        wu_file.write(manif_folder + wu_temp_opt.substitute({'mff':manif_folder + man_file, 'mfn':fw}))
+        fw_bin = requests.get(fw_url)
+        filename = fw_url.split('/')[-1]
+        with open(manif_path + filename,'wb') as output_file:
+            output_file.write(fw_bin.content)
+
+        part_bin = requests.get(fwp_url)
+        filename = fwp_url.split('/')[-1]
+        with open(manif_path + filename,'wb') as output_file:
+            output_file.write(part_bin.content)
+        print('Created: ' + os.path.abspath(man_file))
+
+wu_file.write(wu_temp_end)
+wu_file.close()