From d7d5d0879f864f028c283444abe82af3394afeb9 Mon Sep 17 00:00:00 2001 From: Florian Date: Wed, 10 Jan 2024 18:54:42 -0600 Subject: [PATCH] [CI] Web upload factorizing and ESP Web Tools update (#1862) Update esp web tools Simplify scripts Add serial console --- .github/workflows/build.yml | 2 +- .github/workflows/build_and_docs_to_dev.yml | 2 +- docs/.vuepress/config.js | 2 +- docs/.vuepress/styles/index.styl | 11 + docs/upload/web-install.md | 5 + environments.ini | 8 +- scripts/common_wu.py | 114 ++++++++++ scripts/gen_wu.py | 225 +++++++++----------- scripts/gen_wu_dev.py | 208 ------------------ 9 files changed, 232 insertions(+), 345 deletions(-) create mode 100644 docs/.vuepress/styles/index.styl create mode 100644 scripts/common_wu.py delete mode 100644 scripts/gen_wu_dev.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index aefe960..cd4d7f9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -77,7 +77,7 @@ jobs: - "esp32s3-dev-c1-ble" - "esp32c3-dev-m1-ble" - "airm2m_core_esp32c3" - - "lolin_c3_mini" + - "esp32c3_lolin_mini" - "esp32c3-m5stamp" - "thingpulse-espgateway" - "theengs-bridge" diff --git a/.github/workflows/build_and_docs_to_dev.yml b/.github/workflows/build_and_docs_to_dev.yml index 7f527b4..42c71fe 100644 --- a/.github/workflows/build_and_docs_to_dev.yml +++ b/.github/workflows/build_and_docs_to_dev.yml @@ -49,7 +49,7 @@ jobs: curl -o docs/.vuepress/public/commonConfig.js https://www.theengs.io/commonConfig.js - name: Build documentation run: | - python ./scripts/gen_wu_dev.py + python ./scripts/gen_wu.py --dev npm run docs:build - name: Deploy to GitHub Pages uses: peaceiris/actions-gh-pages@v3 diff --git a/docs/.vuepress/config.js b/docs/.vuepress/config.js index e99ded0..7a1cf73 100644 --- a/docs/.vuepress/config.js +++ b/docs/.vuepress/config.js @@ -17,7 +17,7 @@ module.exports = { ['link', { rel: 'mask-icon', href: '/icons/safari-pinned-tab.svg', color: '#3eaf7c' }], ['meta', { name: 'msapplication-TileImage', content: '/favicon-144x144.png' }], ['meta', { name: 'msapplication-TileColor', content: '#000000' }], - ['script', {type: 'module', src: 'https://unpkg.com/esp-web-tools@3.5.0/dist/web/install-button.js?module'}] + ['script', {type: 'module', src: 'https://unpkg.com/esp-web-tools@9.4.3/dist/web/install-button.js?module'}] ], themeConfig: { repo: '1technophile/OpenMQTTGateway', diff --git a/docs/.vuepress/styles/index.styl b/docs/.vuepress/styles/index.styl new file mode 100644 index 0000000..408652e --- /dev/null +++ b/docs/.vuepress/styles/index.styl @@ -0,0 +1,11 @@ +// index.styl +div.table-generated-page + main + div.content__default, + footer.page-edit, + div.page-nav + // Display full with the API pages + max-width: 65% + +.sidebar + z-index 6 diff --git a/docs/upload/web-install.md b/docs/upload/web-install.md index 2345522..bc9d5b0 100644 --- a/docs/upload/web-install.md +++ b/docs/upload/web-install.md @@ -1,3 +1,7 @@ +--- +pageClass: table-generated-page +--- + # (Option 1) Upload from the web [![Hits](https://hits.seeyoufarm.com/api/count/incr/badge.svg?url=https%3A%2F%2Fdocs.openmqttgateway.com%2Fupload%2Fweb-install.html&count_bg=%2379C83D&title_bg=%23555555&icon=&icon_color=%23E7E7E7&title=hits&edge_flat=false)](https://hits.seeyoufarm.com) @@ -22,6 +26,7 @@ You can upload the firmware to your ESP device directly from here. +Upload powered by [ESP Web Tools](https://esphome.github.io/esp-web-tools/) ## Using OpenMQTTGateway ? Support open-source development through sponsorship and gain exclusive access to our private forum. Your questions, issues, and feature requests will receive priority attention, plus you'll gain insider access to our roadmap. diff --git a/environments.ini b/environments.ini index 22bc9ac..10246b1 100644 --- a/environments.ini +++ b/environments.ini @@ -1716,7 +1716,7 @@ build_flags = custom_description = BLE gateway on Espressiv ESP32-C3-DevKitC-02 without logging ; Wemos Lolin C3 mini v2.1.0 -[env:lolin_c3_mini] +[env:esp32c3_lolin_mini] platform = ${com.esp32_platform} board = lolin_c3_mini board_build.partitions = min_spiffs.csv @@ -1746,10 +1746,10 @@ build_flags = custom_description = BLE gateway on ESP32C3 custom_hardware = Wemos Lolin C3 mini v2.1.0 -[env:lolin_c3_mini_with_serial] -extends = env:lolin_c3_mini +[env:esp32c3_lolin_mini_with_serial] +extends = env:esp32c3_lolin_mini build_flags = - ${env:lolin_c3_mini.build_flags} + ${env:esp32c3_lolin_mini.build_flags} '-DLOG_LEVEL=LOG_LEVEL_NOTICE' ;default '-DWM_DEBUG_LEVEL=1' ;DEBUG_NOTIFY=1, default value ; Serial output via USB diff --git a/scripts/common_wu.py b/scripts/common_wu.py new file mode 100644 index 0000000..a580627 --- /dev/null +++ b/scripts/common_wu.py @@ -0,0 +1,114 @@ +import string + +mf_temp32 = string.Template('''{ + "name": "OpenMQTTGateway", + "new_install_prompt_erase": true, + "builds": [ + { + "chipFamily": "ESP32", + "improv": false, + "parts": [ + { "path": "$cp$bl", "offset": 4096 }, + { "path": "$cp$part", "offset": 32768 }, + { "path": "$cp$boot", "offset": 57344 }, + { "path": "$cp$bin", "offset": 65536 } + ] + } + ] +}''') + +mf_temp32c3 = string.Template('''{ + "name": "OpenMQTTGateway", + "new_install_prompt_erase": true, + "builds": [ + { + "chipFamily": "ESP32-C3", + "improv": false, + "parts": [ + { "path": "$cp$bl", "offset": 0 }, + { "path": "$cp$part", "offset": 32768 }, + { "path": "$cp$boot", "offset": 57344 }, + { "path": "$cp$bin", "offset": 65536 } + ] + } + ] +}''') + +mf_temp32s3 = string.Template('''{ + "name": "OpenMQTTGateway", + "new_install_prompt_erase": true, + "builds": [ + { + "chipFamily": "ESP32-S3", + "improv": false, + "parts": [ + { "path": "$cp$bl", "offset": 0 }, + { "path": "$cp$part", "offset": 32768 }, + { "path": "$cp$boot", "offset": 57344 }, + { "path": "$cp$bin", "offset": 65536 } + ] + } + ] +}''') + +mf_temp8266 = string.Template('''{ + "name": "OpenMQTTGateway", + "new_install_prompt_erase": true, + "builds": [ + { + "chipFamily": "ESP8266", + "parts": [{ "path": "$cp$bin", "offset": 0 }] + } + ] +}''') + +wu_temp_opt = string.Template(''' + + ''') + +wu_temp_p1 = ''' + +''' + +manif_path = 'docs/.vuepress/public/firmware_build/' +vue_path = 'docs/.vuepress/components/' +cors_proxy = '' # 'https://cors.bridged.cc/' +esp32_boot = 'https://github.com/espressif/arduino-esp32/raw/2.0.7/tools/partitions/boot_app0.bin' diff --git a/scripts/gen_wu.py b/scripts/gen_wu.py index a440a4f..50d1746 100644 --- a/scripts/gen_wu.py +++ b/scripts/gen_wu.py @@ -1,136 +1,85 @@ import os import requests import json -import string import argparse import shutil -mf_temp32 = string.Template('''{ - "name": "OpenMQTTGateway", - "builds": [ - { - "chipFamily": "ESP32", - "improv": false, - "parts": [ - { "path": "$cp$bl", "offset": 4096 }, - { "path": "$cp$part", "offset": 32768 }, - { "path": "$cp$boot", "offset": 57344 }, - { "path": "$cp$bin", "offset": 65536 } - ] - } - ] -}''') - -mf_temp8266 = string.Template('''{ - "name": "OpenMQTTGateway", - "builds": [ - { - "chipFamily": "ESP8266", - "parts": [{ "path": "$cp$bin", "offset": 0 }] - } - ] -}''') - -wu_temp_opt = string.Template(''' - - ''') - -wu_temp_p1 = ''' - -''' +from common_wu import mf_temp32, mf_temp32c3, mf_temp8266, wu_temp_opt, wu_temp_p1, wu_temp_p2, wu_temp_p3, wu_temp_p4, wu_temp_end, vue_path, manif_path, cors_proxy, esp32_boot, mf_temp32s3 parser = argparse.ArgumentParser() -parser.add_argument('repo') +parser.add_argument('--dev', action='store_true') +parser.add_argument('repo', nargs='?', default='1technophile/OpenMQTTGateway') args = parser.parse_args() repo = args.repo +dev = args.dev +bin_path = 'toDeploy/' manif_folder = "/firmware_build/" -manif_path = 'docs/.vuepress/public/firmware_build/' -vue_path = 'docs/.vuepress/components/' -cors_proxy = '' # 'https://cors.bridged.cc/' -esp32_blurl = 'https://github.com/espressif/arduino-esp32/raw/2.0.5/tools/sdk/esp32/bin/bootloader_dio_80m.bin' -esp32_boot = 'https://github.com/espressif/arduino-esp32/raw/2.0.5/tools/partitions/boot_app0.bin' -release = requests.get('https://api.github.com/repos/' + - repo + '/releases/latest') -rel_data = json.loads(release.text) - -if 'assets' in rel_data: - assets = rel_data['assets'] -else: - print('Assets not found') - os._exit(1) if not os.path.exists(manif_path): os.makedirs(manif_path) -# copy OTA latest version definition -shutil.move("scripts/latest_version.json", manif_path + "latest_version.json") +if dev: + print('Generate Web Upload in dev mode') + manif_folder = "/dev" + manif_folder + # copy OTA latest version definition + shutil.copy("scripts/latest_version_dev.json", + manif_path + "latest_version_dev.json") + # copy the content of bin_path to manif_path + for name in os.listdir(bin_path): + shutil.copyfile(bin_path + name, (manif_path + name)) +else: + print('Generate Web Upload in release mode') + # copy OTA latest version definition + shutil.copy("scripts/latest_version.json", manif_path + "latest_version.json") + release = requests.get('https://api.github.com/repos/' + + repo + '/releases/latest') + rel_data = json.loads(release.text) + if 'assets' in rel_data: + assets = rel_data['assets'] + # Download assets into manif_path + for item in range(len(assets)): + name = assets[item]['name'] + if 'firmware.bin' in name: + fw_bin = requests.get(assets[item]['browser_download_url']) + filename = assets[item]['browser_download_url'].split('/')[-1] + with open(manif_path + filename, 'wb') as output_file: + output_file.write(fw_bin.content) + print('Downloaded: ' + filename) + if 'partitions.bin' in name: + part_bin = requests.get(assets[item]['browser_download_url']) + filename = assets[item]['browser_download_url'].split('/')[-1] + with open(manif_path + filename, 'wb') as output_file: + output_file.write(part_bin.content) + print('Downloaded: ' + filename) + if 'bootloader.bin' in name: + bl_bin = requests.get(assets[item]['browser_download_url']) + filename = assets[item]['browser_download_url'].split('/')[-1] + with open(manif_path + filename, 'wb') as output_file: + output_file.write(bl_bin.content) + print('Downloaded: ' + filename) + else: + print('Assets not found') + os._exit(1) if not os.path.exists(vue_path): os.makedirs(vue_path) -wu_file = open(vue_path + 'web-uploader.vue', 'w') -wu_file.write(wu_temp_p1) - -bl_bin = requests.get(esp32_blurl) -filename = esp32_blurl.split('/')[-1] -with open(manif_path + filename, 'wb') as output_file: - output_file.write(bl_bin.content) - boot_bin = requests.get(esp32_boot) filename = esp32_boot.split('/')[-1] with open(manif_path + filename, 'wb') as output_file: output_file.write(boot_bin.content) -for item in range(len(assets)): - name = assets[item]['name'] - if 'firmware.bin' in name and ('esp32' in name or 'ttgo' in name or 'heltec' in name or 'thingpulse' in name or 'theengs' in name or 'lilygo' in name or 'shelly' in name or 'lolin_c3' in name or 'tinypico' in name): +wu_file = open(vue_path + 'web-uploader.vue', 'w') +wu_file.write(wu_temp_p1) + +for name in os.listdir(manif_path): + if 'firmware.bin' in name and ('esp32c3' not in name ) and ('esp32s3' not in name ) and ('esp32' in name or 'ttgo' in name or 'heltec' in name or 'thingpulse' in name or 'theengs' in name or 'lilygo' in name or 'shelly' in name or 'tinypico' in name): fw = name.split('-firmware')[0] man_file = fw + '.manifest.json' - fw_url = assets[item]['browser_download_url'] - fwp_url = fw_url.split('-firmware')[0] + '-partitions.bin' - mani_str = mf_temp32.substitute({'cp': cors_proxy, 'part': manif_folder + fwp_url.split('/')[-1], 'bin': manif_folder + fw_url.split( - '/')[-1], 'bl': manif_folder + esp32_blurl.split('/')[-1], 'boot': manif_folder + esp32_boot.split('/')[-1]}) + fwp_name = name.split('-firmware')[0] + '-partitions.bin' + fwb_name = name.split('-firmware')[0] + '-bootloader.bin' + mani_str = mf_temp32.substitute({'cp': cors_proxy, 'part': manif_folder + fwp_name.split('/')[-1], 'bin': manif_folder + name, 'bl': manif_folder + fwb_name, 'boot': manif_folder + esp32_boot.split('/')[-1]}) with open(manif_path + man_file, 'w') as nf: nf.write(mani_str) @@ -138,44 +87,60 @@ for item in range(len(assets)): wu_file.write(wu_temp_opt.substitute( {'mff': manif_folder + man_file, 'mfn': fw})) - fw_bin = requests.get(fw_url) - filename = fw_url.split('/')[-1] - with open(manif_path + filename, 'wb') as output_file: - output_file.write(fw_bin.content) - - part_bin = requests.get(fwp_url) - filename = fwp_url.split('/')[-1] - with open(manif_path + filename, 'wb') as output_file: - output_file.write(part_bin.content) - - print('Created: ' + os.path.abspath(man_file)) + print('Created: ' + man_file) wu_file.write(wu_temp_p2) -for item in range(len(assets)): - name = assets[item]['name'] +for name in os.listdir(manif_path): + if 'firmware.bin' in name and ('esp32c3' in name ): + fw = name.split('-firmware')[0] + man_file = fw + '.manifest.json' + fwp_name = name.split('-firmware')[0] + '-partitions.bin' + fwb_name = name.split('-firmware')[0] + '-bootloader.bin' + mani_str = mf_temp32c3.substitute({'cp': cors_proxy, 'part': manif_folder + fwp_name, 'bin': manif_folder + name, 'bl': manif_folder + fwb_name, 'boot': manif_folder + esp32_boot.split('/')[-1]}) + + with open(manif_path + man_file, 'w') as nf: + nf.write(mani_str) + + wu_file.write(wu_temp_opt.substitute( + {'mff': manif_folder + man_file, 'mfn': fw})) + + print('Created: ' + man_file) + +wu_file.write(wu_temp_p3) + +for name in os.listdir(manif_path): + if 'firmware.bin' in name and ('esp32s3' in name ): + fw = name.split('-firmware')[0] + man_file = fw + '.manifest.json' + fwp_name = name.split('-firmware')[0] + '-partitions.bin' + fwb_name = name.split('-firmware')[0] + '-bootloader.bin' + mani_str = mf_temp32s3.substitute({'cp': cors_proxy, 'part': manif_folder + fwp_name, 'bin': manif_folder + name, 'bl': manif_folder + fwb_name, 'boot': manif_folder + esp32_boot.split('/')[-1]}) + + with open(manif_path + man_file, 'w') as nf: + nf.write(mani_str) + + wu_file.write(wu_temp_opt.substitute( + {'mff': manif_folder + man_file, 'mfn': fw})) + + print('Created: ' + man_file) + +wu_file.write(wu_temp_p4) + +for name in os.listdir(manif_path): if 'firmware.bin' in name and ('nodemcu' in name or 'sonoff' in name or 'rf-wifi-gateway' in name or 'manual-wifi-test' in name or 'rfbridge' in name): fw = name.split('-firmware')[0] man_file = fw + '.manifest.json' - fw_url = assets[item]['browser_download_url'] mani_str = mf_temp8266.substitute( - {'cp': cors_proxy, 'bin': manif_folder + fw_url.split('/')[-1]}) + {'cp': cors_proxy, 'bin': manif_folder + name}) with open(manif_path + man_file, 'w') as nf: nf.write(mani_str) wu_file.write( manif_folder + wu_temp_opt.substitute({'mff': manif_folder + man_file, 'mfn': fw})) - fw_bin = requests.get(fw_url) - filename = fw_url.split('/')[-1] - with open(manif_path + filename, 'wb') as output_file: - output_file.write(fw_bin.content) - part_bin = requests.get(fwp_url) - filename = fwp_url.split('/')[-1] - with open(manif_path + filename, 'wb') as output_file: - output_file.write(part_bin.content) - print('Created: ' + os.path.abspath(man_file)) + print('Created: ' + man_file) wu_file.write(wu_temp_end) wu_file.close() diff --git a/scripts/gen_wu_dev.py b/scripts/gen_wu_dev.py deleted file mode 100644 index 2b8805f..0000000 --- a/scripts/gen_wu_dev.py +++ /dev/null @@ -1,208 +0,0 @@ -import os -import requests -import json -import string -import argparse -import shutil - -mf_temp32 = string.Template('''{ - "name": "OpenMQTTGateway", - "builds": [ - { - "chipFamily": "ESP32", - "improv": false, - "parts": [ - { "path": "$cp$bl", "offset": 4096 }, - { "path": "$cp$part", "offset": 32768 }, - { "path": "$cp$boot", "offset": 57344 }, - { "path": "$cp$bin", "offset": 65536 } - ] - } - ] -}''') - -mf_temp32c3 = string.Template('''{ - "name": "OpenMQTTGateway", - "builds": [ - { - "chipFamily": "ESP32-C3", - "improv": false, - "parts": [ - { "path": "$cp$bl", "offset": 0 }, - { "path": "$cp$part", "offset": 32768 }, - { "path": "$cp$boot", "offset": 57344 }, - { "path": "$cp$bin", "offset": 65536 } - ] - } - ] -}''') - -mf_temp8266 = string.Template('''{ - "name": "OpenMQTTGateway", - "builds": [ - { - "chipFamily": "ESP8266", - "parts": [{ "path": "$cp$bin", "offset": 0 }] - } - ] -}''') - -wu_temp_opt = string.Template(''' - - ''') - -wu_temp_p1 = '''