[CI] Web upload factorizing and ESP Web Tools update (#1862)

Update esp web tools
Simplify scripts
Add serial console
This commit is contained in:
Florian authored and GitHub committed 2024-01-10 18:54:42 -06:00
1 parent 8fa47d2bde
commit d7d5d0879f
9 files changed
+232 -345

No files matched your search

+114
View File
@@ -0,0 +1,114 @@
import string
mf_temp32 = string.Template('''{
"name": "OpenMQTTGateway",
"new_install_prompt_erase": true,
"builds": [
{
"chipFamily": "ESP32",
"improv": false,
"parts": [
{ "path": "$cp$bl", "offset": 4096 },
{ "path": "$cp$part", "offset": 32768 },
{ "path": "$cp$boot", "offset": 57344 },
{ "path": "$cp$bin", "offset": 65536 }
]
}
]
}''')
mf_temp32c3 = string.Template('''{
"name": "OpenMQTTGateway",
"new_install_prompt_erase": true,
"builds": [
{
"chipFamily": "ESP32-C3",
"improv": false,
"parts": [
{ "path": "$cp$bl", "offset": 0 },
{ "path": "$cp$part", "offset": 32768 },
{ "path": "$cp$boot", "offset": 57344 },
{ "path": "$cp$bin", "offset": 65536 }
]
}
]
}''')
mf_temp32s3 = string.Template('''{
"name": "OpenMQTTGateway",
"new_install_prompt_erase": true,
"builds": [
{
"chipFamily": "ESP32-S3",
"improv": false,
"parts": [
{ "path": "$cp$bl", "offset": 0 },
{ "path": "$cp$part", "offset": 32768 },
{ "path": "$cp$boot", "offset": 57344 },
{ "path": "$cp$bin", "offset": 65536 }
]
}
]
}''')
mf_temp8266 = string.Template('''{
"name": "OpenMQTTGateway",
"new_install_prompt_erase": true,
"builds": [
{
"chipFamily": "ESP8266",
"parts": [{ "path": "$cp$bin", "offset": 0 }]
}
]
}''')
wu_temp_opt = string.Template('''
<option
value="$mff"
>
$mfn
</option>
''')
wu_temp_p1 = '''<template>
<div align="center">
<select>
<optgroup label="ESP32">'''
wu_temp_p2 = '''
</optgroup>
<optgroup label="ESP32C3">'''
wu_temp_p3 = '''
</optgroup>
<optgroup label="ESP32S3">'''
wu_temp_p4 = '''
</optgroup>
<optgroup label="ESP8266">'''
wu_temp_end = '''
</optgroup>
</select><br><br>
<esp-web-install-button erase-first></esp-web-install-button>
</div>
</template>
<script>
export default {
mounted () {
const espWebInstallButton = document.querySelector("esp-web-install-button");
espWebInstallButton.addEventListener("state-changed", (ev) => { console.log(ev.detail) });
const selectFW = document.querySelector("select");
espWebInstallButton.manifest = selectFW.value;
selectFW.addEventListener("change", () => {
espWebInstallButton.manifest = selectFW.value;
});
}
}
</script>'''
manif_path = 'docs/.vuepress/public/firmware_build/'
vue_path = 'docs/.vuepress/components/'
cors_proxy = '' # 'https://cors.bridged.cc/'
esp32_boot = 'https://github.com/espressif/arduino-esp32/raw/2.0.7/tools/partitions/boot_app0.bin'
+95 -130
View File
@@ -1,136 +1,85 @@
import os
import requests
import json
import string
import argparse
import shutil
mf_temp32 = string.Template('''{
"name": "OpenMQTTGateway",
"builds": [
{
"chipFamily": "ESP32",
"improv": false,
"parts": [
{ "path": "$cp$bl", "offset": 4096 },
{ "path": "$cp$part", "offset": 32768 },
{ "path": "$cp$boot", "offset": 57344 },
{ "path": "$cp$bin", "offset": 65536 }
]
}
]
}''')
mf_temp8266 = string.Template('''{
"name": "OpenMQTTGateway",
"builds": [
{
"chipFamily": "ESP8266",
"parts": [{ "path": "$cp$bin", "offset": 0 }]
}
]
}''')
wu_temp_opt = string.Template('''
<option
value="$mff"
>
$mfn
</option>
''')
wu_temp_p1 = '''<template>
<div align="center">
<select>
<optgroup label="ESP32">'''
wu_temp_p2 = '''
</optgroup>
<optgroup label="ESP8266">'''
wu_temp_end = '''
</optgroup>
</select><br><br>
<input type="checkbox" id="erase" checked>
<label for="erase"> Erase Flash (erases all saved data)</label><br><br>
<esp-web-install-button erase-first></esp-web-install-button>
</div>
</template>
<script>
export default {
mounted () {
const espWebInstallButton = document.querySelector("esp-web-install-button");
const eraseCheck = document.getElementById("erase");
espWebInstallButton.addEventListener("state-changed", (ev) => { console.log(ev.detail) });
const selectFW = document.querySelector("select");
espWebInstallButton.manifest = selectFW.value;
selectFW.addEventListener("change", () => {
espWebInstallButton.manifest = selectFW.value;
});
eraseCheck.addEventListener("change", (ev) => {
if (eraseCheck.checked) {
espWebInstallButton.setAttribute('erase-first','');
} else {
espWebInstallButton.removeAttribute('erase-first');
}
});
}
}
</script>'''
from common_wu import mf_temp32, mf_temp32c3, mf_temp8266, wu_temp_opt, wu_temp_p1, wu_temp_p2, wu_temp_p3, wu_temp_p4, wu_temp_end, vue_path, manif_path, cors_proxy, esp32_boot, mf_temp32s3
parser = argparse.ArgumentParser()
parser.add_argument('repo')
parser.add_argument('--dev', action='store_true')
parser.add_argument('repo', nargs='?', default='1technophile/OpenMQTTGateway')
args = parser.parse_args()
repo = args.repo
dev = args.dev
bin_path = 'toDeploy/'
manif_folder = "/firmware_build/"
manif_path = 'docs/.vuepress/public/firmware_build/'
vue_path = 'docs/.vuepress/components/'
cors_proxy = '' # 'https://cors.bridged.cc/'
esp32_blurl = 'https://github.com/espressif/arduino-esp32/raw/2.0.5/tools/sdk/esp32/bin/bootloader_dio_80m.bin'
esp32_boot = 'https://github.com/espressif/arduino-esp32/raw/2.0.5/tools/partitions/boot_app0.bin'
release = requests.get('https://api.github.com/repos/' +
repo + '/releases/latest')
rel_data = json.loads(release.text)
if 'assets' in rel_data:
assets = rel_data['assets']
else:
print('Assets not found')
os._exit(1)
if not os.path.exists(manif_path):
os.makedirs(manif_path)
# copy OTA latest version definition
shutil.move("scripts/latest_version.json", manif_path + "latest_version.json")
if dev:
print('Generate Web Upload in dev mode')
manif_folder = "/dev" + manif_folder
# copy OTA latest version definition
shutil.copy("scripts/latest_version_dev.json",
manif_path + "latest_version_dev.json")
# copy the content of bin_path to manif_path
for name in os.listdir(bin_path):
shutil.copyfile(bin_path + name, (manif_path + name))
else:
print('Generate Web Upload in release mode')
# copy OTA latest version definition
shutil.copy("scripts/latest_version.json", manif_path + "latest_version.json")
release = requests.get('https://api.github.com/repos/' +
repo + '/releases/latest')
rel_data = json.loads(release.text)
if 'assets' in rel_data:
assets = rel_data['assets']
# Download assets into manif_path
for item in range(len(assets)):
name = assets[item]['name']
if 'firmware.bin' in name:
fw_bin = requests.get(assets[item]['browser_download_url'])
filename = assets[item]['browser_download_url'].split('/')[-1]
with open(manif_path + filename, 'wb') as output_file:
output_file.write(fw_bin.content)
print('Downloaded: ' + filename)
if 'partitions.bin' in name:
part_bin = requests.get(assets[item]['browser_download_url'])
filename = assets[item]['browser_download_url'].split('/')[-1]
with open(manif_path + filename, 'wb') as output_file:
output_file.write(part_bin.content)
print('Downloaded: ' + filename)
if 'bootloader.bin' in name:
bl_bin = requests.get(assets[item]['browser_download_url'])
filename = assets[item]['browser_download_url'].split('/')[-1]
with open(manif_path + filename, 'wb') as output_file:
output_file.write(bl_bin.content)
print('Downloaded: ' + filename)
else:
print('Assets not found')
os._exit(1)
if not os.path.exists(vue_path):
os.makedirs(vue_path)
wu_file = open(vue_path + 'web-uploader.vue', 'w')
wu_file.write(wu_temp_p1)
bl_bin = requests.get(esp32_blurl)
filename = esp32_blurl.split('/')[-1]
with open(manif_path + filename, 'wb') as output_file:
output_file.write(bl_bin.content)
boot_bin = requests.get(esp32_boot)
filename = esp32_boot.split('/')[-1]
with open(manif_path + filename, 'wb') as output_file:
output_file.write(boot_bin.content)
for item in range(len(assets)):
name = assets[item]['name']
if 'firmware.bin' in name and ('esp32' in name or 'ttgo' in name or 'heltec' in name or 'thingpulse' in name or 'theengs' in name or 'lilygo' in name or 'shelly' in name or 'lolin_c3' in name or 'tinypico' in name):
wu_file = open(vue_path + 'web-uploader.vue', 'w')
wu_file.write(wu_temp_p1)
for name in os.listdir(manif_path):
if 'firmware.bin' in name and ('esp32c3' not in name ) and ('esp32s3' not in name ) and ('esp32' in name or 'ttgo' in name or 'heltec' in name or 'thingpulse' in name or 'theengs' in name or 'lilygo' in name or 'shelly' in name or 'tinypico' in name):
fw = name.split('-firmware')[0]
man_file = fw + '.manifest.json'
fw_url = assets[item]['browser_download_url']
fwp_url = fw_url.split('-firmware')[0] + '-partitions.bin'
mani_str = mf_temp32.substitute({'cp': cors_proxy, 'part': manif_folder + fwp_url.split('/')[-1], 'bin': manif_folder + fw_url.split(
'/')[-1], 'bl': manif_folder + esp32_blurl.split('/')[-1], 'boot': manif_folder + esp32_boot.split('/')[-1]})
fwp_name = name.split('-firmware')[0] + '-partitions.bin'
fwb_name = name.split('-firmware')[0] + '-bootloader.bin'
mani_str = mf_temp32.substitute({'cp': cors_proxy, 'part': manif_folder + fwp_name.split('/')[-1], 'bin': manif_folder + name, 'bl': manif_folder + fwb_name, 'boot': manif_folder + esp32_boot.split('/')[-1]})
with open(manif_path + man_file, 'w') as nf:
nf.write(mani_str)
@@ -138,44 +87,60 @@ for item in range(len(assets)):
wu_file.write(wu_temp_opt.substitute(
{'mff': manif_folder + man_file, 'mfn': fw}))
fw_bin = requests.get(fw_url)
filename = fw_url.split('/')[-1]
with open(manif_path + filename, 'wb') as output_file:
output_file.write(fw_bin.content)
part_bin = requests.get(fwp_url)
filename = fwp_url.split('/')[-1]
with open(manif_path + filename, 'wb') as output_file:
output_file.write(part_bin.content)
print('Created: ' + os.path.abspath(man_file))
print('Created: ' + man_file)
wu_file.write(wu_temp_p2)
for item in range(len(assets)):
name = assets[item]['name']
for name in os.listdir(manif_path):
if 'firmware.bin' in name and ('esp32c3' in name ):
fw = name.split('-firmware')[0]
man_file = fw + '.manifest.json'
fwp_name = name.split('-firmware')[0] + '-partitions.bin'
fwb_name = name.split('-firmware')[0] + '-bootloader.bin'
mani_str = mf_temp32c3.substitute({'cp': cors_proxy, 'part': manif_folder + fwp_name, 'bin': manif_folder + name, 'bl': manif_folder + fwb_name, 'boot': manif_folder + esp32_boot.split('/')[-1]})
with open(manif_path + man_file, 'w') as nf:
nf.write(mani_str)
wu_file.write(wu_temp_opt.substitute(
{'mff': manif_folder + man_file, 'mfn': fw}))
print('Created: ' + man_file)
wu_file.write(wu_temp_p3)
for name in os.listdir(manif_path):
if 'firmware.bin' in name and ('esp32s3' in name ):
fw = name.split('-firmware')[0]
man_file = fw + '.manifest.json'
fwp_name = name.split('-firmware')[0] + '-partitions.bin'
fwb_name = name.split('-firmware')[0] + '-bootloader.bin'
mani_str = mf_temp32s3.substitute({'cp': cors_proxy, 'part': manif_folder + fwp_name, 'bin': manif_folder + name, 'bl': manif_folder + fwb_name, 'boot': manif_folder + esp32_boot.split('/')[-1]})
with open(manif_path + man_file, 'w') as nf:
nf.write(mani_str)
wu_file.write(wu_temp_opt.substitute(
{'mff': manif_folder + man_file, 'mfn': fw}))
print('Created: ' + man_file)
wu_file.write(wu_temp_p4)
for name in os.listdir(manif_path):
if 'firmware.bin' in name and ('nodemcu' in name or 'sonoff' in name or 'rf-wifi-gateway' in name or 'manual-wifi-test' in name or 'rfbridge' in name):
fw = name.split('-firmware')[0]
man_file = fw + '.manifest.json'
fw_url = assets[item]['browser_download_url']
mani_str = mf_temp8266.substitute(
{'cp': cors_proxy, 'bin': manif_folder + fw_url.split('/')[-1]})
{'cp': cors_proxy, 'bin': manif_folder + name})
with open(manif_path + man_file, 'w') as nf:
nf.write(mani_str)
wu_file.write(
manif_folder + wu_temp_opt.substitute({'mff': manif_folder + man_file, 'mfn': fw}))
fw_bin = requests.get(fw_url)
filename = fw_url.split('/')[-1]
with open(manif_path + filename, 'wb') as output_file:
output_file.write(fw_bin.content)
part_bin = requests.get(fwp_url)
filename = fwp_url.split('/')[-1]
with open(manif_path + filename, 'wb') as output_file:
output_file.write(part_bin.content)
print('Created: ' + os.path.abspath(man_file))
print('Created: ' + man_file)
wu_file.write(wu_temp_end)
wu_file.close()
-208
View File
@@ -1,208 +0,0 @@
import os
import requests
import json
import string
import argparse
import shutil
mf_temp32 = string.Template('''{
"name": "OpenMQTTGateway",
"builds": [
{
"chipFamily": "ESP32",
"improv": false,
"parts": [
{ "path": "$cp$bl", "offset": 4096 },
{ "path": "$cp$part", "offset": 32768 },
{ "path": "$cp$boot", "offset": 57344 },
{ "path": "$cp$bin", "offset": 65536 }
]
}
]
}''')
mf_temp32c3 = string.Template('''{
"name": "OpenMQTTGateway",
"builds": [
{
"chipFamily": "ESP32-C3",
"improv": false,
"parts": [
{ "path": "$cp$bl", "offset": 0 },
{ "path": "$cp$part", "offset": 32768 },
{ "path": "$cp$boot", "offset": 57344 },
{ "path": "$cp$bin", "offset": 65536 }
]
}
]
}''')
mf_temp8266 = string.Template('''{
"name": "OpenMQTTGateway",
"builds": [
{
"chipFamily": "ESP8266",
"parts": [{ "path": "$cp$bin", "offset": 0 }]
}
]
}''')
wu_temp_opt = string.Template('''
<option
value="$mff"
>
$mfn
</option>
''')
wu_temp_p1 = '''<template>
<div align="center">
<select>
<optgroup label="ESP32">'''
wu_temp_p2 = '''<template>
</optgroup>
<optgroup label="ESP32C3">'''
wu_temp_p3 = '''
</optgroup>
<optgroup label="ESP8266">'''
wu_temp_end = '''
</optgroup>
</select><br><br>
<input type="checkbox" id="erase" checked>
<label for="erase"> Erase Flash (erases all saved data)</label><br><br>
<esp-web-install-button erase-first></esp-web-install-button>
</div>
</template>
<script>
export default {
mounted () {
const espWebInstallButton = document.querySelector("esp-web-install-button");
const eraseCheck = document.getElementById("erase");
espWebInstallButton.addEventListener("state-changed", (ev) => { console.log(ev.detail) });
const selectFW = document.querySelector("select");
espWebInstallButton.manifest = selectFW.value;
selectFW.addEventListener("change", () => {
espWebInstallButton.manifest = selectFW.value;
});
eraseCheck.addEventListener("change", (ev) => {
if (eraseCheck.checked) {
espWebInstallButton.setAttribute('erase-first','');
} else {
espWebInstallButton.removeAttribute('erase-first');
}
});
}
}
</script>'''
manif_folder = "/dev/firmware_build/"
manif_path = 'docs/.vuepress/public/firmware_build/'
vue_path = 'docs/.vuepress/components/'
bin_path = 'toDeploy/'
cors_proxy = '' # 'https://cors.bridged.cc/'
esp32_blurl = 'https://github.com/espressif/arduino-esp32/raw/2.0.5/tools/sdk/esp32/bin/bootloader_dio_80m.bin'
esp32_boot = 'https://github.com/espressif/arduino-esp32/raw/2.0.5/tools/partitions/boot_app0.bin'
esp32c3_blurl = 'https://github.com/espressif/arduino-esp32/raw/2.0.5/tools/sdk/esp32c3/bin/bootloader_dio_80m.bin'
if not os.path.exists(manif_path):
os.makedirs(manif_path)
# copy OTA latest version definition
shutil.move("scripts/latest_version_dev.json",
manif_path + "latest_version_dev.json")
if not os.path.exists(vue_path):
os.makedirs(vue_path)
wu_file = open(vue_path + 'web-uploader.vue', 'w')
wu_file.write(wu_temp_p1)
bl_bin = requests.get(esp32_blurl)
filename = esp32_blurl.split('/')[-1]
with open(manif_path + filename, 'wb') as output_file:
output_file.write(bl_bin.content)
bl_bin_c3 = requests.get(esp32c3_blurl)
filename_c3 = esp32c3_blurl.split('/')[-1]
# add C3 at thend of the file name
filename_c3 = filename_c3.split('.')[0] + '-C3.' + filename_c3.split('.')[1]
with open(manif_path + filename_c3, 'wb') as output_file:
output_file.write(bl_bin_c3.content)
boot_bin = requests.get(esp32_boot)
filename = esp32_boot.split('/')[-1]
with open(manif_path + filename, 'wb') as output_file:
output_file.write(boot_bin.content)
for name in os.listdir(bin_path):
if 'firmware.bin' in name and ('esp32c3' not in name ) and ('esp32' in name or 'ttgo' in name or 'heltec' in name or 'thingpulse' in name or 'theengs' in name or 'lilygo' in name or 'shelly' in name or 'lolin_c3' in name or 'tinypico' in name):
fw = name.split('-firmware')[0]
man_file = fw + '.manifest.json'
print('Bin name:' + name)
part_name = name.split('-firmware')[0] + '-partitions.bin'
print('Partition name:' + part_name)
mani_str = mf_temp32.substitute({'cp': cors_proxy, 'part': manif_folder + part_name.split('/')[-1], 'bin': manif_folder + name.split(
'/')[-1], 'bl': manif_folder + esp32_blurl.split('/')[-1], 'boot': manif_folder + esp32_boot.split('/')[-1]})
with open(manif_path + man_file, 'w') as nf:
nf.write(mani_str)
wu_file.write(wu_temp_opt.substitute(
{'mff': manif_folder + man_file, 'mfn': fw}))
shutil.copyfile(bin_path + name, (manif_path + name))
shutil.copyfile(bin_path + part_name, (manif_path + part_name))
print('Created: ' + os.path.abspath(man_file))
wu_file.write(wu_temp_p2)
for name in os.listdir(bin_path):
if 'firmware.bin' in name and ('esp32c3' in name ):
fw = name.split('-firmware')[0]
man_file = fw + '.manifest.json'
print('Bin name:' + name)
part_name = name.split('-firmware')[0] + '-partitions.bin'
print('Partition name:' + part_name)
mani_str = mf_temp32c3.substitute({'cp': cors_proxy, 'part': manif_folder + part_name.split('/')[-1], 'bin': manif_folder + name.split(
'/')[-1], 'bl': manif_folder + filename_c3,'boot': manif_folder + esp32_boot.split('/')[-1]})
with open(manif_path + man_file, 'w') as nf:
nf.write(mani_str)
wu_file.write(wu_temp_opt.substitute(
{'mff': manif_folder + man_file, 'mfn': fw}))
shutil.copyfile(bin_path + name, (manif_path + name))
shutil.copyfile(bin_path + part_name, (manif_path + part_name))
print('Created: ' + os.path.abspath(man_file))
wu_file.write(wu_temp_p3)
for name in os.listdir(bin_path):
if 'firmware.bin' in name and ('nodemcu' in name or 'sonoff' in name or 'rf-wifi-gateway' in name or 'manual-wifi-test' in name or 'rfbridge' in name):
fw = name.split('-firmware')[0]
man_file = fw + '.manifest.json'
print('Bin name:' + name)
part_name = name.split('-firmware')[0] + '-partitions.bin'
print('Partition name:' + part_name)
mani_str = mf_temp8266.substitute(
{'cp': cors_proxy, 'bin': manif_folder + name.split('/')[-1]})
with open(manif_path + man_file, 'w') as nf:
nf.write(mani_str)
wu_file.write(
manif_folder + wu_temp_opt.substitute({'mff': manif_folder + man_file, 'mfn': fw}))
shutil.copyfile(bin_path + name, (manif_path + name))
print('Created: ' + os.path.abspath(man_file))
wu_file.write(wu_temp_end)
wu_file.close()