ESP8266MQTTMesh
ESP8266MQTTMesh copied to clipboard
Published
20 hours ago •
PhracturedBlue
PhracturedBlue
Error in send_ota.py
In your latest send_ota.py is error when i run it with Python 3.6.3
Updating firmware on the following nodes:
805de
Traceback (most recent call last):
File "ota.py", line 201, in <module>
main()
File "ota.py", line 197, in main
send_firmware(client, data, [args.node] if args.node else [])
File "ota.py", line 97, in send_firmware
client.publish("{}{}".format(send_topic, str(pos)), b64d)
File "D:\WinPython-64bit-3.6.3.0Qt5\python-3.6.3.amd64\lib\site-packages\paho\
mqtt\client.py", line 871, in publish
raise TypeError('payload must be a string, bytearray, int, float or None.')
TypeError: payload must be a string, bytearray, int, float or None.
when i in send firmware function change in client.publish b64d to b64d.decode(), it will stuck on
Updating firmware on the following nodes:
805de
1 node(s) missed the message, retrying
1 node(s) missed the message and no retires left
1 node(s) missed the message and no retires left
i have latest version of library on that device wich i update
finally is error that b64d must be decoded. I try it modified with this fix, and it works on Python 3.6.3
That is weird. I would expect an implicit conversion from a bytes object to a bytearray. I also don't see this issue on my system using 3.6.3 in linux.
Does it also work with this: b64d = bytearray(base64.b64encode(d))
# -*- coding: utf-8 -*-
#!/usr/bin/python3
from threading import Thread
import tkinter as tk
from tkinter import ttk
from tkinter import filedialog
import paho.mqtt.client as mqtt
import os
import sys
import argparse
import datetime
import hashlib
import base64
import time
import ssl
import re
import queue
class GUI(Thread):
def __init__(self):
super(GUI, self).__init__()
self.start()
def run(self):
self.client = mqtt.Client()
self.q = queue.Queue();
self.data = None
self.send_topic = None
self.master = tk.Tk()
self.master.title("OTA upgrader")
self.master.resizable(False, False)
self.server = dict()
self.port = dict()
self.user = dict()
self.password = dict()
self.ssl = dict()
self.topic = dict()
self.intopic = dict()
self.outtopic = dict()
self.node = dict()
self.firmware = dict()
self.file = dict()
self.frame = tk.Frame(self.master)
self.frame.grid(row=0, column=0)
self.textview = tk.Text(self.frame)
self.textview.grid(row=0, column=0, columnspan=2, rowspan=12, sticky=tk.W+tk.E+tk.N+tk.S, padx=5, pady=5)
self.server["var"] = tk.StringVar()
self.server["text"] = tk.Label(self.frame, text="server")
self.server["text"].grid(row=0, column=2, padx=5, sticky=tk.E)
self.server["entry"] = tk.Entry(self.frame, textvariable=self.server["var"])
self.server["entry"].grid(row=0, column=3, padx=5)
self.server["value"] = str()
self.port["var"] = tk.StringVar()
self.port["text"] = tk.Label(self.frame, text="port")
self.port["text"].grid(row=0, column=4, padx=5, sticky=tk.E)
self.port["entry"] = tk.Entry(self.frame, textvariable=self.port["var"])
self.port["entry"].grid(row=0, column=5, padx=5)
self.port["value"] = int()
self.user["var"] = tk.StringVar()
self.user["text"] = tk.Label(self.frame, text="username")
self.user["text"].grid(row=1, column=2, padx=5, sticky=tk.E)
self.user["entry"] = tk.Entry(self.frame, textvariable=self.user["var"])
self.user["entry"].grid(row=1, column=3, padx=5)
self.user["value"] = str()
self.password["var"] = tk.StringVar()
self.password["text"] = tk.Label(self.frame, text="password")
self.password["text"].grid(row=1, column=4, padx=5, sticky=tk.E)
self.password["entry"] = tk.Entry(self.frame, show="*", textvariable=self.password["var"])
self.password["entry"].grid(row=1, column=5, padx=5)
self.password["value"] = str()
self.ssl["var"] = tk.IntVar()
self.ssl["check"] = tk.Checkbutton(self.frame, text="SSL", variable=self.ssl["var"])
self.ssl["check"].grid(row=2, column=2, padx=5)
self.topic["var"] = tk.StringVar()
self.topic["text"] = tk.Label(self.frame, text="topic")
self.topic["text"].grid(row=3, column=2, padx=5, sticky=tk.E)
self.topic["entry"] = tk.Entry(self.frame, textvariable=self.topic["var"])
self.topic["entry"].grid(row=3, column=3, columnspan=3, padx=5, pady=5, sticky=tk.W+tk.E)
self.topic["value"] = str()
self.intopic["var"] = tk.StringVar()
self.intopic["text"] = tk.Label(self.frame, text="intopic")
self.intopic["text"].grid(row=4, column=2, padx=5, sticky=tk.E)
self.intopic["entry"] = tk.Entry(self.frame, textvariable=self.intopic["var"])
self.intopic["entry"].grid(row=4, column=3, columnspan=3, padx=5, pady=5, sticky=tk.W+tk.E)
self.intopic["value"] = str()
self.outtopic["var"] = tk.StringVar()
self.outtopic["text"] = tk.Label(self.frame, text="outtopic")
self.outtopic["text"].grid(row=5, column=2, padx=5, sticky=tk.E)
self.outtopic["entry"] = tk.Entry(self.frame, textvariable=self.outtopic["var"])
self.outtopic["entry"].grid(row=5, column=3, columnspan=3, padx=5, pady=5, sticky=tk.W+tk.E)
self.outtopic["value"] = str()
self.connect = tk.Button(self.frame, text=">>> CONNECT TO MQTT <<<", command=self.connectcmd)
self.connect.grid(row=6, column=2, columnspan=4, sticky=tk.W+tk.E, padx=5, pady=5)
self.radio=tk.IntVar()
self.node["var"] = tk.StringVar()
self.node["check"] = tk.Radiobutton(self.frame, text="node", variable=self.radio, value=0, command=self.radiocmd)
self.node["check"].grid(row=7, column=2, padx=5, pady=5)
self.node["entry"] = ttk.Combobox(self.frame, textvariable=self.node["var"])
self.node["entry"].grid(row=7, column=4, columnspan=3, padx=5, pady=5, sticky=tk.W+tk.E)
self.node["value"] = None
self.node["list"] = ["--- empty in this moment ---"]
self.node["list_id"] = dict()
self.firmware["var"] = tk.StringVar()
self.firmware["check"] = tk.Radiobutton(self.frame, text="firmware", variable=self.radio, value=1, command=self.radiocmd)
self.firmware["check"].grid(row=7, column=3, padx=5)
self.firmware["value"] = None
self.file["value"] = "---"
self.file["button"] = tk.Button(self.frame, text="OPEN *.BIN FILE", command=self.fileopen)
self.file["button"].grid(row=8, column=2, columnspan=4, sticky=tk.W+tk.E, padx=5)
self.file["text"] = tk.Label(self.frame, text=self.file["value"])
self.file["text"].grid(row=9, column=2, columnspan=4, sticky=tk.W+tk.E, padx=5)
self.send = tk.Button(self.frame, text=">>> SEND TO DEVICE(S) <<<", command=self.sendbin)
self.send.grid(row=10, column=2, columnspan=4, sticky=tk.W+tk.E, padx=5, pady=5)
self.cancel = tk.Button(self.frame, text="CANCEL", command=self.cancel)
self.cancel.grid(row=11, column=2, columnspan=4, sticky=tk.W+tk.E, padx=5, pady=5)
self.textview.delete(1.0, tk.END)
self.textview.insert(tk.END, "Process output...")
self.textview.config(state=tk.DISABLED)
self.master.mainloop()
def radiocmd(self):
if self.radio.get():
try:
self.node["entry"].destroy()
del self.node["entry"]
except:
pass
else:
self.firmware["entry"] = tk.Entry(self.frame, textvariable=self.firmware["var"])
self.firmware["entry"].grid(row=7, column=4, columnspan=3, padx=5, pady=5, sticky=tk.W+tk.E)
else:
try:
self.firmware["entry"].destroy()
del self.firmware["entry"]
except:
pass
else:
self.node["entry"] = ttk.Combobox(self.frame, textvariable=self.node["var"])
self.node["entry"].grid(row=7, column=4, columnspan=3, padx=5, pady=5, sticky=tk.W+tk.E)
finally:
self.node["entry"].config(values = self.node["list"])
self.node["var"].set(self.node["list"][0])
def connectcmd(self):
self.server["value"] = self.server["var"].get().lstrip().rstrip().lower()
try:
self.port["value"] = int(self.port["var"].get())
except:
self.port["value"] = 1883
self.port["var"].set(self.port["value"])
self.topic["value"] = self.topic["var"].get().lstrip().rstrip().lower()
self.intopic["value"] = self.intopic["var"].get().lstrip().rstrip().lower()
self.outtopic["value"] = self.outtopic["var"].get().lstrip().rstrip().lower()
self.clear()
if self.errors():
return
if self.topic["value"]:
self.intopic["value"] = self.topic["value"] + "in"
self.outtopic["value"] = self.topic["value"] + "out"
self.intopic["var"].set(self.intopic["value"])
self.outtopic["var"].set(self.outtopic["value"])
self.user["value"] = self.user["var"].get().lstrip().rstrip().lower()
self.password["value"] = self.password["var"].get().lstrip().rstrip().lower()
if self.ssl["var"].get():
client.tls_set(ca_certs=None, certfile=None, keyfile=None, cert_reqs=ssl.CERT_REQUIRED, tls_version=ssl.PROTOCOL_TLS, ciphers=None)
self.user["value"] = self.user["var"].get().lstrip().rstrip().lower()
self.password["value"] = self.password["var"].get().lstrip().rstrip().lower()
if (self.user["value"]) and (self.password["value"]):
del self.node["list"][:]
self.node["list_id"].clear()
self.client.username_pw_set(self.user["value"], self.password["value"])
self.client.on_connect = self.on_connect
self.client.on_message = self.on_message
self.client.on_publish = self.on_publish
self.client.connect(self.server["value"], self.port["value"], 60)
self.client.loop_start()
self.master.update_idletasks()
def cancel(self):
self.client.loop_stop()
self.client.disconnect()
self.master.destroy()
def on_publish(self, client, obj, mid):
print("i make publish")
pass
def on_connect(self, client, userdata, flags, rc):
self.show("Connected with result code " + str(rc))
# Subscribing in on_connect() means that if we lose the connection and
# reconnect then subscriptions will be renewed.
self.client.subscribe("{}/#".format(self.outtopic["value"]))
#self.client.subscribe("#")
def on_message(self, client, userdata, msg):
#esp8266-out/mesh_esp8266-6/check=MD5 Passed
print(msg.topic + " - " + msg.payload.decode())
match = []
if self.regex(r'/([0-9a-f]+)/ota/erase$', msg.topic, match):
self.q.put(["erase", match[0]])
print("queue on erase")
elif self.regex(r'([0-9a-f]+)/ota/md5/([0-9a-f]+)', msg.topic, match):
self.q.put(["md5", match[0], match[1], msg.payload])
print("queue on md5")
elif self.regex(r'([0-9a-f]+)/ota/check$', msg.topic, match):
self.q.put(["check", match[0], msg.payload])
print("queue on check")
else:
pass
if msg.topic.find("bssid/") != -1 :
ID = msg.topic[msg.topic.find("bssid/") + len("bssid/"):]
payload = msg.payload.decode("utf-8")
index = ID + " " + payload
self.node["list"].append(index)
self.node["list_id"][index] = ID
print("added bssid")
self.node["entry"].config(values = self.node["list"])
def fileopen(self):
name = filedialog.askopenfilename(filetypes=(("BIN files", "*.bin; *.BIN"), ("HEX files", "*.hex; *.HEX"),("All files", "*.*")))
self.file["value"] = name if name else self.file["value"]
self.file["text"].config(text=self.file["value"])
def clear(self):
self.textview.config(state=tk.NORMAL)
self.textview.delete(1.0, tk.END)
self.textview.config(state=tk.DISABLED)
self.master.update_idletasks()
def show(self, text):
self.textview.config(state=tk.NORMAL)
self.textview.insert(tk.END, text + "\n")
self.textview.config(state=tk.DISABLED)
self.master.update_idletasks()
def wait_for(self, nodes, msgtype, maxTime, retries=0, pubcmd=None):
seen = {}
origTime = time.time()
startTime = origTime
while True:
try:
msg = self.q.get(True, 0.1)
if msg[0] == msgtype and (not nodes or msg[1] in nodes):
node = msg[1]
seen[node] = msg
else:
print("Got unexpected {} for node {}".format(msgtype, msg[1], msg))
except queue.Empty:
if time.time() - startTime < maxTime:
continue
if retries:
retries -= 1
print("{} node(s) missed the message, retrying".format(len(nodes) - len(seen.keys())))
self.client.publish(pubcmd[0], pubcmd[1])
print("after publish in waitfor")
startTime = time.time()
else:
print("{} node(s) missed the message and no retires left".format(len(nodes) - len(seen.keys())))
if nodes and len(seen.keys()) == len(nodes):
break
return seen
def send_firmware(self, nodes):
md5 = base64.b64encode(hashlib.md5(self.data).digest())
payload = "md5:%s,len:%d" %(md5.decode(), len(self.data))
print("Erasing...")
self.client.publish("{}start".format(self.send_topic), payload)
nodes = list(self.wait_for(nodes, 'erase', 10).keys())
print("Updating firmware on the following nodes:\n\t{}".format("\n\t".join(nodes)))
pos = 0
while len(self.data):
d = self.data[0:768]
b64d = bytearray(base64.b64encode(d))
self.data = self.data[768:]
self.client.publish("{}{}".format(self.send_topic, str(pos)), b64d)
expected_md5 = hashlib.md5(d).hexdigest().encode('utf-8')
seen = {}
retries = 2
print("before waitfor")
seen = self.wait_for(nodes, 'md5', 1.0, retries, ["{}{}".format(self.send_topic, str(pos)), b64d])
print("after waitfor")
for node in nodes:
if node not in seen:
print("No MD5 found for {} at 0x{}".format(node, pos))
return
addr = int(seen[node][2], 16)
md5 = seen[node][3]
if pos != addr:
print("Got unexpected address 0x{} (expected: 0x{}) from node {}".format(addr, pos, node))
return
if md5 != expected_md5:
print("Got unexpected md5 for node {} at 0x{}".format(node, addr))
print("\t {} (expected: {})".format(md5, expected_md5))
return
pos += len(d)
if pos % (768 * 13) == 0:
print("Transmitted %d bytes" % (pos))
print("Completed send")
self.client.publish("{}check".format(self.send_topic), "")
seen = self.wait_for(nodes, 'check', 5)
err = False
for node in nodes:
if node not in seen:
print("No verify result found for {}".format(node))
err = True
if seen[node][2] != b'MD5 Passed':
print("Node {} did not pass final MD5 check: {}".format(node, seen[node][2]))
err = True
if err:
return
self.show("Checksum verified. Flashing and rebooting now...")
self.client.publish("{}flash".format(self.send_topic), "")
def regex(self, pattern, txt, group):
group.clear()
match = re.search(pattern, txt)
if match:
if match.groupdict():
for k,v in match.groupdict().items():
group[k] = v
else:
group.extend(match.groups())
return True
return False
def errors(self):
status = False
#missing server
if self.server["value"] == "":
self.show("ERROR: No server IP/FQDN!")
status = True
#missing topic or intopic + outtopic
if not(self.topic["value"] != "" or (self.topic["value"] == "" and self.intopic["value"] != "" and self.outtopic["value"] != "")):
self.show("ERROR: No topic!")
status = True
return status
def sendbin(self):
with open(self.file["value"], "rb") as fh:
self.data = fh.read()
if self.radio.get():
self.send_topic = "{}/ota/{}/".format(self.intopic["value"], self.firmware["var"].get()) #firmware
else:
node_number=self.node["list_id"][self.node["var"].get()]
print (node_number)
self.send_topic = "{}/ota/{}/".format(self.intopic["value"], node_number) #node
print(self.send_topic)
self.client.loop_start()
self.master.update_idletasks()
self.send_firmware([node_number] if node_number else [])
print("succesfull send firmware send")
################################################################
####################### OTA upgrader ########################
########################### shajek ########################
####################### 2018 ########################
################################################################
if __name__ == "__main__":
apl = GUI()
This is first draft, for now i dont handle some exceptions around input parameters, but if you enter host, optional username and password and topic (i have standard esp8266-), and click connect, if is all succesfull it will fill up combobox when you click of node radiobutton (or click on firmware and then on node and choose one), then you select your firmware bin file and then click Send to device and here is problem, i successful send erase command, but then i stuck on "wait for" commmand, as you see i add few prints and i see that when i am in wait for i dont get any message can you look at it ? it is basically dressed up your code
# -*- coding: utf-8 -*-
#!/usr/bin/python3
from threading import Thread
import tkinter as tk
from tkinter import ttk
from tkinter import filedialog
import paho.mqtt.client as mqtt
import base64, hashlib, queue, re, time, ssl
class GUI(Thread):
def __init__(self):
super(GUI, self).__init__()
self.start()
def run(self):
self.master = tk.Tk()
self.master.title("OTA upgrader")
self.master.resizable(False, False)
self.server = dict()
self.port = dict()
self.user = dict()
self.password = dict()
self.ssl = dict()
self.topic = dict()
self.intopic = dict()
self.outtopic = dict()
self.node = dict()
self.firmware = dict()
self.file = dict()
self.frame = tk.Frame(self.master)
self.frame.grid(row=0, column=0)
self.textview = tk.Text(self.frame)
self.textview.grid(row=0, column=0, columnspan=2, rowspan=12, sticky=tk.W+tk.E+tk.N+tk.S, padx=5, pady=5)
self.server["var"] = tk.StringVar()
self.server["text"] = tk.Label(self.frame, text="server")
self.server["text"].grid(row=0, column=2, padx=5, sticky=tk.E)
self.server["entry"] = tk.Entry(self.frame, textvariable=self.server["var"])
self.server["entry"].grid(row=0, column=3, padx=5)
self.server["value"] = str()
self.port["var"] = tk.StringVar()
self.port["text"] = tk.Label(self.frame, text="port")
self.port["text"].grid(row=0, column=4, padx=5, sticky=tk.E)
self.port["entry"] = tk.Entry(self.frame, textvariable=self.port["var"])
self.port["entry"].grid(row=0, column=5, padx=5)
self.port["value"] = int()
self.user["var"] = tk.StringVar()
self.user["text"] = tk.Label(self.frame, text="username")
self.user["text"].grid(row=1, column=2, padx=5, sticky=tk.E)
self.user["entry"] = tk.Entry(self.frame, textvariable=self.user["var"])
self.user["entry"].grid(row=1, column=3, padx=5)
self.user["value"] = str()
self.password["var"] = tk.StringVar()
self.password["text"] = tk.Label(self.frame, text="password")
self.password["text"].grid(row=1, column=4, padx=5, sticky=tk.E)
self.password["entry"] = tk.Entry(self.frame, show="*", textvariable=self.password["var"])
self.password["entry"].grid(row=1, column=5, padx=5)
self.password["value"] = str()
self.ssl["var"] = tk.IntVar()
self.ssl["check"] = tk.Checkbutton(self.frame, text="SSL", variable=self.ssl["var"])
self.ssl["check"].grid(row=2, column=2, padx=5)
self.topic["var"] = tk.StringVar()
self.topic["text"] = tk.Label(self.frame, text="topic")
self.topic["text"].grid(row=3, column=2, padx=5, sticky=tk.E)
self.topic["entry"] = tk.Entry(self.frame, textvariable=self.topic["var"])
self.topic["entry"].grid(row=3, column=3, columnspan=3, padx=5, pady=5, sticky=tk.W+tk.E)
self.topic["value"] = str()
self.intopic["var"] = tk.StringVar()
self.intopic["text"] = tk.Label(self.frame, text="intopic")
self.intopic["text"].grid(row=4, column=2, padx=5, sticky=tk.E)
self.intopic["entry"] = tk.Entry(self.frame, textvariable=self.intopic["var"])
self.intopic["entry"].grid(row=4, column=3, columnspan=3, padx=5, pady=5, sticky=tk.W+tk.E)
self.intopic["value"] = str()
self.outtopic["var"] = tk.StringVar()
self.outtopic["text"] = tk.Label(self.frame, text="outtopic")
self.outtopic["text"].grid(row=5, column=2, padx=5, sticky=tk.E)
self.outtopic["entry"] = tk.Entry(self.frame, textvariable=self.outtopic["var"])
self.outtopic["entry"].grid(row=5, column=3, columnspan=3, padx=5, pady=5, sticky=tk.W+tk.E)
self.outtopic["value"] = str()
self.connect = tk.Button(self.frame, text=">>> CONNECT TO MQTT <<<", command = lambda : self.connectcmd(client))
self.connect.grid(row=6, column=2, columnspan=4, sticky=tk.W+tk.E, padx=5, pady=5)
self.radio=tk.IntVar()
self.node["var"] = tk.StringVar()
self.node["check"] = tk.Radiobutton(self.frame, text="node", variable=self.radio, value=0, command=self.radiocmd)
self.node["check"].grid(row=7, column=2, padx=5, pady=5)
self.node["entry"] = ttk.Combobox(self.frame, textvariable=self.node["var"])
self.node["entry"].grid(row=7, column=4, columnspan=3, padx=5, pady=5, sticky=tk.W+tk.E)
self.node["value"] = None
self.node["list"] = ["--- empty in this moment ---"]
self.node["list_id"] = dict()
self.firmware["var"] = tk.StringVar()
self.firmware["check"] = tk.Radiobutton(self.frame, text="firmware", variable=self.radio, value=1, command=self.radiocmd)
self.firmware["check"].grid(row=7, column=3, padx=5)
self.firmware["value"] = None
self.file["value"] = "---"
self.file["button"] = tk.Button(self.frame, text="OPEN *.BIN FILE", command=self.fileopen)
self.file["button"].grid(row=8, column=2, columnspan=4, sticky=tk.W+tk.E, padx=5)
self.file["text"] = tk.Label(self.frame, text=self.file["value"])
self.file["text"].grid(row=9, column=2, columnspan=4, sticky=tk.W+tk.E, padx=5)
self.send = tk.Button(self.frame, text=">>> SEND TO DEVICE(S) <<<", command=self.sendbin)
self.send.grid(row=10, column=2, columnspan=4, sticky=tk.W+tk.E, padx=5, pady=5)
self.cancel = tk.Button(self.frame, text="CANCEL", command=self.cancel)
self.cancel.grid(row=11, column=2, columnspan=4, sticky=tk.W+tk.E, padx=5, pady=5)
self.textview.delete(1.0, tk.END)
self.textview.insert(tk.END, "Process output...")
self.textview.config(state=tk.DISABLED)
self.server["var"].set("mqtt.deepstar.eu")
self.port["var"].set(1883)
self.user["var"].set("test")
self.password["var"].set("test123")
self.topic["var"].set("esp8266-")
self.intopic["var"].set("esp8266-in")
self.outtopic["var"].set("esp8266-out")
self.master.mainloop()
def radiocmd(self):
if self.radio.get():
try:
self.node["entry"].destroy()
del self.node["entry"]
except:
pass
else:
self.firmware["entry"] = tk.Entry(self.frame, textvariable=self.firmware["var"])
self.firmware["entry"].grid(row=7, column=4, columnspan=3, padx=5, pady=5, sticky=tk.W+tk.E)
else:
try:
self.firmware["entry"].destroy()
del self.firmware["entry"]
except:
pass
else:
self.node["entry"] = ttk.Combobox(self.frame, textvariable=self.node["var"])
self.node["entry"].grid(row=7, column=4, columnspan=3, padx=5, pady=5, sticky=tk.W+tk.E)
finally:
self.node["entry"].config(values = self.node["list"])
self.node["var"].set(self.node["list"][0])
def connectcmd(self, client):
self.server["value"] = self.server["var"].get().lstrip().rstrip().lower()
try:
self.port["value"] = int(self.port["var"].get())
except:
self.port["value"] = 1883
self.port["var"].set(self.port["value"])
self.topic["value"] = self.topic["var"].get().lstrip().rstrip().lower()
self.intopic["value"] = self.intopic["var"].get().lstrip().rstrip().lower()
self.outtopic["value"] = self.outtopic["var"].get().lstrip().rstrip().lower()
self.clear()
if self.errors():
return
if self.topic["value"]:
self.intopic["value"] = self.topic["value"] + "in"
self.outtopic["value"] = self.topic["value"] + "out"
self.intopic["var"].set(self.intopic["value"])
self.outtopic["var"].set(self.outtopic["value"])
inTopic = self.intopic["value"]
outTopic = self.outtopic["value"]
self.user["value"] = self.user["var"].get().lstrip().rstrip().lower()
self.password["value"] = self.password["var"].get().lstrip().rstrip().lower()
if self.ssl["var"].get():
client.tls_set(ca_certs=None, certfile=None, keyfile=None, cert_reqs=ssl.CERT_REQUIRED, tls_version=ssl.PROTOCOL_TLS, ciphers=None)
self.user["value"] = self.user["var"].get().lstrip().rstrip().lower()
self.password["value"] = self.password["var"].get().lstrip().rstrip().lower()
if (self.user["value"]) and (self.password["value"]):
del self.node["list"][:]
self.node["list_id"].clear()
client.username_pw_set(self.user["value"], self.password["value"])
client.on_connect = on_connect
client.on_message = on_message
client.connect(self.server["value"], self.port["value"], 60)
client.loop_start()
self.master.update_idletasks()
def cancel(self):
client.loop_stop()
client.disconnect()
self.master.destroy()
def fileopen(self):
name = filedialog.askopenfilename(filetypes=(("BIN files", "*.bin; *.BIN"), ("HEX files", "*.hex; *.HEX"),("All files", "*.*")))
self.file["value"] = name if name else self.file["value"]
self.file["text"].config(text=self.file["value"])
def clear(self):
self.textview.config(state=tk.NORMAL)
self.textview.delete(1.0, tk.END)
self.textview.config(state=tk.DISABLED)
self.master.update_idletasks()
def show(self, text):
self.textview.config(state=tk.NORMAL)
self.textview.insert(tk.END, text + "\n")
self.textview.config(state=tk.DISABLED)
self.master.update_idletasks()
def errors(self):
status = False
#missing server
if self.server["value"] == "":
self.show("ERROR: No server IP/FQDN!")
status = True
#missing topic or intopic + outtopic
if not(self.topic["value"] != "" or (self.topic["value"] == "" and self.intopic["value"] != "" and self.outtopic["value"] != "")):
self.show("ERROR: No topic!")
status = True
return status
def sendbin(self):
with open(self.file["value"], "rb") as fh:
data = fh.read()
if self.radio.get():
self.send_topic = "{}/ota/{}/".format(self.intopic["value"], self.firmware["var"].get()) #firmware
else:
node_number=self.node["list_id"][self.node["var"].get()]
self.send_topic = "{}/ota/{}/".format(self.intopic["value"], node_number) #node
self.master.update_idletasks()
send_firmware(client, data, [node_number] if node_number else [])
###################### end of class GUI #######################
def regex(pattern, txt, group):
group.clear()
match = re.search(pattern, txt)
if match:
if match.groupdict():
for k,v in match.groupdict().items():
group[k] = v
else:
group.extend(match.groups())
return True
return False
def on_connect(client, userdata, flags, rc):
apl.show("Connected with result code {}.".format(str(rc)))
# Subscribing in on_connect() means that if we lose the connection and reconnect then subscriptions will be renewed.
outTopic = apl.outtopic["value"]
client.subscribe("{}/#".format(outTopic))
# The callback for when a PUBLISH message is received from the server.
def on_message(client, userdata, msg):
#esp8266-out/mesh_esp8266-6/check=MD5 Passed
match = []
if regex(r'/([0-9a-f]+)/ota/erase$', msg.topic, match):
q.put(["erase", match[0]])
print("Got erase message {}".format(match[0]))
elif regex(r'([0-9a-f]+)/ota/md5/([0-9a-f]+)', msg.topic, match):
q.put(["md5", match[0], match[1], msg.payload])
print("Got md5 message {} match0 = {}, match1={}".format(msg.payload, match[0], match[1]))
elif regex(r'([0-9a-f]+)/ota/check$', msg.topic, match):
q.put(["check", match[0], msg.payload])
print("Got ota check message {} match = {},".format(msg.topic, match))
if msg.topic.find("bssid/") != -1 :
ID = msg.topic[msg.topic.find("bssid/") + len("bssid/"):]
payload = msg.payload.decode("utf-8")
index = ID + " " + payload
apl.node["list"].append(index)
apl.node["list_id"][index] = ID
apl.show("Bssid {}, {} added.".format(ID, payload))
apl.node["entry"].config(values = apl.node["list"])
def wait_for(nodes, msgtype, maxTime, retries=0, pubcmd=None):
seen = {}
origTime = time.time()
startTime = origTime
while True:
try:
msg = q.get(True, 0.1)
print("Got msg in wait_for = {}".format(msg))
if msg[0] == msgtype and (not nodes or msg[1] in nodes):
node = msg[1]
seen[node] = msg
print("Printing in if in wait_for node ={}, seen[node]= {}".format(node,seen[node]))
else:
print("Got unexpected {} for node {}".format(msgtype, msg[1], msg))
except queue.Empty:
if time.time() - startTime < maxTime:
continue
if retries:
retries -= 1
print("{} node(s) missed the message, retrying".format(len(nodes) - len(seen.keys())))
client.publish(pubcmd[0], pubcmd[1])
startTime = time.time()
else:
print("{} node(s) missed the message and no retires left".format(len(nodes) - len(seen.keys())))
if nodes and len(seen.keys()) == len(nodes):
break
#print("Elapsed time waiting for {} messages: {} seconds".format(msgtype, time.time() - origTime))
return seen
def send_firmware(client, data, nodes):
md5 = base64.b64encode(hashlib.md5(data).digest())
payload = "md5:%s,len:%d" %(md5.decode(), len(data))
print("Erasing...")
send_topic = apl.send_topic
client.publish("{}start".format(send_topic), payload)
time.sleep(2)
nodes = list(wait_for(nodes, 'erase', 10).keys())
print("Updating firmware on the following nodes:\n\t{}".format("\n\t".join(nodes)))
pos = 0
while len(data):
d = data[0:768]
b64d = bytearray(base64.b64encode(d))
data = data[768:]
client.publish("{}{}".format(send_topic, str(pos)), b64d)
expected_md5 = hashlib.md5(d).hexdigest().encode('utf-8')
seen = {}
retries = 1
seen = wait_for(nodes, 'md5', 1.0, 1, ["{}{}".format(send_topic, str(pos)), b64d])
for node in nodes:
if node not in seen:
print("No MD5 found for {} at 0x{}".format(node, pos))
return
addr = int(seen[node][2], 16)
md5 = seen[node][3]
if pos != addr:
print("Got unexpected address 0x{} (expected: 0x{}) from node {}".format(addr, pos, node))
return
if md5 != expected_md5:
print("Got unexpected md5 for node {} at 0x{}".format(node, addr))
print("\t {} (expected: {})".format(md5, expected_md5))
return
pos += len(d)
if pos % (768 * 13) == 0:
print("Transmitted %d bytes" % (pos))
print("Completed send")
client.publish("{}check".format(send_topic), "")
seen = wait_for(nodes, 'check', 5)
err = False
for node in nodes:
if node not in seen:
print("No verify result found for {}".format(node))
err = True
if seen[node][2] != b'MD5 Passed':
print("Node {} did not pass final MD5 check: {}".format(node, seen[node][2]))
err = True
if err:
return
print("Checksum verified. Flashing and rebooting now...")
client.publish("{}flash".format(send_topic), "")
################################################################
####################### OTA upgrader ########################
####################### Stanislav Hajek ########################
####################### 2018 ########################
################################################################
if __name__ == "__main__":
global client, data, send_topic, topic, inTopic, outTopic
client, q, data, send_topic, inTopic, outTopic = mqtt.Client(), queue.Queue(), None, None, None, None
apl = GUI()
Hello, me again. It is definitely wierd problem in mqtt paho and your functions. special to wait_for where blocks any kind of comunicatiom. I for now use you untouched code and i fill it with GUI from that prints you can see where it stuck. I will upgrade graphics, but for now doesnt work intention of this whole thing. I see that you prepare some nodes array, maybe for selective node update ? I will add popup windows where would be tickboxes how many we have bssid retain messages on broker. Please look at it and write some notes or decision or whatever :) Thank you
# -*- coding: utf-8 -*-
#!/usr/bin/python3
from threading import Thread
import tkinter as tk
from tkinter import ttk
from tkinter import filedialog
import paho.mqtt.client as mqtt
import base64, hashlib, queue, re, time, ssl
class GUI(Thread):
def __init__(self):
super(GUI, self).__init__()
self.start()
def run(self):
self.master = tk.Tk()
self.master.title("OTA upgrader")
self.master.resizable(False, False)
self.server = dict()
self.port = dict()
self.user = dict()
self.password = dict()
self.ssl = dict()
self.topic = dict()
self.intopic = dict()
self.outtopic = dict()
self.node = dict()
self.firmware = dict()
self.file = dict()
self.frame = tk.Frame(self.master)
self.frame.grid(row=0, column=0)
self.textview = tk.Text(self.frame)
self.textview.grid(row=0, column=0, columnspan=2, rowspan=12, sticky=tk.W+tk.E+tk.N+tk.S, padx=5, pady=5)
self.server["var"] = tk.StringVar()
self.server["text"] = tk.Label(self.frame, text="server")
self.server["text"].grid(row=0, column=2, padx=5, sticky=tk.E)
self.server["entry"] = tk.Entry(self.frame, textvariable=self.server["var"])
self.server["entry"].grid(row=0, column=3, padx=5)
self.server["value"] = str()
self.port["var"] = tk.StringVar()
self.port["text"] = tk.Label(self.frame, text="port")
self.port["text"].grid(row=0, column=4, padx=5, sticky=tk.E)
self.port["entry"] = tk.Entry(self.frame, textvariable=self.port["var"])
self.port["entry"].grid(row=0, column=5, padx=5)
self.port["value"] = int()
self.user["var"] = tk.StringVar()
self.user["text"] = tk.Label(self.frame, text="username")
self.user["text"].grid(row=1, column=2, padx=5, sticky=tk.E)
self.user["entry"] = tk.Entry(self.frame, textvariable=self.user["var"])
self.user["entry"].grid(row=1, column=3, padx=5)
self.user["value"] = str()
self.password["var"] = tk.StringVar()
self.password["text"] = tk.Label(self.frame, text="password")
self.password["text"].grid(row=1, column=4, padx=5, sticky=tk.E)
self.password["entry"] = tk.Entry(self.frame, show="*", textvariable=self.password["var"])
self.password["entry"].grid(row=1, column=5, padx=5)
self.password["value"] = str()
self.ssl["var"] = tk.IntVar()
self.ssl["check"] = tk.Checkbutton(self.frame, text="SSL", variable=self.ssl["var"])
self.ssl["check"].grid(row=2, column=2, padx=5)
self.topic["var"] = tk.StringVar()
self.topic["text"] = tk.Label(self.frame, text="topic")
self.topic["text"].grid(row=3, column=2, padx=5, sticky=tk.E)
self.topic["entry"] = tk.Entry(self.frame, textvariable=self.topic["var"])
self.topic["entry"].grid(row=3, column=3, columnspan=3, padx=5, pady=5, sticky=tk.W+tk.E)
self.topic["value"] = str()
self.intopic["var"] = tk.StringVar()
self.intopic["text"] = tk.Label(self.frame, text="intopic")
self.intopic["text"].grid(row=4, column=2, padx=5, sticky=tk.E)
self.intopic["entry"] = tk.Entry(self.frame, textvariable=self.intopic["var"])
self.intopic["entry"].grid(row=4, column=3, columnspan=3, padx=5, pady=5, sticky=tk.W+tk.E)
self.intopic["value"] = str()
self.outtopic["var"] = tk.StringVar()
self.outtopic["text"] = tk.Label(self.frame, text="outtopic")
self.outtopic["text"].grid(row=5, column=2, padx=5, sticky=tk.E)
self.outtopic["entry"] = tk.Entry(self.frame, textvariable=self.outtopic["var"])
self.outtopic["entry"].grid(row=5, column=3, columnspan=3, padx=5, pady=5, sticky=tk.W+tk.E)
self.outtopic["value"] = str()
self.connect = tk.Button(self.frame, text=">>> CONNECT TO MQTT <<<", command = lambda : self.connectcmd(client))
self.connect.grid(row=6, column=2, columnspan=4, sticky=tk.W+tk.E, padx=5, pady=5)
self.radio=tk.IntVar()
self.node["var"] = tk.StringVar()
self.node["check"] = tk.Radiobutton(self.frame, text="node", variable=self.radio, value=0, command=self.radiocmd)
self.node["check"].grid(row=7, column=2, padx=5, pady=5)
self.node["entry"] = ttk.Combobox(self.frame, textvariable=self.node["var"])
self.node["entry"].grid(row=7, column=4, columnspan=3, padx=5, pady=5, sticky=tk.W+tk.E)
self.node["value"] = None
self.node["list"] = ["--- empty in this moment ---"]
self.node["list_id"] = dict()
self.firmware["var"] = tk.StringVar()
self.firmware["check"] = tk.Radiobutton(self.frame, text="firmware", variable=self.radio, value=1, command=self.radiocmd)
self.firmware["check"].grid(row=7, column=3, padx=5)
self.firmware["value"] = None
self.file["value"] = "---"
self.file["button"] = tk.Button(self.frame, text="OPEN *.BIN FILE", command=self.fileopen)
self.file["button"].grid(row=8, column=2, columnspan=4, sticky=tk.W+tk.E, padx=5)
self.file["text"] = tk.Label(self.frame, text=self.file["value"])
self.file["text"].grid(row=9, column=2, columnspan=4, sticky=tk.W+tk.E, padx=5)
self.send = tk.Button(self.frame, text=">>> SEND TO DEVICE(S) <<<", command=self.sendbin)
self.send.grid(row=10, column=2, columnspan=4, sticky=tk.W+tk.E, padx=5, pady=5)
self.cancel = tk.Button(self.frame, text="CANCEL", command=self.cancel)
self.cancel.grid(row=11, column=2, columnspan=4, sticky=tk.W+tk.E, padx=5, pady=5)
self.textview.delete(1.0, tk.END)
self.textview.insert(tk.END, "Process output...")
self.textview.config(state=tk.DISABLED)
self.server["var"].set("mqtt.ofyourserver.eu")
self.port["var"].set(1883)
self.user["var"].set("somuser")
self.password["var"].set("somepassword")
self.topic["var"].set("esp8266-")
self.intopic["var"].set("esp8266-in")
self.outtopic["var"].set("esp8266-out")
self.master.mainloop()
def radiocmd(self):
if self.radio.get():
try:
self.node["entry"].destroy()
del self.node["entry"]
except:
pass
else:
self.firmware["entry"] = tk.Entry(self.frame, textvariable=self.firmware["var"])
self.firmware["entry"].grid(row=7, column=4, columnspan=3, padx=5, pady=5, sticky=tk.W+tk.E)
else:
try:
self.firmware["entry"].destroy()
del self.firmware["entry"]
except:
pass
else:
self.node["entry"] = ttk.Combobox(self.frame, textvariable=self.node["var"])
self.node["entry"].grid(row=7, column=4, columnspan=3, padx=5, pady=5, sticky=tk.W+tk.E)
finally:
self.node["entry"].config(values = self.node["list"])
self.node["var"].set(self.node["list"][0])
def connectcmd(self, client):
self.server["value"] = self.server["var"].get().lstrip().rstrip().lower()
try:
self.port["value"] = int(self.port["var"].get())
except:
self.port["value"] = 1883
self.port["var"].set(self.port["value"])
self.topic["value"] = self.topic["var"].get().lstrip().rstrip().lower()
self.intopic["value"] = self.intopic["var"].get().lstrip().rstrip().lower()
self.outtopic["value"] = self.outtopic["var"].get().lstrip().rstrip().lower()
self.clear()
if self.errors():
return
if self.topic["value"]:
self.intopic["value"] = self.topic["value"] + "in"
self.outtopic["value"] = self.topic["value"] + "out"
self.intopic["var"].set(self.intopic["value"])
self.outtopic["var"].set(self.outtopic["value"])
inTopic = self.intopic["value"]
outTopic = self.outtopic["value"]
self.user["value"] = self.user["var"].get().lstrip().rstrip().lower()
self.password["value"] = self.password["var"].get().lstrip().rstrip().lower()
if self.ssl["var"].get():
client.tls_set(ca_certs=None, certfile=None, keyfile=None, cert_reqs=ssl.CERT_REQUIRED, tls_version=ssl.PROTOCOL_TLS, ciphers=None)
self.user["value"] = self.user["var"].get().lstrip().rstrip().lower()
self.password["value"] = self.password["var"].get().lstrip().rstrip().lower()
if (self.user["value"]) and (self.password["value"]):
del self.node["list"][:]
self.node["list_id"].clear()
client.username_pw_set(self.user["value"], self.password["value"])
client.on_connect = on_connect
client.on_message = on_message
client.connect(self.server["value"], self.port["value"], 60)
client.loop_start()
self.master.update_idletasks()
def cancel(self):
client.loop_stop()
client.disconnect()
self.master.destroy()
def fileopen(self):
name = filedialog.askopenfilename(filetypes=(("BIN files", "*.bin; *.BIN"), ("HEX files", "*.hex; *.HEX"),("All files", "*.*")))
self.file["value"] = name if name else self.file["value"]
self.file["text"].config(text=self.file["value"])
def clear(self):
self.textview.config(state=tk.NORMAL)
self.textview.delete(1.0, tk.END)
self.textview.config(state=tk.DISABLED)
self.master.update_idletasks()
def show(self, text):
self.textview.config(state=tk.NORMAL)
self.textview.insert(tk.END, text + "\n")
self.textview.config(state=tk.DISABLED)
self.master.update_idletasks()
def errors(self):
status = False
#missing server
if self.server["value"] == "":
self.show("ERROR: No server IP/FQDN!")
status = True
#missing topic or intopic + outtopic
if not(self.topic["value"] != "" or (self.topic["value"] == "" and self.intopic["value"] != "" and self.outtopic["value"] != "")):
self.show("ERROR: No topic!")
status = True
return status
def sendbin(self):
with open(self.file["value"], "rb") as fh:
data = fh.read()
if self.radio.get():
self.send_topic = "{}/ota/{}/".format(self.intopic["value"], self.firmware["var"].get()) #firmware
else:
node_number=self.node["list_id"][self.node["var"].get()]
self.send_topic = "{}/ota/{}/".format(self.intopic["value"], node_number) #node
self.master.update_idletasks()
send_firmware(client, data, [node_number] if node_number else [])
###################### end of class GUI #######################
def regex(pattern, txt, group):
group.clear()
match = re.search(pattern, txt)
if match:
if match.groupdict():
for k,v in match.groupdict().items():
group[k] = v
else:
group.extend(match.groups())
return True
return False
def on_connect(client, userdata, flags, rc):
apl.show("Connected with result code {}.".format(str(rc)))
# Subscribing in on_connect() means that if we lose the connection and reconnect then subscriptions will be renewed.
outTopic = apl.outtopic["value"]
client.subscribe("{}/#".format(outTopic))
# The callback for when a PUBLISH message is received from the server.
def on_message(client, userdata, msg):
#esp8266-out/mesh_esp8266-6/check=MD5 Passed
match = []
if regex(r'/([0-9a-f]+)/ota/erase$', msg.topic, match):
q.put(["erase", match[0]])
print("Got erase message {}".format(match[0]))
elif regex(r'([0-9a-f]+)/ota/md5/([0-9a-f]+)', msg.topic, match):
q.put(["md5", match[0], match[1], msg.payload])
print("Got md5 message {} match0 = {}, match1={}".format(msg.payload, match[0], match[1]))
elif regex(r'([0-9a-f]+)/ota/check$', msg.topic, match):
q.put(["check", match[0], msg.payload])
print("Got ota check message {} match = {},".format(msg.topic, match))
if msg.topic.find("bssid/") != -1 :
ID = msg.topic[msg.topic.find("bssid/") + len("bssid/"):]
payload = msg.payload.decode("utf-8")
index = ID + " " + payload
apl.node["list"].append(index)
apl.node["list_id"][index] = ID
apl.show("Bssid {}, {} added.".format(ID, payload))
apl.node["entry"].config(values = apl.node["list"])
def wait_for(nodes, msgtype, maxTime, retries=0, pubcmd=None):
seen = {}
origTime = time.time()
startTime = origTime
while True:
try:
msg = q.get(True, 0.1)
print("Got msg in wait_for = {}".format(msg))
if msg[0] == msgtype and (not nodes or msg[1] in nodes):
node = msg[1]
seen[node] = msg
print("Printing in if in wait_for node ={}, seen[node]= {}".format(node,seen[node]))
else:
print("Got unexpected {} for node {}".format(msgtype, msg[1], msg))
except queue.Empty:
if time.time() - startTime < maxTime:
continue
if retries:
retries -= 1
print("{} node(s) missed the message, retrying".format(len(nodes) - len(seen.keys())))
client.publish(pubcmd[0], pubcmd[1])
startTime = time.time()
else:
print("{} node(s) missed the message and no retires left".format(len(nodes) - len(seen.keys())))
if nodes and len(seen.keys()) == len(nodes):
break
#print("Elapsed time waiting for {} messages: {} seconds".format(msgtype, time.time() - origTime))
return seen
def send_firmware(client, data, nodes):
md5 = base64.b64encode(hashlib.md5(data).digest())
payload = "md5:%s,len:%d" %(md5.decode(), len(data))
print("Erasing...")
send_topic = apl.send_topic
client.publish("{}start".format(send_topic), payload)
time.sleep(2)
nodes = list(wait_for(nodes, 'erase', 10).keys())
print("Updating firmware on the following nodes:\n\t{}".format("\n\t".join(nodes)))
pos = 0
while len(data):
d = data[0:768]
b64d = bytearray(base64.b64encode(d))
data = data[768:]
client.publish("{}{}".format(send_topic, str(pos)), b64d)
expected_md5 = hashlib.md5(d).hexdigest().encode('utf-8')
seen = {}
retries = 1
seen = wait_for(nodes, 'md5', 1.0, 1, ["{}{}".format(send_topic, str(pos)), b64d])
for node in nodes:
if node not in seen:
print("No MD5 found for {} at 0x{}".format(node, pos))
return
addr = int(seen[node][2], 16)
md5 = seen[node][3]
if pos != addr:
print("Got unexpected address 0x{} (expected: 0x{}) from node {}".format(addr, pos, node))
return
if md5 != expected_md5:
print("Got unexpected md5 for node {} at 0x{}".format(node, addr))
print("\t {} (expected: {})".format(md5, expected_md5))
return
pos += len(d)
if pos % (768 * 13) == 0:
print("Transmitted %d bytes" % (pos))
print("Completed send")
client.publish("{}check".format(send_topic), "")
seen = wait_for(nodes, 'check', 5)
err = False
for node in nodes:
if node not in seen:
print("No verify result found for {}".format(node))
err = True
if seen[node][2] != b'MD5 Passed':
print("Node {} did not pass final MD5 check: {}".format(node, seen[node][2]))
err = True
if err:
return
print("Checksum verified. Flashing and rebooting now...")
client.publish("{}flash".format(send_topic), "")
################################################################
####################### OTA upgrader ########################
####################### Stanislav Hajek ########################
####################### 2018 ########################
################################################################
if __name__ == "__main__":
global client, data, send_topic, topic, inTopic, outTopic
client, q, data, send_topic, inTopic, outTopic = mqtt.Client(), queue.Queue(), None, None, None, None
apl = GUI()
Hello, me again. It is definitely wierd problem in mqtt paho and your functions. special to wait_for where blocks any kind of comunicatiom. I for now use you untouched code and i fill it with GUI from that prints you can see where it stuck. I will upgrade graphics, but for now doesnt work intention of this whole thing. I see that you prepare some nodes array, maybe for selective node update ? I will add popup windows where would be tickboxes how many we have bssid retain messages on broker. Please look at it and write some notes or decision or whatever :) Thank you
I won't have time to actually try it for a little while, but I don't think using the sendbin function is a great way to do this for an interactive gui. really the send_firmware should either be asynchronous, or at least be sent to its own thread and then use a queue to send messages back to the GUI about progress.