mh
/
mhsw-espurna
1
0
Fork 0
Code Releases 0 Activity
Browse Source

Scripts: memanalyser fixes (#2014) 

* scripts: memanalyzer fixes

- cleanup duplication
- cleanup external dependencies
- properly use subprocess when calling pio, add debug mode
- cleanup argparse usage, use store_true actions instead of counters
- use python3 syntax and python3 shebang by default
- fix some issues with module dependencies

* move to scripts

* use simple return

* quick fix for rfm

* codacy unused import

* formatting

* show defaults

* while at it, rewrite to make pylint happy

* fix unused

* add windows support + note
master
Max Prokhorov 5 years ago
committed by GitHub
parent
960b32cd93
commit
0b5d200de1
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 427 additions and 310 deletions
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. +0
    -1
      code/espurna/config/arduino.h
  2. +5
    -0
      code/espurna/config/dependencies.h
  3. +6
    -2
      code/espurna/i2c.ino
  4. +2
    -4
      code/espurna/sensor.ino
  5. +0
    -303
      code/memanalyzer.py
  6. +414
    -0
      code/scripts/memanalyzer.py

+ 0
- 1
code/espurna/config/arduino.h View File

@ -174,7 +174,6 @@
//#define RFM69_SUPPORT 1
//#define RPN_RULES_SUPPORT 0
//#define SCHEDULER_SUPPORT 0
//#define SENSOR_SUPPORT 1
//#define SPIFFS_SUPPORT 1
//#define SSDP_SUPPORT 1
//#define TELNET_SUPPORT 0


+ 5
- 0
code/espurna/config/dependencies.h View File

@ -46,8 +46,13 @@
#if RPN_RULES_SUPPORT
#undef BROKER_SUPPORT
#define BROKER_SUPPORT 1 // If RPN Rules enabled enable BROKER
#undef WEB_SUPPORT
#define WEB_SUPPORT 1
#undef MQTT_SUPPORT
#define MQTT_SUPPORT 1
#endif
#if INFLUXDB_SUPPORT
#undef BROKER_SUPPORT
#define BROKER_SUPPORT 1 // If InfluxDB enabled enable BROKER


+ 6
- 2
code/espurna/i2c.ino View File

@ -351,7 +351,9 @@ void i2cScan() {
if (nDevices == 0) DEBUG_MSG_P(PSTR("[I2C] No devices found\n"));
}
void i2cCommands() {
#if TERMINAL_SUPPORT
void _i2cInitCommands() {
terminalRegisterCommand(F("I2C.SCAN"), [](Embedis* e) {
i2cScan();
@ -365,6 +367,8 @@ void i2cCommands() {
}
#endif // TERMINAL_SUPPORT
void i2cSetup() {
unsigned char sda = getSetting("i2cSDA", I2C_SDA_PIN).toInt();
@ -381,7 +385,7 @@ void i2cSetup() {
DEBUG_MSG_P(PSTR("[I2C] Using GPIO%u for SDA and GPIO%u for SCL\n"), sda, scl);
#if TERMINAL_SUPPORT
i2cCommands();
_i2cInitCommands();
#endif
#if I2C_CLEAR_BUS


+ 2
- 4
code/espurna/sensor.ino View File

@ -1543,11 +1543,9 @@ void _sensorReport(unsigned char index, double value) {
char buffer[64];
dtostrf(value, 1, decimals, buffer);
#if BROKER_SUPPORT
#if not BROKER_REAL_TIME
#if BROKER_SUPPORT && (not BROKER_REAL_TIME)
SensorBroker::Publish(magnitudeTopic(magnitude.type), magnitude.global, value, buffer);
#endif
#endif
#if MQTT_SUPPORT
@ -1803,7 +1801,7 @@ void sensorLoop() {
// -------------------------------------------------------------
value_show = _magnitudeProcess(magnitude.type, magnitude.decimals, value_raw);
#if BROKER_REAL_TIME
#if BROKER_SUPPORT && BROKER_REAL_TIME
{
char buffer[64];
dtostrf(value_show, 1-sizeof(buffer), magnitude.decimals, buffer);


+ 0
- 303
code/memanalyzer.py View File

@ -1,303 +0,0 @@
#!/usr/bin/env python
# coding=utf-8
# -------------------------------------------------------------------------------
# ESPurna module memory analyser
# xose.perez@gmail.com
#
# Based on:
# https://github.com/letscontrolit/ESPEasy/blob/mega/memanalyzer.py
# by psy0rz <edwin@datux.nl>
# https://raw.githubusercontent.com/SmingHub/Sming/develop/tools/memanalyzer.py
# by Slavey Karadzhov <slav@attachix.com>
# https://github.com/Sermus/ESP8266_memory_analyzer
# by Andrey Filimonov
#
# -------------------------------------------------------------------------------
from __future__ import print_function
import argparse
import os
import re
import shlex
import subprocess
import sys
from collections import OrderedDict
from sortedcontainers import SortedDict
if sys.version_info > (3, 0):
from subprocess import getstatusoutput
else:
from commands import getstatusoutput
# -------------------------------------------------------------------------------
TOTAL_IRAM = 32786
TOTAL_DRAM = 81920
env = "esp8266-4m-ota"
objdump_binary = "xtensa-lx106-elf-objdump"
sections = OrderedDict([
("data", "Initialized Data (RAM)"),
("rodata", "ReadOnly Data (RAM)"),
("bss", "Uninitialized Data (RAM)"),
("text", "Cached Code (IRAM)"),
("irom0_text", "Uncached Code (SPI)")
])
description = "ESPurna Memory Analyzer v0.1"
# -------------------------------------------------------------------------------
def file_size(file):
try:
return os.stat(file).st_size
except OSError:
return 0
def analyse_memory(elf_file):
command = "{} -t '{}'".format(objdump_binary, elf_file)
response = subprocess.check_output(shlex.split(command))
if isinstance(response, bytes):
response = response.decode('utf-8')
lines = response.split('\n')
# print("{0: >10}|{1: >30}|{2: >12}|{3: >12}|{4: >8}".format("Section", "Description", "Start (hex)", "End (hex)", "Used space"));
# print("------------------------------------------------------------------------------");
ret = {}
for (id_, _) in list(sections.items()):
section_start_token = " _{}_start".format(id_)
section_end_token = " _{}_end".format(id_)
section_start = -1
section_end = -1
for line in lines:
if section_start_token in line:
data = line.split(' ')
section_start = int(data[0], 16)
if section_end_token in line:
data = line.split(' ')
section_end = int(data[0], 16)
if section_start != -1 and section_end != -1:
break
section_length = section_end - section_start
# if i < 3:
# usedRAM += section_length
# if i == 3:
# usedIRAM = TOTAL_IRAM - section_length;
ret[id_] = section_length
# print("{0: >10}|{1: >30}|{2:12X}|{3:12X}|{4:8}".format(id_, descr, section_start, section_end, section_length))
# i += 1
# print("Total Used RAM : {:d}".format(usedRAM))
# print("Free RAM : {:d}".format(TOTAL_DRAM - usedRAM))
# print("Free IRam : {:d}".format(usedIRAM))
return ret
def run(env_, modules_):
flags = ""
for k, v in modules_.items():
flags += "-D{}_SUPPORT={:d} ".format(k, v)
os_env = os.environ.copy()
os_env["ESPURNA_BOARD"] = "WEMOS_D1_MINI_RELAYSHIELD"
os_env["ESPURNA_FLAGS"] = flags
os_env["ESPURNA_PIO_SHARED_LIBRARIES"] = "y"
command = "platformio run --silent --environment {} 2>/dev/null".format(env_)
subprocess.check_call(command, shell=True)
def calc_free(module):
free = 80 * 1024 - module['data'] - module['rodata'] - module['bss']
free = free + (16 - free % 16)
module['free'] = free
def modules_get():
modules_ = SortedDict()
for line in open("espurna/config/arduino.h"):
m = re.search(r'(\w*)_SUPPORT', line)
if m:
modules_[m.group(1)] = 0
del modules_['LLMNR']
del modules_['NETBIOS']
return modules_
if __name__ == '__main__':
# Parse command line options
parser = argparse.ArgumentParser(description=description)
parser.add_argument("modules", nargs='*', help="Modules to test (use ALL to test them all)")
parser.add_argument("-c", "--core", help="use core as base configuration instead of default", default=0, action='count')
parser.add_argument("-l", "--list", help="list available modules", default=0, action='count')
args = parser.parse_args()
# Hello
print()
print(description)
print()
# Check xtensa-lx106-elf-objdump is in the path
status, result = getstatusoutput(objdump_binary)
if status != 2 and status != 512:
print("xtensa-lx106-elf-objdump not found, please check it is in your PATH")
sys.exit(1)
# Load list of all modules
available_modules = modules_get()
if args.list > 0:
print("List of available modules:\n")
for key, value in available_modules.items():
print("* " + key)
print()
sys.exit(0)
# Which modules to test?
test_modules = []
if len(args.modules) > 0:
if "ALL" in args.modules:
test_modules = available_modules.keys()
else:
test_modules = args.modules
# Check test modules exist
for module in test_modules:
if module not in available_modules:
print("Module {} not found".format(module))
sys.exit(2)
# Define base configuration
if args.core == 0:
modules = SortedDict()
for m in test_modules:
modules[m] = 0
else:
modules = available_modules
# Show init message
if len(test_modules) > 0:
print("Analyzing module(s) {} on top of {} configuration\n".format(", ".join(test_modules), "CORE" if args.core > 0 else "DEFAULT"))
else:
print("Analyzing {} configuration\n".format("CORE" if args.core > 0 else "DEFAULT"))
output_format = "{:<20}|{:<15}|{:<15}|{:<15}|{:<15}|{:<15}|{:<15}|{:<15}"
print(output_format.format(
"Module",
"Cache IRAM",
"Init RAM",
"R.O. RAM",
"Uninit RAM",
"Available RAM",
"Flash ROM",
"Binary size"
))
print(output_format.replace("<", ">").format(
"",
".text",
".data",
".rodata",
".bss",
"heap + stack",
".irom0.text",
""
))
print(output_format.format(
"-" * 20,
"-" * 15,
"-" * 15,
"-" * 15,
"-" * 15,
"-" * 15,
"-" * 15,
"-" * 15
))
# Build the core without modules to get base memory usage
run(env, modules)
base = analyse_memory(".pio/build/{}/firmware.elf".format(env))
base['size'] = file_size(".pio/build/{}/firmware.bin".format(env))
calc_free(base)
print(output_format.format(
"CORE" if args.core == 1 else "DEFAULT",
base['text'],
base['data'],
base['rodata'],
base['bss'],
base['free'],
base['irom0_text'],
base['size'],
))
# Test each module
results = {}
for module in test_modules:
modules[module] = 1
run(env, modules)
results[module] = analyse_memory(".pio/build/{}/firmware.elf".format(env))
results[module]['size'] = file_size(".pio/build/{}/firmware.bin".format(env))
calc_free(results[module])
modules[module] = 0
print(output_format.format(
module,
results[module]['text'] - base['text'],
results[module]['data'] - base['data'],
results[module]['rodata'] - base['rodata'],
results[module]['bss'] - base['bss'],
results[module]['free'] - base['free'],
results[module]['irom0_text'] - base['irom0_text'],
results[module]['size'] - base['size'],
))
# Test all modules
if len(test_modules) > 0:
for module in test_modules:
modules[module] = 1
run(env, modules)
total = analyse_memory(".pio/build/{}/firmware.elf".format(env))
total['size'] = file_size(".pio/build/{}/firmware.bin".format(env))
calc_free(total)
print(output_format.format(
"-" * 20,
"-" * 15,
"-" * 15,
"-" * 15,
"-" * 15,
"-" * 15,
"-" * 15,
"-" * 15
))
if len(test_modules) > 1:
print(output_format.format(
"ALL MODULES",
total['text'] - base['text'],
total['data'] - base['data'],
total['rodata'] - base['rodata'],
total['bss'] - base['bss'],
total['free'] - base['free'],
total['irom0_text'] - base['irom0_text'],
total['size'] - base['size'],
))
print(output_format.format(
"TOTAL",
total['text'],
total['data'],
total['rodata'],
total['bss'],
total['free'],
total['irom0_text'],
total['size'],
))
print("\n")

+ 414
- 0
code/scripts/memanalyzer.py View File

@ -0,0 +1,414 @@
#!/usr/bin/env python3
# pylint: disable=C0301,C0114,C0116,W0511
# coding=utf-8
# -------------------------------------------------------------------------------
# ESPurna module memory analyser
# xose.perez@gmail.com
#
# Rewritten for python-3 by Maxim Prokhorov
# prokhorov.max@outlook.com
#
# Based on:
# https://github.com/letscontrolit/ESPEasy/blob/mega/memanalyzer.py
# by psy0rz <edwin@datux.nl>
# https://raw.githubusercontent.com/SmingHub/Sming/develop/tools/memanalyzer.py
# by Slavey Karadzhov <slav@attachix.com>
# https://github.com/Sermus/ESP8266_memory_analyzer
# by Andrey Filimonov
#
# -------------------------------------------------------------------------------
#
# When using Windows with non-default installation at the C:\.platformio,
# you would need to specify toolchain path manually. For example:
#
# $ py -3 scripts\memanalyzer.py --toolchain-prefix C:\.platformio\packages\toolchain-xtensa\bin\xtensa-lx106-elf- <args>
#
# You could also change the path to platformio binary in a similar fashion:
# $ py -3 scripts\memanalyzer.py --platformio-prefix C:\Users\Max\platformio-penv\Scripts\
#
# -------------------------------------------------------------------------------
from __future__ import print_function
import argparse
import os
import re
import subprocess
import sys
from collections import OrderedDict
from subprocess import getstatusoutput
__version__ = (0, 2)
# -------------------------------------------------------------------------------
TOTAL_IRAM = 32786
TOTAL_DRAM = 81920
DEFAULT_ENV = "nodemcu-lolin"
OBJDUMP_PREFIX = "~/.platformio/packages/toolchain-xtensa/bin/xtensa-lx106-elf-"
SECTIONS = OrderedDict(
[
("data", "Initialized Data (RAM)"),
("rodata", "ReadOnly Data (RAM)"),
("bss", "Uninitialized Data (RAM)"),
("text", "Cached Code (IRAM)"),
("irom0_text", "Uncached Code (SPI)"),
]
)
DESCRIPTION = "ESPurna Memory Analyzer v{}".format(
".".join(str(x) for x in __version__)
)
# -------------------------------------------------------------------------------
def objdump_path(prefix):
return "{}objdump".format(os.path.expanduser(prefix))
def file_size(file):
try:
return os.stat(file).st_size
except OSError:
return 0
def analyse_memory(elf_file, objdump):
proc = subprocess.Popen(
[objdump, "-t", elf_file], stdout=subprocess.PIPE, universal_newlines=True
)
lines = proc.stdout.readlines()
# print("{0: >10}|{1: >30}|{2: >12}|{3: >12}|{4: >8}".format("Section", "Description", "Start (hex)", "End (hex)", "Used space"));
# print("------------------------------------------------------------------------------");
ret = {}
for (id_, _) in list(SECTIONS.items()):
section_start_token = " _{}_start".format(id_)
section_end_token = " _{}_end".format(id_)
section_start = -1
section_end = -1
for line in lines:
line = line.strip()
if section_start_token in line:
data = line.split(" ")
section_start = int(data[0], 16)
if section_end_token in line:
data = line.split(" ")
section_end = int(data[0], 16)
if section_start != -1 and section_end != -1:
break
section_length = section_end - section_start
# if i < 3:
# usedRAM += section_length
# if i == 3:
# usedIRAM = TOTAL_IRAM - section_length;
ret[id_] = section_length
# print("{0: >10}|{1: >30}|{2:12X}|{3:12X}|{4:8}".format(id_, descr, section_start, section_end, section_length))
# i += 1
# print("Total Used RAM : {:d}".format(usedRAM))
# print("Free RAM : {:d}".format(TOTAL_DRAM - usedRAM))
# print("Free IRam : {:d}".format(usedIRAM))
return ret
def run(prefix, env, modules, debug):
flags = " ".join("-D{}_SUPPORT={:d}".format(k, v) for k, v in modules.items())
os_env = os.environ.copy()
os_env["PLATFORMIO_SRC_BUILD_FLAGS"] = flags
os_env["PLATFORMIO_BUILD_CACHE_DIR"] = "test/pio_cache"
os_env["ESPURNA_PIO_SHARED_LIBRARIES"] = "y"
command = [os.path.join(prefix, "platformio"), "run"]
if not debug:
command.append("--silent")
command.extend(["--environment", env])
output = None if debug else subprocess.DEVNULL
try:
subprocess.check_call(
command, shell=False, env=os_env, stdout=output, stderr=output
)
except subprocess.CalledProcessError:
print(" - Command failed: {}".format(command))
print(" - Selected flags: {}".format(flags))
sys.exit(1)
def get_available_modules():
modules = []
for line in open("espurna/config/arduino.h"):
match = re.search(r"(\w*)_SUPPORT", line)
if match:
modules.append((match.group(1), 0))
modules.sort(key=lambda item: item[0])
return OrderedDict(modules)
def parse_commandline_args():
parser = argparse.ArgumentParser(
description=DESCRIPTION, formatter_class=argparse.ArgumentDefaultsHelpFormatter
)
parser.add_argument(
"-e", "--environment", help="platformio envrionment to use", default=DEFAULT_ENV
)
parser.add_argument(
"--toolchain-prefix",
help="where to find the xtensa toolchain binaries",
default=OBJDUMP_PREFIX,
)
parser.add_argument(
"--platformio-prefix",
help="where to find the platformio executable",
default="",
)
parser.add_argument(
"-c",
"--core",
help="use core as base configuration instead of default",
action="store_true",
default=False,
)
parser.add_argument(
"-l",
"--list",
help="list available modules",
action="store_true",
default=False,
)
parser.add_argument("-d", "--debug", action="store_true", default=False)
parser.add_argument(
"modules", nargs="*", help="Modules to test (use ALL to test them all)"
)
return parser.parse_args()
def objdump_check(args):
status, _ = getstatusoutput(objdump_path(args.toolchain_prefix))
if status not in (2, 512):
print("objdump not found, please check that the --toolchain-prefix is correct")
sys.exit(1)
def get_modules(args):
# Load list of all modules
available_modules = get_available_modules()
if args.list:
print("List of available modules:\n")
for module in available_modules:
print("* " + module)
print()
sys.exit(0)
modules = []
if args.modules:
if "ALL" in args.modules:
modules.extend(available_modules.keys())
else:
modules.extend(args.modules)
modules.sort()
# Check test modules exist
for module in modules:
if module not in available_modules:
print("Module {} not found".format(module))
sys.exit(2)
# Either use all of modules or specified subset
if args.core:
modules = available_modules
else:
modules = OrderedDict((x, 0) for x in modules)
configuration = "CORE" if args.core else "DEFAULT"
return configuration, modules
# -------------------------------------------------------------------------------
class Analyser:
"""Run platformio and print info about the resulting binary."""
OUTPUT_FORMAT = "{:<20}|{:<15}|{:<15}|{:<15}|{:<15}|{:<15}|{:<15}|{:<15}"
ELF_FORMAT = ".pio/build/{env}/firmware.elf"
BIN_FORMAT = ".pio/build/{env}/firmware.bin"
class _Enable:
def __init__(self, analyser, module=None):
self.analyser = analyser
self.module = module
def __enter__(self):
if not self.module:
for name in self.analyser.modules:
self.analyser.modules[name] = 1
else:
self.analyser.modules[self.module] = 1
return self.analyser
def __exit__(self, *args, **kwargs):
if not self.module:
for name in self.analyser.modules:
self.analyser.modules[name] = 0
else:
self.analyser.modules[self.module] = 0
analyser = None
module = None
def __init__(self, args, modules):
self._debug = args.debug
self._platformio_prefix = args.platformio_prefix
self._toolchain_prefix = args.toolchain_prefix
self._environment = args.environment
self._elf_path = self.ELF_FORMAT.format(env=args.environment)
self._bin_path = self.BIN_FORMAT.format(env=args.environment)
self.modules = modules
self.baseline = None
def enable(self, module=None):
return self._Enable(self, module)
def print(self, *args):
print(self.OUTPUT_FORMAT.format(*args))
def print_delimiters(self):
self.print(
"-" * 20,
"-" * 15,
"-" * 15,
"-" * 15,
"-" * 15,
"-" * 15,
"-" * 15,
"-" * 15,
)
def begin(self, name):
self.print(
"Module",
"Cache IRAM",
"Init RAM",
"R.O. RAM",
"Uninit RAM",
"Available RAM",
"Flash ROM",
"Binary size",
)
self.print(
"", ".text", ".data", ".rodata", ".bss", "heap + stack", ".irom0.text", ""
)
self.print_delimiters()
self.baseline = self.run()
self.print_values(name, self.baseline)
def print_values(self, header, values):
self.print(
header,
values["text"],
values["data"],
values["rodata"],
values["bss"],
values["free"],
values["irom0_text"],
values["size"],
)
# TODO: sensor modules need to be print_compared with SENSOR as baseline
# TODO: some modules need to be print_compared with WEB as baseline
def print_compare(self, header, values):
self.print(
header,
values["text"] - self.baseline["text"],
values["data"] - self.baseline["data"],
values["rodata"] - self.baseline["rodata"],
values["bss"] - self.baseline["bss"],
values["free"] - self.baseline["free"],
values["irom0_text"] - self.baseline["irom0_text"],
values["size"] - self.baseline["size"],
)
def run(self):
run(self._platformio_prefix, self._environment, self.modules, self._debug)
values = analyse_memory(self._elf_path, objdump_path(self._toolchain_prefix))
free = 80 * 1024 - values["data"] - values["rodata"] - values["bss"]
free = free + (16 - free % 16)
values["free"] = free
values["size"] = file_size(self._bin_path)
return values
_debug = False
_platformio_prefix = None
_toolchain_prefix = None
_environment = None
_bin_path = None
_elf_path = None
modules = None
baseline = None
def main(args):
# Check xtensa-lx106-elf-objdump is in the path
objdump_check(args)
# Which modules to test?
configuration, modules = get_modules(args)
# print_values init message
print('Selected environment "{}"'.format(args.environment), end="")
if modules:
print(" with modules: {}".format(" ".join(modules.keys())))
else:
print()
print()
print("Analyzing {} configuration".format(configuration))
print()
# Build the core without any modules to get base memory usage
analyser = Analyser(args, modules)
analyser.begin(configuration)
# Test each module separately
results = {}
for module in analyser.modules:
with analyser.enable(module):
results[module] = analyser.run()
analyser.print_compare(module, results[module])
# Test all modules
if analyser.modules:
with analyser.enable():
total = analyser.run()
analyser.print_delimiters()
if len(analyser.modules) > 1:
analyser.print_compare("ALL MODULES", total)
analyser.print_values("TOTAL", total)
if __name__ == "__main__":
main(parse_commandline_args())

Write Preview
Loading…
Cancel
Save