From c18490cc4aca69327b275260966fa6c113aab60f Mon Sep 17 00:00:00 2001 From: Max Prokhorov Date: Wed, 16 Oct 2019 14:41:43 +0300 Subject: [PATCH] PIO: pass APP_REVISION via extra scripts (#1946) * utils/version: use git description token as version, add memoization * pio: move extra scripts to a separate directory * pio: add -DAPP_REVISION=... as local build flag --- .gitignore | 2 + code/build.sh | 3 - code/espurna/utils.ino | 41 +++-- code/extra_scripts.py | 171 ------------------ code/platformio.ini | 2 +- code/scripts/espurna_utils/__init__.py | 6 + code/scripts/espurna_utils/checks.py | 29 +++ code/scripts/espurna_utils/display.py | 40 ++++ code/scripts/espurna_utils/float_support.py | 8 + code/scripts/espurna_utils/git.py | 23 +++ code/scripts/espurna_utils/ldscripts.py | 24 +++ code/scripts/espurna_utils/lwip.py | 56 ++++++ code/scripts/espurna_utils/postmortem.py | 13 ++ code/scripts/pio_main.py | 38 ++++ .../pio_pre.py} | 10 +- 15 files changed, 275 insertions(+), 191 deletions(-) delete mode 100644 code/extra_scripts.py create mode 100644 code/scripts/espurna_utils/__init__.py create mode 100644 code/scripts/espurna_utils/checks.py create mode 100644 code/scripts/espurna_utils/display.py create mode 100644 code/scripts/espurna_utils/float_support.py create mode 100644 code/scripts/espurna_utils/git.py create mode 100644 code/scripts/espurna_utils/ldscripts.py create mode 100644 code/scripts/espurna_utils/lwip.py create mode 100644 code/scripts/espurna_utils/postmortem.py create mode 100644 code/scripts/pio_main.py rename code/{extra_script_pre.py => scripts/pio_pre.py} (91%) diff --git a/.gitignore b/.gitignore index e61c68af..45820bab 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,5 @@ custom.h .env .DS_Store .vscode +_pycache_/ +*.py[cod] diff --git a/code/build.sh b/code/build.sh index e7447465..ec1505cb 100755 --- a/code/build.sh +++ b/code/build.sh @@ -61,9 +61,6 @@ list_envs() { travis=$(list_envs | grep travis | sort) available=$(list_envs | grep -Ev -- '-ota$|-ssl$|^travis' | sort) -# Build tools settings -export PLATFORMIO_BUILD_FLAGS="${PLATFORMIO_BUILD_FLAGS} -DAPP_REVISION='\"$git_revision\"'" - # Functions print_available() { echo "--------------------------------------------------------------" diff --git a/code/espurna/utils.ino b/code/espurna/utils.ino index e95d7d75..688dd23f 100644 --- a/code/espurna/utils.ino +++ b/code/espurna/utils.ino @@ -37,23 +37,36 @@ String getAdminPass() { return getSetting("adminPass", ADMIN_PASS); } -String getCoreVersion() { - String version = ESP.getCoreVersion(); - #ifdef ARDUINO_ESP8266_RELEASE - if (version.equals("00000000")) { - version = String(ARDUINO_ESP8266_RELEASE); - } - #endif - version.replace("_", "."); +const String& getCoreVersion() { + static String version; + if (!version.length()) { + #ifdef ARDUINO_ESP8266_RELEASE + version = ESP.getCoreVersion(); + if (version.equals("00000000")) { + version = String(ARDUINO_ESP8266_RELEASE); + } + version.replace("_", "."); + #else + #define _GET_COREVERSION_STR(X) #X + #define GET_COREVERSION_STR(X) _GET_COREVERSION_STR(X) + version = GET_COREVERSION_STR(ARDUINO_ESP8266_GIT_DESC); + #undef _GET_COREVERSION_STR + #undef GET_COREVERSION_STR + #endif + } return version; } -String getCoreRevision() { - #ifdef ARDUINO_ESP8266_GIT_VER - return String(ARDUINO_ESP8266_GIT_VER, 16); - #else - return String(""); - #endif +const String& getCoreRevision() { + static String revision; + if (!revision.length()) { + #ifdef ARDUINO_ESP8266_GIT_VER + revision = String(ARDUINO_ESP8266_GIT_VER, 16); + #else + revision = ""; + #endif + } + return revision; } unsigned char getHeartbeatMode() { diff --git a/code/extra_scripts.py b/code/extra_scripts.py deleted file mode 100644 index eaca0e7a..00000000 --- a/code/extra_scripts.py +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/env python -from __future__ import print_function - -import os -import sys -from subprocess import call -import click - -Import("env", "projenv") - -PIO_PLATFORM = env.PioPlatform() -FRAMEWORK_DIR = PIO_PLATFORM.get_package_dir("framework-arduinoespressif8266") - -# ------------------------------------------------------------------------------ -# Utils -# ------------------------------------------------------------------------------ - -class Color(object): - BLACK = '\x1b[1;30m' - RED = '\x1b[1;31m' - GREEN = '\x1b[1;32m' - YELLOW = '\x1b[1;33m' - BLUE = '\x1b[1;34m' - MAGENTA = '\x1b[1;35m' - CYAN = '\x1b[1;36m' - WHITE = '\x1b[1;37m' - LIGHT_GREY = '\x1b[0;30m' - LIGHT_RED = '\x1b[0;31m' - LIGHT_GREEN = '\x1b[0;32m' - LIGHT_YELLOW = '\x1b[0;33m' - LIGHT_BLUE = '\x1b[0;34m' - LIGHT_MAGENTA = '\x1b[0;35m' - LIGHT_CYAN = '\x1b[0;36m' - LIGHT_WHITE = '\x1b[0;37m' - -def clr(color, text): - return color + str(text) + '\x1b[0m' - -def print_warning(message, color=Color.LIGHT_YELLOW): - print(clr(color, message), file=sys.stderr) - -def print_filler(fill, color=Color.WHITE, err=False): - width, _ = click.get_terminal_size() - if len(fill) > 1: - fill = fill[0] - - out = sys.stderr if err else sys.stdout - print(clr(color, fill * width), file=out) - -def ldscript_inject_libpath(): - - # espressif8266@1.5.0 did not append this directory into the LIBPATH - libpath_sdk = os.path.join(FRAMEWORK_DIR, "tools", "sdk", "ld") - env.Append(LIBPATH=[libpath_sdk]) - - libpath_base = os.path.join("$PROJECT_DIR", "..", "dist", "ld") - env.Append(LIBPATH=[ - os.path.join(libpath_base, "pre_2.5.0") - ]) - - # local.eagle.app.v6.common.ld exists only with Core >2.5.0 - def check_local_ld(target ,source, env): - local_ld = env.subst(os.path.join("$BUILD_DIR", "ld", "local.eagle.app.v6.common.ld")) - if os.path.exists(local_ld): - env.Prepend(LIBPATH=[ - os.path.join(libpath_base, "latest") - ]) - - env.AddPreAction( - os.path.join("$BUILD_DIR", "firmware.elf"), - check_local_ld - ) - - -# ------------------------------------------------------------------------------ -# Callbacks -# ------------------------------------------------------------------------------ - -def remove_float_support(): - - flags = " ".join(env['LINKFLAGS']) - flags = flags.replace("-u _printf_float", "") - flags = flags.replace("-u _scanf_float", "") - newflags = flags.split() - - env.Replace( - LINKFLAGS = newflags - ) - -def cpp_check(target, source, env): - print("Started cppcheck...\n") - call(["cppcheck", os.getcwd()+"/espurna", "--force", "--enable=all"]) - print("Finished cppcheck...\n") - -def check_size(target, source, env): - (binary,) = target - path = binary.get_abspath() - size = os.stat(path).st_size - print(clr(Color.LIGHT_BLUE, "Binary size: {} bytes".format(size))) - - # Warn 1MB variants about exceeding OTA size limit - flash_size = int(env.BoardConfig().get("upload.maximum_size", 0)) - if (flash_size == 1048576) and (size >= 512000): - print_filler("*", color=Color.LIGHT_YELLOW, err=True) - print_warning("File is too large for OTA! Here you can find instructions on how to flash it:") - print_warning("https://github.com/xoseperez/espurna/wiki/TwoStepUpdates", color=Color.LIGHT_CYAN) - print_filler("*", color=Color.LIGHT_YELLOW, err=True) - -def dummy_ets_printf(target, source, env): - (postmortem_src_file, ) = source - (postmortem_obj_file, ) = target - - cmd = ["xtensa-lx106-elf-objcopy"] - - # recent Core switched to cpp+newlib & ets_printf_P - cmd.extend(["--redefine-sym", "ets_printf=dummy_ets_printf"]) - cmd.extend(["--redefine-sym", "ets_printf_P=dummy_ets_printf"]) - - cmd.append(postmortem_obj_file.get_abspath()) - env.Execute(env.VerboseAction(" ".join(cmd), "Removing ets_printf / ets_printf_P")) - env.Depends(postmortem_obj_file,"$BUILD_DIR/src/dummy_ets_printf.c.o") - -def patch_lwip(): - # ignore when building with lwip2 - if "lwip_gcc" not in env["LIBS"]: - return - - toolchain_prefix = os.path.join(PIO_PLATFORM.get_package_dir("toolchain-xtensa"), "bin", "xtensa-lx106-elf-") - - patch_action = env.VerboseAction(" ".join([ - "-patch", "-u", "-N", "-d", - os.path.join(FRAMEWORK_DIR, "tools", "sdk", "lwip"), - os.path.join("src", "core", "tcp_out.c"), - env.subst(os.path.join("$PROJECT_DIR", "..", "dist", "patches", "lwip_mtu_issue_1610.patch")) - ]), "Patching lwip source") - build_action = env.VerboseAction(" ".join([ - "make", "-C", os.path.join(FRAMEWORK_DIR, "tools", "sdk", "lwip", "src"), - "install", - "TOOLS_PATH={}".format(toolchain_prefix), - "LWIP_LIB=liblwip_gcc.a" - ]), "Rebuilding lwip") - - patcher = env.Alias("patch-lwip", None, patch_action) - builder = env.Alias("build-lwip", patcher, build_action) - if os.environ.get("ESPURNA_PIO_PATCH_ISSUE_1610"): - env.Depends("$BUILD_DIR/${PROGNAME}.elf", builder) - env.AlwaysBuild(patcher) - env.AlwaysBuild(builder) - -# ------------------------------------------------------------------------------ -# Hooks -# ------------------------------------------------------------------------------ - -# Always show warnings for project code -projenv.ProcessUnFlags("-w") - -# 2.4.0 and up -remove_float_support() -ldscript_inject_libpath() - -# two-step update hint when using 1MB boards -env.AddPostAction("$BUILD_DIR/${PROGNAME}.bin", check_size) - -# disable postmortem printing to the uart. another one is in eboot, but this is what causes the most harm -if "DISABLE_POSTMORTEM_STACKDUMP" in env["CPPFLAGS"]: - env.AddPostAction("$BUILD_DIR/FrameworkArduino/core_esp8266_postmortem.c.o", dummy_ets_printf) - env.AddPostAction("$BUILD_DIR/FrameworkArduino/core_esp8266_postmortem.cpp.o", dummy_ets_printf) - -# patch lwip1 sources conditionally: -# https://github.com/xoseperez/espurna/issues/1610 -patch_lwip() diff --git a/code/platformio.ini b/code/platformio.ini index 8d8965f9..521219f6 100644 --- a/code/platformio.ini +++ b/code/platformio.ini @@ -90,7 +90,7 @@ framework = arduino board_build.flash_mode = dout monitor_speed = 115200 upload_speed = 115200 -extra_scripts = pre:extra_script_pre.py, extra_scripts.py +extra_scripts = pre:scripts/pio_pre.py, scripts/pio_main.py lib_extra_dirs = ${common.shared_libdeps_dir} diff --git a/code/scripts/espurna_utils/__init__.py b/code/scripts/espurna_utils/__init__.py new file mode 100644 index 00000000..62387a3c --- /dev/null +++ b/code/scripts/espurna_utils/__init__.py @@ -0,0 +1,6 @@ +from .checks import check_cppcheck, check_printsize +from .float_support import remove_float_support +from .ldscripts import ldscripts_inject_libpath +from .lwip import lwip_inject_patcher +from .postmortem import dummy_ets_printf +from .git import app_inject_revision diff --git a/code/scripts/espurna_utils/checks.py b/code/scripts/espurna_utils/checks.py new file mode 100644 index 00000000..7aaf9cfd --- /dev/null +++ b/code/scripts/espurna_utils/checks.py @@ -0,0 +1,29 @@ +import os + +from .display import Color, clr, print_filler, print_warning + + +def check_printsize(target, source, env): + (binary,) = target + path = binary.get_abspath() + size = os.stat(path).st_size + print(clr(Color.LIGHT_BLUE, "Binary size: {} bytes".format(size))) + + # Warn 1MB variants about exceeding OTA size limit + flash_size = int(env.BoardConfig().get("upload.maximum_size", 0)) + if (flash_size == 1048576) and (size >= 512000): + print_filler("*", color=Color.LIGHT_YELLOW, err=True) + print_warning( + "File is too large for OTA! Here you can find instructions on how to flash it:" + ) + print_warning( + "https://github.com/xoseperez/espurna/wiki/TwoStepUpdates", + color=Color.LIGHT_CYAN, + ) + print_filler("*", color=Color.LIGHT_YELLOW, err=True) + + +def check_cppcheck(target, source, env): + print_warning("Started cppcheck...\n") + call(["cppcheck", os.getcwd() + "/espurna", "--force", "--enable=all"]) + print_warning("Finished cppcheck...\n") diff --git a/code/scripts/espurna_utils/display.py b/code/scripts/espurna_utils/display.py new file mode 100644 index 00000000..0628f94b --- /dev/null +++ b/code/scripts/espurna_utils/display.py @@ -0,0 +1,40 @@ +from __future__ import print_function + +import sys +import click + + +class Color(object): + BLACK = "\x1b[1;30m" + RED = "\x1b[1;31m" + GREEN = "\x1b[1;32m" + YELLOW = "\x1b[1;33m" + BLUE = "\x1b[1;34m" + MAGENTA = "\x1b[1;35m" + CYAN = "\x1b[1;36m" + WHITE = "\x1b[1;37m" + LIGHT_GREY = "\x1b[0;30m" + LIGHT_RED = "\x1b[0;31m" + LIGHT_GREEN = "\x1b[0;32m" + LIGHT_YELLOW = "\x1b[0;33m" + LIGHT_BLUE = "\x1b[0;34m" + LIGHT_MAGENTA = "\x1b[0;35m" + LIGHT_CYAN = "\x1b[0;36m" + LIGHT_WHITE = "\x1b[0;37m" + + +def clr(color, text): + return color + str(text) + "\x1b[0m" + + +def print_warning(message, color=Color.LIGHT_YELLOW): + print(clr(color, message), file=sys.stderr) + + +def print_filler(fill, color=Color.WHITE, err=False): + width, _ = click.get_terminal_size() + if len(fill) > 1: + fill = fill[0] + + out = sys.stderr if err else sys.stdout + print(clr(color, fill * width), file=out) diff --git a/code/scripts/espurna_utils/float_support.py b/code/scripts/espurna_utils/float_support.py new file mode 100644 index 00000000..f53b49d2 --- /dev/null +++ b/code/scripts/espurna_utils/float_support.py @@ -0,0 +1,8 @@ +def remove_float_support(env): + + flags = " ".join(env["LINKFLAGS"]) + flags = flags.replace("-u _printf_float", "") + flags = flags.replace("-u _scanf_float", "") + newflags = flags.split() + + env.Replace(LINKFLAGS=newflags) diff --git a/code/scripts/espurna_utils/git.py b/code/scripts/espurna_utils/git.py new file mode 100644 index 00000000..91067f57 --- /dev/null +++ b/code/scripts/espurna_utils/git.py @@ -0,0 +1,23 @@ +import os +import subprocess + +def git(*args): + cmd = ["git"] + cmd.extend(args) + proc = subprocess.Popen( + cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True + ) + return proc.stdout.readlines()[0].strip() + +def app_inject_revision(env): + revision = "" + try: + revision = "\\\"{}\\\"".format(git("rev-parse", "--short=8", "HEAD")) + except: # pylint: disable=broad-except + pass + + # Note: code expects this as undefined when empty + if revision: + env.Append(CPPDEFINES=[ + ("APP_REVISION", revision) + ]) diff --git a/code/scripts/espurna_utils/ldscripts.py b/code/scripts/espurna_utils/ldscripts.py new file mode 100644 index 00000000..0b2f5176 --- /dev/null +++ b/code/scripts/espurna_utils/ldscripts.py @@ -0,0 +1,24 @@ +import os + + +def ldscripts_inject_libpath(env): + + platform = env.PioPlatform() + framework_dir = platform.get_package_dir("framework-arduinoespressif8266") + + # espressif8266@1.5.0 did not append this directory into the LIBPATH + libpath_sdk = os.path.join(framework_dir, "tools", "sdk", "ld") + env.Append(LIBPATH=[libpath_sdk]) + + libpath_base = os.path.join("$PROJECT_DIR", "..", "dist", "ld") + env.Append(LIBPATH=[os.path.join(libpath_base, "pre_2.5.0")]) + + # local.eagle.app.v6.common.ld exists only with Core >2.5.0 + def check_local_ld(target, source, env): + local_ld = env.subst( + os.path.join("$BUILD_DIR", "ld", "local.eagle.app.v6.common.ld") + ) + if os.path.exists(local_ld): + env.Prepend(LIBPATH=[os.path.join(libpath_base, "latest")]) + + env.AddPreAction(os.path.join("$BUILD_DIR", "firmware.elf"), check_local_ld) diff --git a/code/scripts/espurna_utils/lwip.py b/code/scripts/espurna_utils/lwip.py new file mode 100644 index 00000000..a742fcd7 --- /dev/null +++ b/code/scripts/espurna_utils/lwip.py @@ -0,0 +1,56 @@ +import os + + +def lwip_inject_patcher(env): + # ignore when building with lwip2 + if "lwip_gcc" not in env["LIBS"]: + return + + platform = env.PioPlatform() + framework_dir = platform.get_package_dir("framework-arduinoespressif8266") + toolchain_prefix = os.path.join( + platform.get_package_dir("toolchain-xtensa"), "bin", "xtensa-lx106-elf-" + ) + + patch_action = env.VerboseAction( + " ".join( + [ + "-patch", + "-u", + "-N", + "-d", + os.path.join(framework_dir, "tools", "sdk", "lwip"), + os.path.join("src", "core", "tcp_out.c"), + env.subst( + os.path.join( + "$PROJECT_DIR", + "..", + "dist", + "patches", + "lwip_mtu_issue_1610.patch", + ) + ), + ] + ), + "Patching lwip source", + ) + build_action = env.VerboseAction( + " ".join( + [ + "make", + "-C", + os.path.join(framework_dir, "tools", "sdk", "lwip", "src"), + "install", + "TOOLS_PATH={}".format(toolchain_prefix), + "LWIP_LIB=liblwip_gcc.a", + ] + ), + "Rebuilding lwip", + ) + + patcher = env.Alias("patch-lwip", None, patch_action) + builder = env.Alias("build-lwip", patcher, build_action) + if os.environ.get("ESPURNA_PIO_PATCH_ISSUE_1610"): + env.Depends("$BUILD_DIR/${PROGNAME}.elf", builder) + env.AlwaysBuild(patcher) + env.AlwaysBuild(builder) diff --git a/code/scripts/espurna_utils/postmortem.py b/code/scripts/espurna_utils/postmortem.py new file mode 100644 index 00000000..0422f14c --- /dev/null +++ b/code/scripts/espurna_utils/postmortem.py @@ -0,0 +1,13 @@ +def dummy_ets_printf(target, source, env): + (postmortem_src_file,) = source + (postmortem_obj_file,) = target + + cmd = ["xtensa-lx106-elf-objcopy"] + + # recent Core switched to cpp+newlib & ets_printf_P + cmd.extend(["--redefine-sym", "ets_printf=dummy_ets_printf"]) + cmd.extend(["--redefine-sym", "ets_printf_P=dummy_ets_printf"]) + + cmd.append(postmortem_obj_file.get_abspath()) + env.Execute(env.VerboseAction(" ".join(cmd), "Removing ets_printf / ets_printf_P")) + env.Depends(postmortem_obj_file, "$BUILD_DIR/src/dummy_ets_printf.c.o") diff --git a/code/scripts/pio_main.py b/code/scripts/pio_main.py new file mode 100644 index 00000000..313bc8c3 --- /dev/null +++ b/code/scripts/pio_main.py @@ -0,0 +1,38 @@ +# Run this script every time building an env: + +from espurna_utils import ( + check_printsize, + remove_float_support, + ldscripts_inject_libpath, + lwip_inject_patcher, + app_inject_revision, + dummy_ets_printf +) + +Import("env", "projenv") + +# Always show warnings for project code +projenv.ProcessUnFlags("-w") + +# 2.4.0 and up +remove_float_support(env) +ldscripts_inject_libpath(env) + +# two-step update hint when using 1MB boards +env.AddPostAction("$BUILD_DIR/${PROGNAME}.bin", check_printsize) + +# disable postmortem printing to the uart. another one is in eboot, but this is what causes the most harm +if "DISABLE_POSTMORTEM_STACKDUMP" in env["CPPFLAGS"]: + env.AddPostAction( + "$BUILD_DIR/FrameworkArduino/core_esp8266_postmortem.c.o", dummy_ets_printf + ) + env.AddPostAction( + "$BUILD_DIR/FrameworkArduino/core_esp8266_postmortem.cpp.o", dummy_ets_printf + ) + +# patch lwip1 sources conditionally: +# https://github.com/xoseperez/espurna/issues/1610 +lwip_inject_patcher(env) + +# when using git, add -DAPP_REVISION=(git-commit-hash) +app_inject_revision(projenv) diff --git a/code/extra_script_pre.py b/code/scripts/pio_pre.py similarity index 91% rename from code/extra_script_pre.py rename to code/scripts/pio_pre.py index 04818885..adcdc13b 100644 --- a/code/extra_script_pre.py +++ b/code/scripts/pio_pre.py @@ -71,11 +71,17 @@ def ensure_platform_updated(): print("updating platform packages", file=sys.stderr) PIO_PLATFORM.update_packages() except Exception: - print("Warning: no connection, cannot check for outdated packages", file=sys.stderr) + print( + "Warning: no connection, cannot check for outdated packages", + file=sys.stderr, + ) + # latest toolchain is still optional with PIO (TODO: recheck after 2.6.0!) # also updates arduino core git to the latest master commit -if TRAVIS and (env.GetProjectOption("platform") == CONFIG.get("common", "arduino_core_git")): +if TRAVIS and ( + env.GetProjectOption("platform") == CONFIG.get("common", "arduino_core_git") +): ensure_platform_updated() # to speed-up build process, install libraries in either global or local shared storage