PIO: pass APP_REVISION via extra scripts (#1946)

* utils/version: use git description token as version, add memoization

* pio: move extra scripts to a separate directory

* pio: add -DAPP_REVISION=... as local build flag

.gitignore

@@ -15,3 +15,5 @@ custom.h
 .env
 .DS_Store
 .vscode
+_pycache_/
+*.py[cod]

code/build.sh

@@ -61,9 +61,6 @@ list_envs() {
 travis=$(list_envs | grep travis | sort)
 available=$(list_envs | grep -Ev -- '-ota$|-ssl$|^travis' | sort)
 
-# Build tools settings
-export PLATFORMIO_BUILD_FLAGS="${PLATFORMIO_BUILD_FLAGS} -DAPP_REVISION='\"$git_revision\"'"
-
 # Functions
 print_available() {
     echo "--------------------------------------------------------------"

code/espurna/utils.ino

@@ -37,23 +37,36 @@ String getAdminPass() {
     return getSetting("adminPass", ADMIN_PASS);
 }
 
-String getCoreVersion() {
-    String version = ESP.getCoreVersion();
-    #ifdef ARDUINO_ESP8266_RELEASE
-        if (version.equals("00000000")) {
-            version = String(ARDUINO_ESP8266_RELEASE);
-        }
-    #endif
-    version.replace("_", ".");
+const String& getCoreVersion() {
+    static String version;
+    if (!version.length()) {
+        #ifdef ARDUINO_ESP8266_RELEASE
+            version = ESP.getCoreVersion();
+            if (version.equals("00000000")) {
+                version = String(ARDUINO_ESP8266_RELEASE);
+            }
+            version.replace("_", ".");
+        #else
+            #define _GET_COREVERSION_STR(X) #X
+            #define GET_COREVERSION_STR(X) _GET_COREVERSION_STR(X)
+            version = GET_COREVERSION_STR(ARDUINO_ESP8266_GIT_DESC);
+            #undef _GET_COREVERSION_STR
+            #undef GET_COREVERSION_STR
+        #endif
+    }
     return version;
 }
 
-String getCoreRevision() {
-    #ifdef ARDUINO_ESP8266_GIT_VER
-        return String(ARDUINO_ESP8266_GIT_VER, 16);
-    #else
-        return String("");
-    #endif
+const String& getCoreRevision() {
+    static String revision;
+    if (!revision.length()) {
+        #ifdef ARDUINO_ESP8266_GIT_VER
+            revision = String(ARDUINO_ESP8266_GIT_VER, 16);
+        #else
+            revision = "";
+        #endif
+    }
+    return revision;
 }
 
 unsigned char getHeartbeatMode() {

code/extra_scripts.py

@@ -1,171 +0,0 @@
-#!/usr/bin/env python
-from __future__ import print_function
-
-import os
-import sys
-from subprocess import call
-import click
-
-Import("env", "projenv")
-
-PIO_PLATFORM = env.PioPlatform()
-FRAMEWORK_DIR = PIO_PLATFORM.get_package_dir("framework-arduinoespressif8266")
-
-# ------------------------------------------------------------------------------
-# Utils
-# ------------------------------------------------------------------------------
-
-class Color(object):
-    BLACK = '\x1b[1;30m'
-    RED = '\x1b[1;31m'
-    GREEN = '\x1b[1;32m'
-    YELLOW = '\x1b[1;33m'
-    BLUE = '\x1b[1;34m'
-    MAGENTA = '\x1b[1;35m'
-    CYAN = '\x1b[1;36m'
-    WHITE = '\x1b[1;37m'
-    LIGHT_GREY = '\x1b[0;30m'
-    LIGHT_RED = '\x1b[0;31m'
-    LIGHT_GREEN = '\x1b[0;32m'
-    LIGHT_YELLOW = '\x1b[0;33m'
-    LIGHT_BLUE = '\x1b[0;34m'
-    LIGHT_MAGENTA = '\x1b[0;35m'
-    LIGHT_CYAN = '\x1b[0;36m'
-    LIGHT_WHITE = '\x1b[0;37m'
-
-def clr(color, text):
-    return color + str(text) + '\x1b[0m'
-
-def print_warning(message, color=Color.LIGHT_YELLOW):
-    print(clr(color, message), file=sys.stderr)
-
-def print_filler(fill, color=Color.WHITE, err=False):
-    width, _ = click.get_terminal_size()
-    if len(fill) > 1:
-        fill = fill[0]
-
-    out = sys.stderr if err else sys.stdout
-    print(clr(color, fill * width), file=out)
-
-def ldscript_inject_libpath():
-
-    # espressif8266@1.5.0 did not append this directory into the LIBPATH
-    libpath_sdk = os.path.join(FRAMEWORK_DIR, "tools", "sdk", "ld")
-    env.Append(LIBPATH=[libpath_sdk])
-
-    libpath_base = os.path.join("$PROJECT_DIR", "..", "dist", "ld")
-    env.Append(LIBPATH=[
-        os.path.join(libpath_base, "pre_2.5.0")
-    ])
-
-    # local.eagle.app.v6.common.ld exists only with Core >2.5.0
-    def check_local_ld(target ,source, env):
-        local_ld = env.subst(os.path.join("$BUILD_DIR", "ld", "local.eagle.app.v6.common.ld"))
-        if os.path.exists(local_ld):
-            env.Prepend(LIBPATH=[
-                os.path.join(libpath_base, "latest")
-            ])
-
-    env.AddPreAction(
-        os.path.join("$BUILD_DIR", "firmware.elf"),
-        check_local_ld
-    )
-
-
-# ------------------------------------------------------------------------------
-# Callbacks
-# ------------------------------------------------------------------------------
-
-def remove_float_support():
-
-    flags = " ".join(env['LINKFLAGS'])
-    flags = flags.replace("-u _printf_float", "")
-    flags = flags.replace("-u _scanf_float", "")
-    newflags = flags.split()
-
-    env.Replace(
-        LINKFLAGS = newflags
-    )
-
-def cpp_check(target, source, env):
-    print("Started cppcheck...\n")
-    call(["cppcheck", os.getcwd()+"/espurna", "--force", "--enable=all"])
-    print("Finished cppcheck...\n")
-
-def check_size(target, source, env):
-    (binary,) = target
-    path = binary.get_abspath()
-    size = os.stat(path).st_size
-    print(clr(Color.LIGHT_BLUE, "Binary size: {} bytes".format(size)))
-
-    # Warn 1MB variants about exceeding OTA size limit
-    flash_size = int(env.BoardConfig().get("upload.maximum_size", 0))
-    if (flash_size == 1048576) and (size >= 512000):
-        print_filler("*", color=Color.LIGHT_YELLOW, err=True)
-        print_warning("File is too large for OTA! Here you can find instructions on how to flash it:")
-        print_warning("https://github.com/xoseperez/espurna/wiki/TwoStepUpdates", color=Color.LIGHT_CYAN)
-        print_filler("*", color=Color.LIGHT_YELLOW, err=True)
-
-def dummy_ets_printf(target, source, env):
-    (postmortem_src_file, ) = source
-    (postmortem_obj_file, ) = target
-
-    cmd = ["xtensa-lx106-elf-objcopy"]
-
-    # recent Core switched to cpp+newlib & ets_printf_P
-    cmd.extend(["--redefine-sym", "ets_printf=dummy_ets_printf"])
-    cmd.extend(["--redefine-sym", "ets_printf_P=dummy_ets_printf"])
-
-    cmd.append(postmortem_obj_file.get_abspath())
-    env.Execute(env.VerboseAction(" ".join(cmd), "Removing ets_printf / ets_printf_P"))
-    env.Depends(postmortem_obj_file,"$BUILD_DIR/src/dummy_ets_printf.c.o")
-
-def patch_lwip():
-    # ignore when building with lwip2
-    if "lwip_gcc" not in env["LIBS"]:
-        return
-
-    toolchain_prefix = os.path.join(PIO_PLATFORM.get_package_dir("toolchain-xtensa"), "bin", "xtensa-lx106-elf-")
-
-    patch_action = env.VerboseAction(" ".join([
-        "-patch", "-u", "-N", "-d",
-        os.path.join(FRAMEWORK_DIR, "tools", "sdk", "lwip"),
-        os.path.join("src", "core", "tcp_out.c"),
-        env.subst(os.path.join("$PROJECT_DIR", "..", "dist", "patches", "lwip_mtu_issue_1610.patch"))
-    ]), "Patching lwip source")
-    build_action = env.VerboseAction(" ".join([
-        "make", "-C", os.path.join(FRAMEWORK_DIR, "tools", "sdk", "lwip", "src"),
-        "install",
-        "TOOLS_PATH={}".format(toolchain_prefix),
-        "LWIP_LIB=liblwip_gcc.a"
-    ]), "Rebuilding lwip")
-
-    patcher = env.Alias("patch-lwip", None, patch_action)
-    builder = env.Alias("build-lwip", patcher, build_action)
-    if os.environ.get("ESPURNA_PIO_PATCH_ISSUE_1610"):
-        env.Depends("$BUILD_DIR/${PROGNAME}.elf", builder)
-    env.AlwaysBuild(patcher)
-    env.AlwaysBuild(builder)
-
-# ------------------------------------------------------------------------------
-# Hooks
-# ------------------------------------------------------------------------------
-
-# Always show warnings for project code
-projenv.ProcessUnFlags("-w")
-
-# 2.4.0 and up
-remove_float_support()
-ldscript_inject_libpath()
-
-# two-step update hint when using 1MB boards
-env.AddPostAction("$BUILD_DIR/${PROGNAME}.bin", check_size)
-
-# disable postmortem printing to the uart. another one is in eboot, but this is what causes the most harm
-if "DISABLE_POSTMORTEM_STACKDUMP" in env["CPPFLAGS"]:
-    env.AddPostAction("$BUILD_DIR/FrameworkArduino/core_esp8266_postmortem.c.o", dummy_ets_printf)
-    env.AddPostAction("$BUILD_DIR/FrameworkArduino/core_esp8266_postmortem.cpp.o", dummy_ets_printf)
-
-# patch lwip1 sources conditionally:
-# https://github.com/xoseperez/espurna/issues/1610
-patch_lwip()

code/platformio.ini

@@ -90,7 +90,7 @@ framework = arduino
 board_build.flash_mode = dout
 monitor_speed = 115200
 upload_speed = 115200
-extra_scripts = pre:extra_script_pre.py, extra_scripts.py
+extra_scripts = pre:scripts/pio_pre.py, scripts/pio_main.py
 lib_extra_dirs =
     ${common.shared_libdeps_dir}
 

code/scripts/espurna_utils/__init__.py

@@ -0,0 +1,6 @@
+from .checks import check_cppcheck, check_printsize
+from .float_support import remove_float_support
+from .ldscripts import ldscripts_inject_libpath
+from .lwip import lwip_inject_patcher
+from .postmortem import dummy_ets_printf
+from .git import app_inject_revision

code/scripts/espurna_utils/checks.py

@@ -0,0 +1,29 @@
+import os
+
+from .display import Color, clr, print_filler, print_warning
+
+
+def check_printsize(target, source, env):
+    (binary,) = target
+    path = binary.get_abspath()
+    size = os.stat(path).st_size
+    print(clr(Color.LIGHT_BLUE, "Binary size: {} bytes".format(size)))
+
+    # Warn 1MB variants about exceeding OTA size limit
+    flash_size = int(env.BoardConfig().get("upload.maximum_size", 0))
+    if (flash_size == 1048576) and (size >= 512000):
+        print_filler("*", color=Color.LIGHT_YELLOW, err=True)
+        print_warning(
+            "File is too large for OTA! Here you can find instructions on how to flash it:"
+        )
+        print_warning(
+            "https://github.com/xoseperez/espurna/wiki/TwoStepUpdates",
+            color=Color.LIGHT_CYAN,
+        )
+        print_filler("*", color=Color.LIGHT_YELLOW, err=True)
+
+
+def check_cppcheck(target, source, env):
+    print_warning("Started cppcheck...\n")
+    call(["cppcheck", os.getcwd() + "/espurna", "--force", "--enable=all"])
+    print_warning("Finished cppcheck...\n")

code/scripts/espurna_utils/display.py

@@ -0,0 +1,40 @@
+from __future__ import print_function
+
+import sys
+import click
+
+
+class Color(object):
+    BLACK = "\x1b[1;30m"
+    RED = "\x1b[1;31m"
+    GREEN = "\x1b[1;32m"
+    YELLOW = "\x1b[1;33m"
+    BLUE = "\x1b[1;34m"
+    MAGENTA = "\x1b[1;35m"
+    CYAN = "\x1b[1;36m"
+    WHITE = "\x1b[1;37m"
+    LIGHT_GREY = "\x1b[0;30m"
+    LIGHT_RED = "\x1b[0;31m"
+    LIGHT_GREEN = "\x1b[0;32m"
+    LIGHT_YELLOW = "\x1b[0;33m"
+    LIGHT_BLUE = "\x1b[0;34m"
+    LIGHT_MAGENTA = "\x1b[0;35m"
+    LIGHT_CYAN = "\x1b[0;36m"
+    LIGHT_WHITE = "\x1b[0;37m"
+
+
+def clr(color, text):
+    return color + str(text) + "\x1b[0m"
+
+
+def print_warning(message, color=Color.LIGHT_YELLOW):
+    print(clr(color, message), file=sys.stderr)
+
+
+def print_filler(fill, color=Color.WHITE, err=False):
+    width, _ = click.get_terminal_size()
+    if len(fill) > 1:
+        fill = fill[0]
+
+    out = sys.stderr if err else sys.stdout
+    print(clr(color, fill * width), file=out)

code/scripts/espurna_utils/float_support.py

@@ -0,0 +1,8 @@
+def remove_float_support(env):
+
+    flags = " ".join(env["LINKFLAGS"])
+    flags = flags.replace("-u _printf_float", "")
+    flags = flags.replace("-u _scanf_float", "")
+    newflags = flags.split()
+
+    env.Replace(LINKFLAGS=newflags)

code/scripts/espurna_utils/git.py

@@ -0,0 +1,23 @@
+import os
+import subprocess
+
+def git(*args):
+    cmd = ["git"]
+    cmd.extend(args)
+    proc = subprocess.Popen(
+        cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True
+    )
+    return proc.stdout.readlines()[0].strip()
+
+def app_inject_revision(env):
+    revision = ""
+    try:
+        revision = "\\\"{}\\\"".format(git("rev-parse", "--short=8", "HEAD"))
+    except: # pylint: disable=broad-except
+        pass
+
+    # Note: code expects this as undefined when empty
+    if revision:
+        env.Append(CPPDEFINES=[
+            ("APP_REVISION", revision)
+        ])

code/scripts/espurna_utils/ldscripts.py

@@ -0,0 +1,24 @@
+import os
+
+
+def ldscripts_inject_libpath(env):
+
+    platform = env.PioPlatform()
+    framework_dir = platform.get_package_dir("framework-arduinoespressif8266")
+
+    # espressif8266@1.5.0 did not append this directory into the LIBPATH
+    libpath_sdk = os.path.join(framework_dir, "tools", "sdk", "ld")
+    env.Append(LIBPATH=[libpath_sdk])
+
+    libpath_base = os.path.join("$PROJECT_DIR", "..", "dist", "ld")
+    env.Append(LIBPATH=[os.path.join(libpath_base, "pre_2.5.0")])
+
+    # local.eagle.app.v6.common.ld exists only with Core >2.5.0
+    def check_local_ld(target, source, env):
+        local_ld = env.subst(
+            os.path.join("$BUILD_DIR", "ld", "local.eagle.app.v6.common.ld")
+        )
+        if os.path.exists(local_ld):
+            env.Prepend(LIBPATH=[os.path.join(libpath_base, "latest")])
+
+    env.AddPreAction(os.path.join("$BUILD_DIR", "firmware.elf"), check_local_ld)

code/scripts/espurna_utils/lwip.py

@@ -0,0 +1,56 @@
+import os
+
+
+def lwip_inject_patcher(env):
+    # ignore when building with lwip2
+    if "lwip_gcc" not in env["LIBS"]:
+        return
+
+    platform = env.PioPlatform()
+    framework_dir = platform.get_package_dir("framework-arduinoespressif8266")
+    toolchain_prefix = os.path.join(
+        platform.get_package_dir("toolchain-xtensa"), "bin", "xtensa-lx106-elf-"
+    )
+
+    patch_action = env.VerboseAction(
+        " ".join(
+            [
+                "-patch",
+                "-u",
+                "-N",
+                "-d",
+                os.path.join(framework_dir, "tools", "sdk", "lwip"),
+                os.path.join("src", "core", "tcp_out.c"),
+                env.subst(
+                    os.path.join(
+                        "$PROJECT_DIR",
+                        "..",
+                        "dist",
+                        "patches",
+                        "lwip_mtu_issue_1610.patch",
+                    )
+                ),
+            ]
+        ),
+        "Patching lwip source",
+    )
+    build_action = env.VerboseAction(
+        " ".join(
+            [
+                "make",
+                "-C",
+                os.path.join(framework_dir, "tools", "sdk", "lwip", "src"),
+                "install",
+                "TOOLS_PATH={}".format(toolchain_prefix),
+                "LWIP_LIB=liblwip_gcc.a",
+            ]
+        ),
+        "Rebuilding lwip",
+    )
+
+    patcher = env.Alias("patch-lwip", None, patch_action)
+    builder = env.Alias("build-lwip", patcher, build_action)
+    if os.environ.get("ESPURNA_PIO_PATCH_ISSUE_1610"):
+        env.Depends("$BUILD_DIR/${PROGNAME}.elf", builder)
+    env.AlwaysBuild(patcher)
+    env.AlwaysBuild(builder)

code/scripts/espurna_utils/postmortem.py

@@ -0,0 +1,13 @@
+def dummy_ets_printf(target, source, env):
+    (postmortem_src_file,) = source
+    (postmortem_obj_file,) = target
+
+    cmd = ["xtensa-lx106-elf-objcopy"]
+
+    # recent Core switched to cpp+newlib & ets_printf_P
+    cmd.extend(["--redefine-sym", "ets_printf=dummy_ets_printf"])
+    cmd.extend(["--redefine-sym", "ets_printf_P=dummy_ets_printf"])
+
+    cmd.append(postmortem_obj_file.get_abspath())
+    env.Execute(env.VerboseAction(" ".join(cmd), "Removing ets_printf / ets_printf_P"))
+    env.Depends(postmortem_obj_file, "$BUILD_DIR/src/dummy_ets_printf.c.o")

code/scripts/pio_main.py

@@ -0,0 +1,38 @@
+# Run this script every time building an env:
+
+from espurna_utils import (
+    check_printsize,
+    remove_float_support,
+    ldscripts_inject_libpath,
+    lwip_inject_patcher,
+    app_inject_revision,
+    dummy_ets_printf
+)
+
+Import("env", "projenv")
+
+# Always show warnings for project code
+projenv.ProcessUnFlags("-w")
+
+# 2.4.0 and up
+remove_float_support(env)
+ldscripts_inject_libpath(env)
+
+# two-step update hint when using 1MB boards
+env.AddPostAction("$BUILD_DIR/${PROGNAME}.bin", check_printsize)
+
+# disable postmortem printing to the uart. another one is in eboot, but this is what causes the most harm
+if "DISABLE_POSTMORTEM_STACKDUMP" in env["CPPFLAGS"]:
+    env.AddPostAction(
+        "$BUILD_DIR/FrameworkArduino/core_esp8266_postmortem.c.o", dummy_ets_printf
+    )
+    env.AddPostAction(
+        "$BUILD_DIR/FrameworkArduino/core_esp8266_postmortem.cpp.o", dummy_ets_printf
+    )
+
+# patch lwip1 sources conditionally:
+# https://github.com/xoseperez/espurna/issues/1610
+lwip_inject_patcher(env)
+
+# when using git, add -DAPP_REVISION=(git-commit-hash)
+app_inject_revision(projenv)

code/extra_script_pre.py → code/scripts/pio_pre.py

@@ -71,11 +71,17 @@ def ensure_platform_updated():
             print("updating platform packages", file=sys.stderr)
             PIO_PLATFORM.update_packages()
     except Exception:
-        print("Warning: no connection, cannot check for outdated packages", file=sys.stderr)
+        print(
+            "Warning: no connection, cannot check for outdated packages",
+            file=sys.stderr,
+        )
+
 
 # latest toolchain is still optional with PIO (TODO: recheck after 2.6.0!)
 # also updates arduino core git to the latest master commit
-if TRAVIS and (env.GetProjectOption("platform") == CONFIG.get("common", "arduino_core_git")):
+if TRAVIS and (
+    env.GetProjectOption("platform") == CONFIG.get("common", "arduino_core_git")
+):
     ensure_platform_updated()
 
 # to speed-up build process, install libraries in either global or local shared storage