pio: add build-and-copy, more configuration options for version

Multiple ways to specify version string through environment variables:
- `ESPURNA_BUILD_FULL_VERSION` to set full version string
By default it is empty, and the version is combined using the values specified below
- `ESPURNA_BUILD_VERSION` to modify the first part of the version string (1.15.0-dev)
By default, uses espurna/config/version.h APP_VERSION value
- `ESPURNA_BUILD_REVISION` to specify revision part of the version string
For example 12345678, which is expanded as either .git12345678 or -git12345678
(depending on whether the version string contains hyphen)
By default, use git to retrieve the first 8 characters of the current HEAD SHA value
- `ESPURNA_BUILD_VERSION_SUFFIX` to specify build metadata part of the string
For example nightly20210607, which is added to the full version as 1.15.0-dev+nightly20210607
Empty by defauld

Adds -t build-and-copy which uses the values above to copy firmware.bin, configurable with:
- `ESPURNA_BUILD_NAME` to set the suffix of the filename.
By default, uses the $PIOENV (aka the string after the env: in the .ini file)
- `ESPURNA_BUILD_DESTINATION` to specify where to copy the .bin files
By default, uses $PROJECT_DIR

Resulting file is stored at:
${ESPURNA_BUILD_DESTINATION}/${ESPURNA_BUILD_FULL_VERSION}/espurna-${ESPURNA_BUILD_FULL_VERSION}-${ESPURNA_BUILD_NAME}.bin

In addition, modify generate_release_sh.py to use the new environment variables.

code/espurna/config/version.h

@@ -4,8 +4,22 @@
 
 #pragma once
 
+#ifndef APP_NAME
 #define APP_NAME                "ESPURNA"
+#endif
+
+#ifndef APP_VERSION
 #define APP_VERSION             "1.15.0-dev"
+#endif
+
+#ifndef APP_AUTHOR
 #define APP_AUTHOR              "xose.perez@gmail.com"
+#endif
+
+#ifndef APP_WEBSITE
 #define APP_WEBSITE             "http://tinkerman.cat"
+#endif
+
+#ifndef CFG_VERSION
 #define CFG_VERSION             6
+#endif

code/espurna/utils.cpp

@@ -79,14 +79,7 @@ const String& getCoreRevision() {
 }
 
 const char* getVersion() {
-    static const char version[] {
-#if defined(APP_REVISION)
-        APP_VERSION APP_REVISION
-#else
-        APP_VERSION
-#endif
-    };
-
+    static const char version[] = APP_VERSION;
     return version;
 }
 

code/scripts/espurna_utils/__init__.py

@@ -4,7 +4,7 @@
 # Copyright (C) 2016-2019 by Xose Pérez <xose dot perez at gmail dot com>
 #
 # ldscripts, lwip patching, updated postmortem flags and git support
-# Copyright (C) 2019 by Maxim Prokhorov <prokhorov dot max at outlook dot com>
+# Copyright (C) 2019-2021 by Maxim Prokhorov <prokhorov dot max at outlook dot com>
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -19,21 +19,24 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+from .build import firmware_destination, app_add_target_build_and_copy
 from .checks import check_cppcheck, check_printsize
+from .flags import app_inject_flags
 from .float_support import remove_float_support
 from .ldscripts import ldscripts_inject_libpath
 from .postmortem import dummy_ets_printf
-from .git import app_inject_revision
-from .release import copy_release
-from .flags import app_inject_flags
+from .version import app_inject_version, app_full_version_for_env
 
 __all__ = [
+    "app_add_target_build_and_copy",
+    "app_full_version_for_env",
+    "app_inject_flags",
+    "app_inject_version",
+    "app_version",
     "check_cppcheck",
     "check_printsize",
-    "remove_float_support",
-    "ldscripts_inject_libpath",
     "dummy_ets_printf",
-    "app_inject_revision",
-    "app_inject_flags",
-    "copy_release",
+    "firmware_destination",
+    "ldscripts_inject_libpath",
+    "remove_float_support",
 ]

code/scripts/espurna_utils/build.py

@@ -0,0 +1,64 @@
+import atexit
+import os
+import shutil
+import tempfile
+import functools
+
+from .display import print_warning
+from .version import app_full_version_for_env
+
+
+def try_remove(path):
+    try:
+        os.remove(path)
+    except:  # pylint: disable=bare-except
+        print_warning("Please manually remove the file `{}`".format(path))
+
+
+# emulate .ino concatenation to speed up compilation times
+def merge_cpp(sources, output):
+    with tempfile.TemporaryFile() as tmp:
+        tmp.write(b"// !!! Automatically generated file; DO NOT EDIT !!! \n")
+        tmp.write(b'#include "espurna.h"\n')
+        for source in sources:
+            src_include = '#include "{}"\n'.format(source)
+            tmp.write(src_include.encode("utf-8"))
+
+        tmp.seek(0)
+
+        with open(output, "wb") as fobj:
+            shutil.copyfileobj(tmp, fobj)
+        atexit.register(try_remove, output)
+
+
+def firmware_prefix(env):
+    return "espurna-{}".format(app_full_version_for_env(env))
+
+
+# generate an common name for the current build
+def firmware_filename(env):
+    suffix = "{}.bin".format(env["ESPURNA_BUILD_NAME"] or env["PIOENV"])
+    return "-".join([firmware_prefix(env), suffix])
+
+
+def firmware_destination(env):
+    destdir = env["ESPURNA_BUILD_DESTINATION"] or env["PROJECT_DIR"]
+    subdir = os.path.join(destdir, firmware_prefix(env))
+
+    return os.path.join(subdir, firmware_filename(env))
+
+
+def app_add_target_build_and_copy(env):
+    from SCons.Script import Copy
+
+    copy_dest = firmware_destination(env)
+    copy = env.Command(
+        copy_dest, "${BUILD_DIR}/${PROGNAME}.bin", Copy("$TARGET", "$SOURCE")
+    )
+    env.AddTarget(
+        "build-and-copy",
+        copy_dest,
+        actions=None,  # command invocation already handles this
+        title="Build firmware.bin and store a copy",
+        description="Build and store firmware.bin as $ESPURNA_BUILD_DESTINATION/espurna-<version>-$ESPURNA_BUILD_NAME.bin (default destination is $PROJECT_DIR)",
+    )

code/scripts/espurna_utils/git.py

@@ -1,24 +0,0 @@
-import os
-import subprocess
-
-def git(*args):
-    cmd = ["git"]
-    cmd.extend(args)
-    proc = subprocess.Popen(
-        cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True
-    )
-    return proc.stdout.readlines()[0].strip()
-
-def app_inject_revision(env):
-    revision = env.get("ESPURNA_RELEASE_REVISION", "")
-    if not revision:
-        try:
-            revision = ".git" + git("rev-parse", "--short=8", "HEAD")
-        except: # pylint: disable=broad-except
-            pass
-
-    # Note: code expects this as undefined when empty
-    if revision:
-        env.Append(CPPDEFINES=[
-            ("APP_REVISION", "\\\"{}\\\"".format(revision))
-        ])

code/scripts/espurna_utils/release.py

@@ -1,49 +0,0 @@
-import atexit
-import os
-import shutil
-import tempfile
-
-from .display import print_warning
-
-
-def try_remove(path):
-    try:
-        os.remove(path)
-    except:  # pylint: disable=bare-except
-        print_warning("Please manually remove the file `{}`".format(path))
-
-
-def copy_release(target, source, env):
-    # target filename and subdir for release files
-    name = env["ESPURNA_RELEASE_NAME"]
-    version = env["ESPURNA_RELEASE_VERSION"]
-    destdir = env["ESPURNA_RELEASE_DESTINATION"]
-
-    if not name or not version or not destdir:
-        raise ValueError("Cannot set up release without release variables present")
-
-    if not os.path.exists(destdir):
-        os.makedirs(destdir)
-
-    dest = os.path.join(
-        destdir, "espurna-{version}-{name}.bin".format(version=version, name=name)
-    )
-    src = env.subst("$BUILD_DIR/${PROGNAME}.bin")
-
-    shutil.copy(src, dest)
-
-
-# emulate .ino concatenation to speed up compilation times
-def merge_cpp(sources, output):
-    with tempfile.TemporaryFile() as tmp:
-        tmp.write(b"// !!! Automatically generated file; DO NOT EDIT !!! \n")
-        tmp.write(b'#include "espurna.h"\n')
-        for source in sources:
-            src_include = '#include "{}"\n'.format(source)
-            tmp.write(src_include.encode('utf-8'))
-
-        tmp.seek(0)
-
-        with open(output, "wb") as fobj:
-            shutil.copyfileobj(tmp, fobj)
-        atexit.register(try_remove, output)

code/scripts/espurna_utils/version.py

@@ -0,0 +1,89 @@
+import os
+import functools
+import subprocess
+
+from .display import print_warning
+
+
+try:
+    cached = functools.cache
+except AttributeError:
+    cached = functools.lru_cache(None)
+
+
+@cached
+def app_revision():
+    def git(*args):
+        cmd = ["git"]
+        cmd.extend(args)
+        proc = subprocess.Popen(
+            cmd, stderr=subprocess.PIPE, stdout=subprocess.PIPE, universal_newlines=True
+        )
+        return proc.stdout.readlines()[0].strip()
+
+    revision = None
+    try:
+        revision = git("rev-parse", "--short=8", "HEAD")
+    except subprocess.CalledProcessError:
+        pass
+    except FileNotFoundError:
+        pass
+
+    return revision
+
+
+@cached
+def app_version(version_h):
+    version = None
+    with open(version_h, "r") as f:
+        for line in f:
+            if "define" in line and "APP_VERSION" in line:
+                version = line.split(" ")[-1]
+                version = version.strip().replace('"', "")
+                break
+
+    return version
+
+
+def app_version_for_env(env):
+    return env.get("ESPURNA_BUILD_VERSION") or app_version(
+        os.path.join(env.get("PROJECT_DIR"), "espurna/config/version.h")
+    )
+
+
+def app_revision_for_env(env):
+    return env.get("ESPURNA_BUILD_REVISION") or app_revision()
+
+
+def app_suffix_for_env(env):
+    return env.get("ESPURNA_BUILD_VERSION_SUFFIX", "")
+
+
+def app_combined_version(env):
+    version = app_version_for_env(env)
+    if not version:
+        raise ValueError("Version string cannot be empty")
+
+    revision = app_revision_for_env(env)
+    if revision:
+        # handle both 1.2.3-dev.git... and 1.2.3-git...
+        # and avoid 1.2.3.git... that cannot be parsed by the semantic_version module
+        middle = ".git" if "-" in version else "-git"
+        version = middle.join([version, revision])
+
+    suffix = app_suffix_for_env(env)
+    if suffix:
+        version = "+".join([version, suffix])
+
+    return version
+
+
+def app_full_version_for_env(env):
+    return env.get("ESPURNA_BUILD_FULL_VERSION") or app_combined_version(env)
+
+
+def app_inject_version(env):
+    def inject_string(env, flag, value):
+        env.Append(CPPDEFINES=[(flag, '\\"{}\\"'.format(value))])
+
+    inject_string(env, "APP_VERSION", app_full_version_for_env(env))

code/scripts/generate_release_sh.py

@@ -15,14 +15,12 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-import os
 import argparse
 import re
 import shlex
 import configparser
 import collections
 
-CI = "true" == os.environ.get("CI", "false")
 Build = collections.namedtuple("Build", "env extends build_flags src_build_flags")
 
 
@@ -96,12 +94,11 @@ def generate_lines(builds, ignore):
             flags.append('PLATFORMIO_BUILD_FLAGS="{}"'.format(build.build_flags))
         if build.src_build_flags:
             flags.append('ESPURNA_FLAGS="{}"'.format(build.src_build_flags))
-        flags.append('ESPURNA_RELEASE_NAME="{env}"'.format(env=build.env))
-        flags.append("ESPURNA_BUILD_SINGLE_SOURCE=1")
+        flags.append('ESPURNA_BUILD_NAME="{env}"'.format(env=build.env))
 
         cmd = ["env"]
         cmd.extend(flags)
-        cmd.extend(["pio", "run", "-e", build.extends, "-s", "-t", "release"])
+        cmd.extend(["pio", "run", "-e", build.extends, "-s", "-t", "build-and-copy"])
 
         line = " ".join(cmd)
 
@@ -123,22 +120,50 @@ def every(seq, nth, total):
         index = (index + 1) % total
 
 
-if __name__ == "__main__":
-    if not CI:
-        raise ValueError("* Not in CI *")
-
+def parse_args():
     parser = argparse.ArgumentParser()
-    parser.add_argument("--version", required=True)
-    parser.add_argument("--destination", required=True)
-    parser.add_argument("--ignore", action="append")
-    args = parser.parse_args()
+    parser.add_argument(
+        "--destination", help="Where to place the resulting .bin", required=True
+    )
+    parser.add_argument(
+        "--single-source",
+        help="Combine .cpp files into one to speed up compilation",
+        default=True,
+    )
+    parser.add_argument(
+        "--ignore", help="Do not build envs that contain the string(s)", action="append"
+    )
+
+    builder_thread = parser.add_argument_group(
+        title="Builder thread control for CI parallel builds"
+    )
+    builder_thread.add_argument("--builder-thread", type=int, required=True)
+    builder_thread.add_argument("--builder-total-threads", type=int, required=True)
+
+    full_version = parser.add_argument_group(
+        title="Fully replace the version string for the build system"
+    )
+    full_version.add_argument("--full-version")
+
+    version_parts = parser.add_argument_group(
+        "Replace parts of the version string that would have been detected by the build system"
+    )
+    version_parts.add_argument("--version")
+    version_parts.add_argument("--revision")
+    version_parts.add_argument("--suffix")
+
+    parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
 
     Config = configparser.ConfigParser()
     with open("platformio.ini", "r") as f:
         Config.read_file(f)
 
-    builder_total_threads = int(os.environ["BUILDER_TOTAL_THREADS"])
-    builder_thread = int(os.environ["BUILDER_THREAD"])
+    builder_total_threads = args.builder_total_threads
+    builder_thread = args.builder_thread
     if builder_thread >= builder_total_threads:
         raise ValueError("* Builder thread index out of range *")
 
@@ -146,9 +171,18 @@ if __name__ == "__main__":
 
     print("#!/bin/bash")
     print("set -e -x")
-    print('export ESPURNA_RELEASE_VERSION="{}"'.format(args.version))
-    print('export ESPURNA_RELEASE_DESTINATION="{}"'.format(args.destination))
-    print('trap "ls -l ${ESPURNA_RELEASE_DESTINATION}" EXIT')
+    print('export ESPURNA_BUILD_DESTINATION="{}"'.format(args.destination))
+    print("export ESPURNA_BUILD_SINGLE_SOURCE={}".format(int(args.single_source)))
+    if args.full_version:
+        print('export ESPURNA_BUILD_FULL_VERSION="{}"'.format(args.full_version))
+    else:
+        if args.version:
+            print('export ESPURNA_BUILD_VERSION="{}"'.format(args.version))
+        if args.suffix:
+            print('export ESPURNA_BUILD_REVISION="{}"'.format(args.revision))
+        if args.suffix:
+            print('export ESPURNA_BUILD_VERSION_SUFFIX="{}"'.format(args.suffix))
+    print('trap "ls -R ${ESPURNA_BUILD_DESTINATION}" EXIT')
     print(
         'echo "Selected thread #{} out of {}"'.format(
             builder_thread + 1, builder_total_threads

code/scripts/pio_main.py

@@ -13,10 +13,10 @@ from espurna_utils import (
     check_printsize,
     remove_float_support,
     ldscripts_inject_libpath,
-    app_inject_revision,
+    app_inject_version,
     dummy_ets_printf,
     app_inject_flags,
-    copy_release,
+    app_add_target_build_and_copy
 )
 
 
@@ -46,11 +46,11 @@ if "DISABLE_POSTMORTEM_STACKDUMP" in env["CPPFLAGS"]:
         "$BUILD_DIR/FrameworkArduino/core_esp8266_postmortem.cpp.o", dummy_ets_printf
     )
 
-# when using git, add -DAPP_REVISION=(git-commit-hash)
-app_inject_revision(projenv)
+# handle add -DAPP_VERSION=... that was set and / or detected
+app_inject_version(projenv)
 
 # handle OTA board and flags here, since projenv is not available in pre-scripts
 app_inject_flags(projenv)
 
-# handle `-t release` when CI does a tagged build
-env.AlwaysBuild(env.Alias("release", "${BUILD_DIR}/${PROGNAME}.bin", copy_release))
+# handle when CI does a tagged build or user explicitly asked to store the firmware.bin
+app_add_target_build_and_copy(projenv)

code/scripts/pio_pre.py

@@ -19,8 +19,7 @@ import sys
 
 from SCons.Script import ARGUMENTS
 
-from espurna_utils.release import merge_cpp
-
+from espurna_utils.build import merge_cpp
 
 CI = "true" == os.environ.get("CI")
 PIO_PLATFORM = env.PioPlatform()
@@ -112,14 +111,19 @@ if ESPURNA_OTA_PORT:
 else:
     env.Replace(UPLOAD_PROTOCOL="esptool")
 
-# handle `-t release` parameters
-if CI:
-    env.Append(
-        ESPURNA_RELEASE_REVISION=os.environ.get("ESPURNA_RELEASE_REVISION", ""),
-        ESPURNA_RELEASE_NAME=os.environ.get("ESPURNA_RELEASE_NAME", ""),
-        ESPURNA_RELEASE_VERSION=os.environ.get("ESPURNA_RELEASE_VERSION", ""),
-        ESPURNA_RELEASE_DESTINATION=os.environ.get("ESPURNA_RELEASE_DESTINATION", ""),
-    )
+# handle `-t build-and-copy` parameters
+env.Append(
+    # what is the name suffix of the .bin
+    ESPURNA_BUILD_NAME=os.environ.get("ESPURNA_BUILD_NAME", ""),
+    # where to copy the resulting .bin
+    ESPURNA_BUILD_DESTINATION=os.environ.get("ESPURNA_BUILD_DESTINATION", ""),
+    # set the full string for the build, no need to change individual parts
+    ESPURNA_BUILD_FULL_VERSION=os.environ.get("ESPURNA_BUILD_FULL_VERSION", ""),
+    # or, replace parts of the version string that would've been auto-detected
+    ESPURNA_BUILD_VERSION=os.environ.get("ESPURNA_BUILD_VERSION", ""),
+    ESPURNA_BUILD_REVISION=os.environ.get("ESPURNA_BUILD_REVISION", ""),
+    ESPURNA_BUILD_VERSION_SUFFIX=os.environ.get("ESPURNA_BUILD_VERSION_SUFFIX", ""),
+)
 
 # updates arduino core git to the latest master commit
 if CI: