PIO: fix env definitions (again) (#2212)

* base envs

* CI: speed up release process (since we still want some .bin files)

* fixup! CI: speed up release process (since we still want some .bin files)

* release dry run

* fixup! release dry run

* adjust

* fix .bin name

* it works

* minor cleanup for current git

* use pio suggestion about ldscript, reduce build_flags impact

* fix linker

* parse ${vars} instead of ignoring them

* add filtering and override file (sneak peak into tasmota's pio config)

* don't generate secure client (for now)

* formatting

* codacy

.travis.yml

@@ -29,10 +29,10 @@ jobs:
     - stage: Test Host
     - stage: Test WebUI
     - stage: Test PlatformIO Build
-      env: TEST_ENV=travis-2_3_0
-    - env: TEST_ENV=travis-latest
+      env: TEST_ENV=esp8266-4m-base
+    - env: TEST_ENV=esp8266-latest-4m-base
     - env: >-
-        TEST_ENV=travis-git TEST_EXTRA_ARGS="-a
+        TEST_ENV=esp8266-git-4m-base TEST_EXTRA_ARGS="-a
         test/build/extra/secure_client.h"
     - stage: Release
       env: BUILDER_THREAD=0
@@ -40,6 +40,7 @@ jobs:
     - env: BUILDER_THREAD=2
     - env: BUILDER_THREAD=3
 before_deploy:
+  - mkdir -p firmware/
   - mv firmware/*/espurna-*.bin firmware/
 deploy:
   provider: releases

code/build.sh

@@ -24,6 +24,8 @@ version=$(grep -E '^#define APP_VERSION' $version_file | awk '{print $3}' | sed
 script_build_environments=true
 script_build_webui=true
 
+release_mode=false
+
 if ${TRAVIS:-false}; then
     git_revision=${TRAVIS_COMMIT::7}
     git_tag=${TRAVIS_TAG}
@@ -42,26 +44,12 @@ if [[ -n $git_tag ]]; then
     trap "git checkout -- $version_file" EXIT
 fi
 
-par_build=false
-par_thread=${BUILDER_THREAD:-0}
-par_total_threads=${BUILDER_TOTAL_THREADS:-4}
-if [ ${par_thread} -ne ${par_thread} -o \
-    ${par_total_threads} -ne ${par_total_threads} ]; then
-    echo "Parallel threads should be a number."
-    exit
-fi
-if [ ${par_thread} -ge ${par_total_threads} ]; then
-    echo "Current thread is greater than total threads. Doesn't make sense"
-    exit
-fi
-
 # Available environments
 list_envs() {
-    grep env: platformio.ini | sed 's/\[env:\(.*\)\]/\1/g'
+    grep -E '^\[env:' platformio.ini | sed 's/\[env:\(.*\)\]/\1/g'
 }
 
-travis=$(list_envs | grep travis | sort)
-available=$(list_envs | grep -Ev -- '-ota$|-ssl$|^travis' | sort)
+available=$(list_envs | grep -Ev -- '-ota$|-ssl$|-secure-client.*$|^esp8266-.*base$' | sort)
 
 # Functions
 print_available() {
@@ -81,20 +69,6 @@ print_environments() {
 }
 
 set_default_environments() {
-    # Hook to build in parallel when using travis
-    if [[ "${TRAVIS_BUILD_STAGE_NAME}" = "Release" ]] && ${par_build}; then
-        environments=$(echo ${available} | \
-            awk -v par_thread=${par_thread} -v par_total_threads=${par_total_threads} \
-            '{ for (i = 1; i <= NF; i++) if (++j % par_total_threads == par_thread ) print $i; }')
-        return
-    fi
-
-    # Only build travisN
-    if [[ "${TRAVIS_BUILD_STAGE_NAME}" = "Test" ]]; then
-        environments=$travis
-        return
-    fi
-
     # Fallback to all available environments
     environments=$available
 }
@@ -119,6 +93,14 @@ build_webui() {
     fi
 }
 
+build_release() {
+    echo "--------------------------------------------------------------"
+    echo "Building release images..."
+    python scripts/generate_release_sh.py --ignore secure-client $version > release.sh
+    bash release.sh
+    echo "--------------------------------------------------------------"
+}
+
 build_environments() {
     echo "--------------------------------------------------------------"
     echo "Building firmware images..."
@@ -146,19 +128,16 @@ Options:
 
   -f VALUE    Filter build stage by name to skip it
               Supported VALUEs are "environments" and "webui"
-              Can be specified multiple times
+              Can be specified multiple times. 
+  -r          Release mode
+              Generate build list through an external script.
   -l          Print available environments
   -d VALUE    Destination to move .bin files after building environments
-  -p          Enable parallel build
-              Depends on following exported variables:
-                BUILDER_THREAD=<number>        (default 0...4)
-                BUILDER_TOTAL_THREADS=<number> (default 4)
-              When building platformio environments, will only pick every <BUILDER_THREAD>th
   -h          Display this message
 EOF
 }
 
-while getopts "f:lpd:h" opt; do
+while getopts "f:lrpd:h" opt; do
   case $opt in
     f)
         case "$OPTARG" in
@@ -170,12 +149,12 @@ while getopts "f:lpd:h" opt; do
         print_available
         exit
         ;;
-    p)
-        par_build=true
-        ;;
     d)
         destination=$OPTARG
         ;;
+    r)
+        release_mode=true
+        ;;
     h)
         print_getopts_help
         exit
@@ -201,9 +180,10 @@ if $script_build_environments ; then
         set_default_environments
     fi
 
-    if ${CI:-false}; then
-        print_environments
+    if $release_mode ; then
+        build_release
+    else
+        build_environments
     fi
-
-    build_environments
 fi
+

code/espurna/config/hardware.h

@@ -1693,6 +1693,43 @@
     #define ECH1560_MISO_PIN    5
     #define ECH1560_INVERTED    0
 
+// -----------------------------------------------------------------------------
+// PZEM004T
+// -----------------------------------------------------------------------------
+
+#elif defined(GENERIC_PZEM004T)
+
+    // Info
+    #define MANUFACTURER        "GENERIC"
+    #define DEVICE              "PZEM004T"
+
+    #define PZEM004T_SUPPORT     1
+
+    #define ALEXA_SUPPORT              0
+    #define DEBUG_SERIAL_SUPPORT       0
+
+// -----------------------------------------------------------------------------
+// ESP-01 generic esp8266 board with 512 kB flash
+// -----------------------------------------------------------------------------
+
+#elif defined(GENERIC_ESP01_512KB)
+
+    // Info
+    #define MANUFACTURER        "GENERIC"
+    #define DEVICE              "ESP01_512KB"
+
+    // Relays
+    #define RELAY1_PIN          2
+    #ifndef RELAY1_TYPE
+    #define RELAY1_TYPE         RELAY_TYPE_NORMAL
+    #endif
+
+    // No need for OTA
+    #define OTA_WEB_SUPPORT          0
+    #define OTA_ARDUINOOTA_SUPPORT   0
+    #define OTA_CLIENT               OTA_CLIENT_NONE
+
+
 // -----------------------------------------------------------------------------
 // ESPLive
 // https://github.com/ManCaveMade/ESP-Live
@@ -4558,27 +4595,6 @@
     #define RELAY3_TYPE             RELAY_TYPE_NORMAL
     #define RELAY4_TYPE             RELAY_TYPE_NORMAL
 
-// -----------------------------------------------------------------------------
-// ESP-01 generic esp8266 board with 512 kB flash
-// -----------------------------------------------------------------------------
-
-#elif defined(GENERIC_ESP01_512KB)
-
-    // Info
-    #define MANUFACTURER        "GENERIC"
-    #define DEVICE              "ESP01_512KB"
-
-    // Relays
-    #define RELAY1_PIN          2
-    #ifndef RELAY1_TYPE
-    #define RELAY1_TYPE         RELAY_TYPE_NORMAL
-    #endif
-
-    // No need for OTA
-    #define OTA_WEB_SUPPORT          0
-    #define OTA_ARDUINOOTA_SUPPORT   0
-    #define OTA_CLIENT               OTA_CLIENT_NONE
-
 // -----------------------------------------------------------------------------
 
 #else

code/espurna/ota_asynctcp.ino

@@ -146,7 +146,7 @@ void _otaClientOnConnect(void* arg, AsyncClient* client) {
 
     #if ASYNC_TCP_SSL_ENABLED
         const auto check = getSetting("otaScCheck", OTA_SECURE_CLIENT_CHECK);
-        if ((check == SECURE_CLIENT_CHECK_FINGERPRINT) && (443 == _ota_url->port)) {
+        if ((check == SECURE_CLIENT_CHECK_FINGERPRINT) && (443 == ota_client->url.port)) {
             uint8_t fp[20] = {0};
             sslFingerPrintArray(getSetting("otaFP", OTA_FINGERPRINT).c_str(), fp);
             SSL * ssl = client->getSSL();

code/espurna/ota_httpupdate.ino

@@ -38,15 +38,6 @@ Copyright (C) 2019 by Maxim Prokhorov <prokhorov dot max at outlook dot com>
 // Configuration templates
 // -----------------------------------------------------------------------------
 
-template <typename T>
-void _otaFollowRedirects(const std::true_type&, T& instance) {
-    instance.followRedirects(true);
-}
-
-template <typename T>
-void _otaFollowRedirects(const std::false_type&, T& instance) {
-}
-
 template <typename T>
 t_httpUpdate_return _otaClientUpdate(const std::true_type&, T& instance, WiFiClient* client, const String& url) {
     return instance.update(*client, url);
@@ -58,12 +49,6 @@ t_httpUpdate_return _otaClientUpdate(const std::false_type&, T& instance, WiFiCl
 }
 
 namespace ota {
-    template <typename T>
-    using has_followRedirects_t = decltype(std::declval<T>().followRedirects(std::declval<bool>()));
-
-    template <typename T>
-    using has_followRedirects = is_detected<has_followRedirects_t, T>;
-
     template <typename T>
     using has_WiFiClient_argument_t = decltype(std::declval<T>().update(std::declval<WiFiClient&>(), std::declval<const String&>()));
 
@@ -114,19 +99,16 @@ SecureClientConfig _ota_sc_config {
 // Generic update methods
 // -----------------------------------------------------------------------------
 
-void _otaClientRunUpdater(WiFiClient* client, const String& url, const String& fp = "") {
-
-    UNUSED(client);
-    UNUSED(fp);
+void _otaClientRunUpdater(__attribute__((unused)) WiFiClient* client, const String& url, __attribute__((unused)) const String& fp = "") {
 
     // Disabling EEPROM rotation to prevent writing to EEPROM after the upgrade
     eepromRotate(false);
 
     DEBUG_MSG_P(PSTR("[OTA] Downloading %s ...\n"), url.c_str());
 
-    // TODO: support currentVersion (string arg after 'url')
-    // NOTE: ESPhttpUpdate.update(..., fp) will **always** fail with empty fingerprint
-    _otaFollowRedirects(ota::has_followRedirects<decltype(ESPhttpUpdate)>{}, ESPhttpUpdate);
+    #if not defined(ARDUINO_ESP8266_RELEASE_2_3_0)
+        ESPhttpUpdate.setFollowRedirects(HTTPC_STRICT_FOLLOW_REDIRECTS);
+    #endif
 
     ESPhttpUpdate.rebootOnUpdate(false);
     t_httpUpdate_return result = HTTP_UPDATE_NO_UPDATES;
@@ -139,6 +121,7 @@ void _otaClientRunUpdater(WiFiClient* client, const String& url, const String& f
             result = ESPhttpUpdate.update(url);
         }
     #else
+    // TODO: support currentVersion (string arg after 'url')
     // TODO: implement through callbacks?
     //       see https://github.com/esp8266/Arduino/pull/6796
         result = _otaClientUpdate(ota::has_WiFiClient_argument<decltype(ESPhttpUpdate)>{}, ESPhttpUpdate, client, url);

code/ota.py

@@ -409,7 +409,7 @@ def get_platformio_env(arduino_core, size):
         )
     if arduino_core != "2_3_0":
         env_prefix = "{}-{}".format(env_prefix, arduino_core)
-    return "{env_prefix}-{size:d}m-ota".format(env_prefix=env_prefix, size=size)
+    return "{env_prefix}-{size:d}m-base".format(env_prefix=env_prefix, size=size)
 
 
 def main(args):

code/scripts/espurna_utils/__init__.py

@@ -25,6 +25,7 @@ from .ldscripts import ldscripts_inject_libpath
 from .lwip import lwip_inject_patcher
 from .postmortem import dummy_ets_printf
 from .git import app_inject_revision
+from .release import copy_release
 from .flags import app_inject_flags
 
 __all__ = [
@@ -36,4 +37,5 @@ __all__ = [
     "dummy_ets_printf",
     "app_inject_revision",
     "app_inject_flags",
+    "copy_release",
 ]

code/scripts/espurna_utils/release.py

@@ -0,0 +1,22 @@
+import os
+import shutil
+
+def copy_release(target, source, env):
+    # target filename and subdir for release files
+    name = env["ESPURNA_NAME"]
+    version = env["ESPURNA_VERSION"]
+
+    if not name or not version:
+        raise ValueError("Cannot set up release without release variables present")
+
+    destdir = os.path.join(env.subst("$PROJECT_DIR"), "..", "firmware", version)
+    if not os.path.exists(destdir):
+        os.makedirs(destdir)
+
+    dest = os.path.join(
+        destdir, "espurna-{version}-{name}.bin".format(version=version, name=name)
+    )
+    src = env.subst("$BUILD_DIR/${PROGNAME}.bin")
+
+    shutil.copy(src, dest)
+

code/scripts/generate_release_sh.py

@@ -0,0 +1,138 @@
+import os
+import argparse
+import re
+import shlex
+import configparser
+import collections
+
+CI = any([os.environ.get("TRAVIS"), os.environ.get("CI")])
+Build = collections.namedtuple("Build", "env extends build_flags src_build_flags")
+
+
+def expand_variables(cfg, value):
+    RE_VARS = re.compile("\$\{.*?\}")
+
+    for var in RE_VARS.findall(value):
+        section, option = var.replace("${", "").replace("}", "").split(".", 1)
+        value = value.replace(var, expand_variables(cfg, cfg.get(section, option)))
+
+    return value
+
+
+def get_builds(cfg):
+    RE_NEWLINE = re.compile("\r\n|\n")
+    BASE_BUILD_FLAGS = set(
+        shlex.split(expand_variables(cfg, cfg.get("env", "build_flags")))
+    )
+
+    for section in cfg.sections():
+        if (not section.startswith("env:")) or (
+            section.startswith("env:esp8266-") and section.endswith("-base")
+        ):
+            continue
+
+        build_flags = None
+        src_build_flags = None
+
+        try:
+            build_flags = cfg.get(section, "build_flags")
+            build_flags = RE_NEWLINE.sub(" ", build_flags).strip()
+            build_flags = " ".join(
+                BASE_BUILD_FLAGS ^ set(shlex.split(expand_variables(cfg, build_flags)))
+            )
+        except configparser.NoOptionError:
+            pass
+
+        try:
+            src_build_flags = cfg.get(section, "src_build_flags")
+            src_build_flags = RE_NEWLINE.sub(" ", src_build_flags).strip()
+            src_build_flags = expand_variables(cfg, src_build_flags)
+        except configparser.NoOptionError:
+            pass
+
+        yield Build(
+            section.replace("env:", ""),
+            cfg.get(section, "extends").replace("env:", ""),
+            build_flags,
+            src_build_flags,
+        )
+
+
+def find_any(string, values):
+    for value in values:
+        if value in string:
+            return True
+
+    return False
+
+
+def generate_lines(builds, ignore):
+    cores = []
+    generic = []
+
+    for build in builds:
+        if find_any(build.env, ignore):
+            continue
+
+        flags = []
+        if build.build_flags:
+            flags.append('PLATFORMIO_BUILD_FLAGS="{}"'.format(build.build_flags))
+        if build.src_build_flags:
+            flags.append('ESPURNA_FLAGS="{}"'.format(build.src_build_flags))
+        flags.append('ESPURNA_NAME="{env}"'.format(env=build.env))
+
+        cmd = ["env"]
+        cmd.extend(flags)
+        cmd.extend(["pio", "run", "-e", build.extends, "-s", "-t", "release"])
+
+        line = " ".join(cmd)
+
+        # push core variants to the front as they definetly include global build_flags
+        output = generic
+        if "ESPURNA_CORE" in build.src_build_flags:
+            output = cores
+
+        output.append(line)
+
+    return cores + generic
+
+
+def every(seq, nth, total):
+    index = 0
+    for value in seq:
+        if index == nth:
+            yield value
+        index = (index + 1) % total
+
+
+if __name__ == "__main__":
+    if not CI:
+        raise ValueError("* Not in CI *")
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("version")
+    parser.add_argument("--ignore", action="append")
+    args = parser.parse_args()
+
+    Config = configparser.ConfigParser()
+    with open("platformio.ini", "r") as f:
+        Config.read_file(f)
+
+    builder_total_threads = int(os.environ["BUILDER_TOTAL_THREADS"])
+    builder_thread = int(os.environ["BUILDER_THREAD"])
+    if builder_thread >= builder_total_threads:
+        raise ValueError("* Builder thread index out of range *")
+
+    builds = every(get_builds(Config), builder_thread, builder_total_threads)
+
+    print("#!/bin/bash")
+    print("set -e -x")
+    print('export ESPURNA_VERSION="{}"'.format(args.version))
+    print('trap "ls -l ${TRAVIS_BUILD_DIR}/firmware/${ESPURNA_VERSION}" EXIT')
+    print(
+        'echo "Selected thread #{} out of {}"'.format(
+            builder_thread + 1, builder_total_threads
+        )
+    )
+    for line in generate_lines(builds, args.ignore or ()):
+        print(line)

code/scripts/pio_main.py

@@ -17,6 +17,7 @@ from espurna_utils import (
     app_inject_revision,
     dummy_ets_printf,
     app_inject_flags,
+    copy_release,
 )
 
 Import("env", "projenv")
@@ -50,3 +51,6 @@ app_inject_revision(projenv)
 
 # handle OTA board and flags here, since projenv is not available in pre-scripts
 app_inject_flags(projenv)
+
+# handle `-t release` when CI does a tagged build
+env.AlwaysBuild(env.Alias("release", "${BUILD_DIR}/${PROGNAME}.bin", copy_release))

code/scripts/pio_pre.py

@@ -94,9 +94,11 @@ def ensure_platform_updated():
         log("Warning: no connection, cannot check for outdated packages", verbose=True)
 
 
-# handle OTA uploads
-# using env instead of ini to fix platformio ini changing hash on every change
+# handle build flags through os environment.
+# using env instead of ini to avoid platformio ini changing hash on every change
 env.Append(
+    ESPURNA_VERSION=os.environ.get("ESPURNA_VERSION", ""),
+    ESPURNA_NAME=os.environ.get("ESPURNA_NAME", ""),
     ESPURNA_BOARD=os.environ.get("ESPURNA_BOARD", ""),
     ESPURNA_AUTH=os.environ.get("ESPURNA_AUTH", ""),
     ESPURNA_FLAGS=os.environ.get("ESPURNA_FLAGS", "")
@@ -110,12 +112,13 @@ if ESPURNA_OTA_PORT:
 else:
     env.Replace(UPLOAD_PROTOCOL="esptool")
 
-# latest toolchain is still optional with PIO (TODO: recheck after 2.6.0!)
-# also updates arduino core git to the latest master commit
-if TRAVIS and (
-    env.GetProjectOption("platform") == CONFIG.get("common", "arduino_core_git")
-):
-    ensure_platform_updated()
+# updates arduino core git to the latest master commit
+if TRAVIS:
+    package_overrides = env.GetProjectOption("platform_packages")
+    for package in package_overrides:
+        if "https://github.com/esp8266/Arduino.git" in package:
+            ensure_platform_updated()
+            break
 
 # to speed-up build process, install libraries in either global or local shared storage
 if os.environ.get("ESPURNA_PIO_SHARED_LIBRARIES"):

travis_script.sh

@@ -12,7 +12,7 @@ elif [ "${TRAVIS_BUILD_STAGE_NAME}" = "Test PlatformIO Build" ]; then
     # shellcheck disable=SC2086
     scripts/test_build.py -e "$TEST_ENV" $TEST_EXTRA_ARGS
 elif [ "${TRAVIS_BUILD_STAGE_NAME}" = "Release" ]; then
-    ./build.sh -p
+    ./build.sh -r
 else
     echo -e "\e[1;33mUnknown stage name, exiting!\e[0m"
     exit 1