Build with platformio 4 (#1805)

- update file paths: .pioenvs -> .pio/build, .piolibdeps -> .pio/libdeps
- modify envs to use common settings
- enable shared libs in travis and ota scripts

.travis.yml

@@ -7,7 +7,6 @@ cache:
   directories:
   - "~/.npm"
   - "~/.platformio"
-  - "$TRAVIS_BUILD_DIR/code/.piolibdeps"
 install:
 - pip install -U platformio
 - npm install -g npm@latest
@@ -16,6 +15,7 @@ env:
   global:
   - BUILDER_TOTAL_THREADS=4
   - ESPURNA_PIO_PATCH_ISSUE_1610=y
+  - ESPURNA_PIO_SHARED_LIBRARIES=y
 script:
 - cd code && ./build.sh -p &&  cd ..
 stages:

code/.gitignore

@@ -1,15 +1,13 @@
-.clang_complete
-core_version.h
-custom.h
-.DS_Store
-.gcc-flags.json
-.pioenvs
-.piolibdeps
-.python-version
-.travis.yml
-.vscode
-.vscode/.browse.c_cpp.db*
-.vscode/c_cpp_properties.json
-.vscode/launch.json
-.pioenvs
-.piolibdeps
+.clang_complete
+core_version.h
+custom.h
+.DS_Store
+.gcc-flags.json
+.python-version
+.travis.yml
+.vscode
+.vscode/.browse.c_cpp.db*
+.vscode/c_cpp_properties.json
+.vscode/launch.json
+.pio
+libraries/

code/build.sh

@@ -128,9 +128,9 @@ build_environments() {
     for environment in $environments; do
         echo -n "* espurna-$version-$environment.bin --- "
         platformio run --silent --environment $environment || exit 1
-        stat_bytes .pioenvs/$environment/firmware.bin
+        stat_bytes .pio/build/$environment/firmware.bin
         [[ "${TRAVIS_BUILD_STAGE_NAME}" = "Test" ]] || \
-            mv .pioenvs/$environment/firmware.bin $destination/espurna-$version/espurna-$version-$environment.bin
+            mv .pio/build/$environment/firmware.bin $destination/espurna-$version/espurna-$version-$environment.bin
     done
     echo "--------------------------------------------------------------"
 }

code/debug.sh

@@ -49,7 +49,7 @@ done
 
 # check environment folder
 if [ ! -f $ELF ]; then
-    ELF=.pioenvs/$ENVIRONMENT/firmware.elf
+    ELF=.pio/build/$ENVIRONMENT/firmware.elf
 fi
 if [ ! -f $ELF ]; then
     echo "Could not find ELF file for the selected environment: $ELF"

code/extra_script_libdeps.py

@@ -0,0 +1,73 @@
+from __future__ import print_function
+
+Import("env")
+
+import os
+import sys
+
+
+TRAVIS = os.environ.get("TRAVIS")
+
+
+class ExtraScriptError(Exception):
+    pass
+
+
+# Most portable way, without depending on platformio internals
+def subprocess_libdeps(lib_deps, storage=None, silent=True):
+    import subprocess
+
+    args = [env.subst("$PYTHONEXE"), "-mplatformio", "lib"]
+    if not storage:
+        args.append("-g")
+    else:
+        args.extend(["-d", storage])
+    args.append("install")
+    if silent:
+        args.append("-s")
+
+    args.extend(lib_deps)
+
+    subprocess.check_call(args)
+
+
+# Avoid spawning pio lib every time, hook into the LibraryManager API (sort-of internal)
+def library_manager_libdeps(lib_deps, storage=None):
+    from platformio.managers.lib import LibraryManager
+    from platformio.project.helpers import get_project_global_lib_dir
+
+    if not storage:
+        manager = LibraryManager(get_project_global_lib_dir())
+    else:
+        manager = LibraryManager(storage)
+
+    for lib in lib_deps:
+        if manager.get_package_dir(*manager.parse_pkg_uri(lib)):
+            continue
+        print("installing: {}".format(lib), file=sys.stderr)
+        manager.install(lib)
+
+
+def get_shared_libdeps_dir(section, name):
+    cfg = env.GetProjectConfig()
+
+    if not cfg.has_option(section, name):
+        raise ExtraScriptError("{}.{} is required to be set".format(section, name))
+
+    opt = cfg.get(section, name)
+
+    if not opt in env.GetProjectOption("lib_extra_dirs"):
+        raise ExtraScriptError("lib_extra_dirs must contain {}.{}".format(section, name))
+
+    return os.path.join(env["PROJECT_DIR"], opt)
+
+
+if os.environ.get("ESPURNA_PIO_SHARED_LIBRARIES"):
+    if TRAVIS:
+        storage = None
+        print("using global library storage", file=sys.stderr)
+    else:
+        storage = get_shared_libdeps_dir("common", "shared_libdeps_dir")
+        print("using shared library storage: ", storage, file=sys.stderr)
+
+    subprocess_libdeps(env.GetProjectOption("lib_deps"), storage)

code/libraries/README

@@ -0,0 +1 @@
+Shared lib_deps storage, see code/extra_script_libdeps.py

code/memanalyzer.py

@@ -103,7 +103,11 @@ def run(env_, modules_):
     flags = ""
     for k, v in modules_.items():
         flags += "-D{}_SUPPORT={:d} ".format(k, v)
-    command = "ESPURNA_BOARD=\"WEMOS_D1_MINI_RELAYSHIELD\" ESPURNA_FLAGS=\"{}\" platformio run --silent --environment {} 2>/dev/null".format(flags, env_)
+    os_env = os.environ.copy()
+    os_env["ESPURNA_BOARD"] = "WEMOS_D1_MINI_RELAYSHIELD"
+    os_env["ESPURNA_FLAGS"] = flags
+    os_env["ESPURNA_PIO_SHARED_LIBRARIES"] = "y"
+    command = "platformio run --silent --environment {} 2>/dev/null".format(env_)
     subprocess.check_call(command, shell=True)
 
 
@@ -215,8 +219,8 @@ if __name__ == '__main__':
 
     # Build the core without modules to get base memory usage
     run(env, modules)
-    base = analyse_memory(".pioenvs/{}/firmware.elf".format(env))
-    base['size'] = file_size(".pioenvs/{}/firmware.bin".format(env))
+    base = analyse_memory(".pio/build/{}/firmware.elf".format(env))
+    base['size'] = file_size(".pio/build/{}/firmware.bin".format(env))
     calc_free(base)
     print(output_format.format(
             "CORE" if args.core == 1 else "DEFAULT",
@@ -235,8 +239,8 @@ if __name__ == '__main__':
 
         modules[module] = 1
         run(env, modules)
-        results[module] = analyse_memory(".pioenvs/{}/firmware.elf".format(env))
-        results[module]['size'] = file_size(".pioenvs/{}/firmware.bin".format(env))
+        results[module] = analyse_memory(".pio/build/{}/firmware.elf".format(env))
+        results[module]['size'] = file_size(".pio/build/{}/firmware.bin".format(env))
         calc_free(results[module])
         modules[module] = 0
 
@@ -257,8 +261,8 @@ if __name__ == '__main__':
         for module in test_modules:
             modules[module] = 1
         run(env, modules)
-        total = analyse_memory(".pioenvs/{}/firmware.elf".format(env))
-        total['size'] = file_size(".pioenvs/{}/firmware.bin".format(env))
+        total = analyse_memory(".pio/build/{}/firmware.elf".format(env))
+        total['size'] = file_size(".pio/build/{}/firmware.bin".format(env))
         calc_free(total)
 
         print(output_format.format(

code/ota.py

@@ -240,8 +240,8 @@ def boardname(board):
 
 
 def store(device, env):
-    source = ".pioenvs/{}/firmware.elf".format(env)
-    destination = ".pioenvs/elfs/{}.elf".format(boardname(device).lower())
+    source = ".pio/build/{}/firmware.elf".format(env)
+    destination = ".pio/build/elfs/{}.elf".format(boardname(device).lower())
 
     dst_dir = os.path.dirname(destination)
     if not os.path.exists(dst_dir):
@@ -257,6 +257,7 @@ def run(device, env):
     environ["ESPURNA_BOARD"] = device["board"]
     environ["ESPURNA_AUTH"] = device["auth"]
     environ["ESPURNA_FLAGS"] = device["flags"]
+    environ["ESPURNA_PIO_SHARED_LIBRARIES"] = "y"
 
     command = ("platformio", "run", "--silent", "--environment", env, "-t", "upload")
     subprocess.check_call(command, env=environ)

code/symbols.sh

@@ -47,7 +47,7 @@ if [ $ENVIRONMENT == "" ]; then
     help
     exit 1
 fi
-ELF=.pioenvs/$ENVIRONMENT/firmware.elf
+ELF=.pio/build/$ENVIRONMENT/firmware.elf
 if [ ! -f $ELF ]; then
     echo "Could not find ELF file for the selected environment: $ELF"
     exit 2