pio: trying to work out 6.0.0 changes

.github/workflows/push.yml

@@ -49,8 +49,6 @@ jobs:
     strategy:
       matrix:
         env: [esp8266-4m-base, esp8266-4m-latest-base]
-    env:
-      ESPURNA_PIO_SHARED_LIBRARIES: "1"
     steps:
     - uses: actions/checkout@v2
     - uses: actions/setup-python@v2

code/ota.py

@@ -301,7 +301,6 @@ def run(device, env):
     environ["ESPURNA_BOARD"] = device["board"]
     environ["ESPURNA_AUTH"] = device["auth"]
     environ["ESPURNA_FLAGS"] = device["flags"]
-    environ["ESPURNA_PIO_SHARED_LIBRARIES"] = "1"
 
     command = ("platformio", "run", "--silent", "--environment", env, "-t", "upload")
     subprocess.check_call(command, env=environ)

code/platformio.ini

@@ -125,13 +125,20 @@ debug_flags =
 #   -DDEBUG_ESP_PORT=Serial
 #   -DDEBUG_ESP_HWDT
 
-shared_libdeps_dir = libraries/
 
-git_platform_packages =
-    platformio/framework-arduinoespressif8266 @ https://github.com/esp8266/Arduino.git
-    mcspr/toolchain-xtensa @ ~5.100300.211127
+# ------------------------------------------------------------------------------
+# LIBRARIES: required dependencies
+#   Please note that we don't always use the latest version of a library.
 
-lib_deps =
+#   We don't use `lib_deps` directly, since PlatformIO 6.0.0 our libraries
+#   will always be installed into the environment-specific directory.
+#   To install them in a single location, there's a manual update & install.
+#
+#   See https://github.com/platformio/platformio-core/issues/4206
+# ------------------------------------------------------------------------------
+
+shared_lib_dir = libraries/
+shared_lib_deps =
     https://github.com/me-no-dev/ESPAsyncTCP.git#7e9ed22
     bblanchon/ArduinoJson@^5.13.4
     https://github.com/marvinroger/async-mqtt-client#v0.8.1
@@ -158,6 +165,10 @@ lib_deps =
     https://github.com/BoschSensortec/BSEC-Arduino-library.git#v1.6.1480
     adafruit/Adafruit NeoPixel@^1.7.0
 
+git_platform_packages =
+    platformio/framework-arduinoespressif8266 @ https://github.com/esp8266/Arduino.git
+    mcspr/toolchain-xtensa @ ~5.100300.211127
+
 # ------------------------------------------------------------------------------
 # COMMON ENVIRONMENT SETTINGS:
 # ------------------------------------------------------------------------------
@@ -169,16 +180,9 @@ build_flags = ${common.build_flags}
 monitor_speed = 115200
 upload_speed = 115200
 extra_scripts = pre:scripts/pio_pre.py, scripts/pio_main.py
-lib_extra_dirs =
-    ${common.shared_libdeps_dir}
-
-# ------------------------------------------------------------------------------
-# LIBRARIES: required dependencies
-#   Please note that we don't always use the latest version of a library.
-# ------------------------------------------------------------------------------
-lib_deps =
-    ${common.lib_deps}
 
+lib_extra_dirs =
+    libraries/
 lib_ignore =
     AsyncTCP
     Brzo I2C

code/scripts/memanalyzer.py

@@ -101,7 +101,6 @@ def run(prefix, env, modules, debug):
     os_env = os.environ.copy()
     os_env["PLATFORMIO_SRC_BUILD_FLAGS"] = flags
     os_env["PLATFORMIO_BUILD_CACHE_DIR"] = "test/pio_cache"
-    os_env["ESPURNA_PIO_SHARED_LIBRARIES"] = "y"
 
     command = [os.path.join(prefix, "platformio"), "run"]
     if not debug:

code/scripts/pio_pre.py

@@ -9,14 +9,20 @@
 
 # Run this script every time building an env BEFORE platform-specific code is loaded
 
+import logging
 import os
+import shutil
 import sys
 
-from SCons.Script import Import, ARGUMENTS
+from SCons.Script import Delete, Move, Import, ARGUMENTS
 
 from espurna_utils import check_env
 from espurna_utils.build import app_add_builder_single_source, app_add_target_build_re2c
 
+from platformio.package.manager.library import LibraryPackageManager
+from platformio.package.meta import PackageSpec
+
+
 Import("env")
 env = globals()["env"]
 
@@ -36,30 +42,35 @@ def log(message, verbose=False, file=sys.stderr):
         print(message, file=file)
 
 
-# Most portable way, without depending on platformio internals
-def subprocess_libdeps(lib_deps, storage, verbose=False):
-    import subprocess
+def get_shared_libdeps(config, section="common", name="shared_lib_deps"):
+    raw = config.getraw(section, name)
+    return config.parse_multi_values(raw)
+
+
+def get_shared_libdir(config, section="common", name="shared_lib_dir"):
+    return config.getraw(section, name)
 
-    args = [env.subst("$PYTHONEXE"), "-mplatformio", "lib", "-d", storage, "install"]
-    if not verbose:
-        args.append("-s")
 
-    args.extend(lib_deps)
-    subprocess.check_call(args)
+def migrate_libraries(storage):
+    target = env.Dir(f"$PROJECT_DIR/{storage}")
+    if target.exists():
+        return
 
+    old_lib_deps = env.Dir("$PROJECT_LIBDEPS_DIR/$PIOENV")
+    if not old_lib_deps.exists() or old_lib_deps.islink():
+        return
 
-def get_shared_libdeps_dir(section, name):
-    if not CONFIG.has_option(section, name):
-        raise ExtraScriptError("{}.{} is required to be set".format(section, name))
+    env.Execute(env.VerboseAction(Move(target, old_lib_deps), "Migrating $TARGET"))
 
-    opt = CONFIG.get(section, name)
 
-    if opt not in env.GetProjectOption("lib_extra_dirs"):
-        raise ExtraScriptError(
-            "lib_extra_dirs must contain {}.{}".format(section, name)
-        )
+def install_libraries(specs, storage, verbose=False):
+    lm = LibraryPackageManager(storage)
+    lm.set_log_level(logging.DEBUG if verbose else logging.INFO)
 
-    return os.path.join(env["PROJECT_DIR"], opt)
+    for spec in specs:
+        pkg = lm.get_package(spec)
+        if not pkg:
+            lm.install(spec, skip_dependencies=True)
 
 
 def ensure_platform_updated():
@@ -110,9 +121,11 @@ if CI:
             break
 
 # to speed-up build process, install libraries in a way they are shared between our envs
-if check_env("ESPURNA_PIO_SHARED_LIBRARIES", "0"):
-    storage = get_shared_libdeps_dir("common", "shared_libdeps_dir")
-    subprocess_libdeps(env.GetProjectOption("lib_deps"), storage, verbose=VERBOSE)
+# b/c lib_extra_dirs = ... was deprecated and global libs are not an option, just re-use the local custom lib storage
+# (...while it still works :/...)
+SHARED_LIBDIR = get_shared_libdir(CONFIG)
+migrate_libraries(SHARED_LIBDIR)
+install_libraries(get_shared_libdeps(CONFIG), SHARED_LIBDIR, verbose=VERBOSE)
 
 
 # tweak build system to ignore espurna.ino, but include user code